blob: 32499f42679777e8e05a0e207f7c792099d1b89f [file] [log] [blame]
Eric Laurent6d607012021-07-05 11:54:40 +02001/*
2**
3** Copyright 2021, The Android Open Source Project
4**
5** Licensed under the Apache License, Version 2.0 (the "License");
6** you may not use this file except in compliance with the License.
7** You may obtain a copy of the License at
8**
9** http://www.apache.org/licenses/LICENSE-2.0
10**
11** Unless required by applicable law or agreed to in writing, software
12** distributed under the License is distributed on an "AS IS" BASIS,
13** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14** See the License for the specific language governing permissions and
15** limitations under the License.
16*/
17
18
19#define LOG_TAG "Spatializer"
20//#define LOG_NDEBUG 0
21#include <utils/Log.h>
22
23#include <limits.h>
24#include <stdint.h>
25#include <sys/types.h>
26
27#include <android/content/AttributionSourceState.h>
Eric Laurent2be8b292021-08-23 09:44:33 -070028#include <android/sensor.h>
Eric Laurent6d607012021-07-05 11:54:40 +020029#include <audio_utils/fixedfft.h>
30#include <cutils/bitops.h>
Eric Laurent2be8b292021-08-23 09:44:33 -070031#include <hardware/sensors.h>
Eric Laurent6d607012021-07-05 11:54:40 +020032#include <media/audiohal/EffectsFactoryHalInterface.h>
Eric Laurent8a4259f2021-09-14 16:04:00 +020033#include <media/stagefright/foundation/AHandler.h>
34#include <media/stagefright/foundation/AMessage.h>
35#include <media/ShmemCompat.h>
Eric Laurent6d607012021-07-05 11:54:40 +020036#include <mediautils/ServiceUtilities.h>
37#include <utils/Thread.h>
38
39#include "Spatializer.h"
40
41namespace android {
42
43using aidl_utils::statusTFromBinderStatus;
44using aidl_utils::binderStatusFromStatusT;
45using android::content::AttributionSourceState;
46using binder::Status;
Eric Laurent2be8b292021-08-23 09:44:33 -070047using media::HeadTrackingMode;
48using media::Pose3f;
Eric Laurent6d607012021-07-05 11:54:40 +020049using media::SpatializationLevel;
Eric Laurent2be8b292021-08-23 09:44:33 -070050using media::SpatializationMode;
Ytai Ben-Tsvia16a9df2021-08-05 08:57:06 -070051using media::SpatializerHeadTrackingMode;
Eric Laurent2be8b292021-08-23 09:44:33 -070052using media::SensorPoseProvider;
53
Eric Laurent2be8b292021-08-23 09:44:33 -070054using namespace std::chrono_literals;
Eric Laurent6d607012021-07-05 11:54:40 +020055
56#define VALUE_OR_RETURN_BINDER_STATUS(x) \
57 ({ auto _tmp = (x); \
58 if (!_tmp.ok()) return aidl_utils::binderStatusFromStatusT(_tmp.error()); \
59 std::move(_tmp.value()); })
60
Eric Laurent6d607012021-07-05 11:54:40 +020061// ---------------------------------------------------------------------------
62
Eric Laurent8a4259f2021-09-14 16:04:00 +020063class Spatializer::EngineCallbackHandler : public AHandler {
64public:
65 EngineCallbackHandler(wp<Spatializer> spatializer)
66 : mSpatializer(spatializer) {
67 }
68
69 enum {
70 // Device state callbacks
71 kWhatOnFramesProcessed, // AudioEffect::EVENT_FRAMES_PROCESSED
72 kWhatOnHeadToStagePose, // SpatializerPoseController::Listener::onHeadToStagePose
73 kWhatOnActualModeChange, // SpatializerPoseController::Listener::onActualModeChange
74 };
75 static constexpr const char *kNumFramesKey = "numFrames";
76 static constexpr const char *kModeKey = "mode";
77 static constexpr const char *kTranslation0Key = "translation0";
78 static constexpr const char *kTranslation1Key = "translation1";
79 static constexpr const char *kTranslation2Key = "translation2";
80 static constexpr const char *kRotation0Key = "rotation0";
81 static constexpr const char *kRotation1Key = "rotation1";
82 static constexpr const char *kRotation2Key = "rotation2";
83
84 void onMessageReceived(const sp<AMessage> &msg) override {
85 switch (msg->what()) {
86 case kWhatOnFramesProcessed: {
87 sp<Spatializer> spatializer = mSpatializer.promote();
88 if (spatializer == nullptr) {
89 ALOGW("%s: Cannot promote spatializer", __func__);
90 return;
91 }
92 int numFrames;
93 if (!msg->findInt32(kNumFramesKey, &numFrames)) {
94 ALOGE("%s: Cannot find num frames!", __func__);
95 return;
96 }
97 if (numFrames > 0) {
98 spatializer->calculateHeadPose();
99 }
100 } break;
101 case kWhatOnHeadToStagePose: {
102 sp<Spatializer> spatializer = mSpatializer.promote();
103 if (spatializer == nullptr) {
104 ALOGW("%s: Cannot promote spatializer", __func__);
105 return;
106 }
107 std::vector<float> headToStage(sHeadPoseKeys.size());
108 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
109 if (!msg->findFloat(sHeadPoseKeys[i], &headToStage[i])) {
110 ALOGE("%s: Cannot find kTranslation0Key!", __func__);
111 return;
112 }
113 }
114 spatializer->onHeadToStagePoseMsg(headToStage);
115 } break;
116 case kWhatOnActualModeChange: {
117 sp<Spatializer> spatializer = mSpatializer.promote();
118 if (spatializer == nullptr) {
119 ALOGW("%s: Cannot promote spatializer", __func__);
120 return;
121 }
122 int mode;
123 if (!msg->findInt32(EngineCallbackHandler::kModeKey, &mode)) {
124 ALOGE("%s: Cannot find actualMode!", __func__);
125 return;
126 }
127 spatializer->onActualModeChangeMsg(static_cast<HeadTrackingMode>(mode));
128 } break;
129 default:
130 LOG_ALWAYS_FATAL("Invalid callback message %d", msg->what());
131 }
132 }
133private:
134 wp<Spatializer> mSpatializer;
135};
136
137const std::vector<const char *> Spatializer::sHeadPoseKeys = {
138 Spatializer::EngineCallbackHandler::kTranslation0Key,
139 Spatializer::EngineCallbackHandler::kTranslation1Key,
140 Spatializer::EngineCallbackHandler::kTranslation2Key,
141 Spatializer::EngineCallbackHandler::kRotation0Key,
142 Spatializer::EngineCallbackHandler::kRotation1Key,
143 Spatializer::EngineCallbackHandler::kRotation2Key,
144};
145
146// ---------------------------------------------------------------------------
Eric Laurent6d607012021-07-05 11:54:40 +0200147sp<Spatializer> Spatializer::create(SpatializerPolicyCallback *callback) {
148 sp<Spatializer> spatializer;
149
150 sp<EffectsFactoryHalInterface> effectsFactoryHal = EffectsFactoryHalInterface::create();
151 if (effectsFactoryHal == nullptr) {
152 ALOGW("%s failed to create effect factory interface", __func__);
153 return spatializer;
154 }
155
156 std::vector<effect_descriptor_t> descriptors;
157 status_t status =
Eric Laurent1c5e2e32021-08-18 18:50:28 +0200158 effectsFactoryHal->getDescriptors(FX_IID_SPATIALIZER, &descriptors);
Eric Laurent6d607012021-07-05 11:54:40 +0200159 if (status != NO_ERROR) {
160 ALOGW("%s failed to get spatializer descriptor, error %d", __func__, status);
161 return spatializer;
162 }
163 ALOG_ASSERT(!descriptors.empty(),
164 "%s getDescriptors() returned no error but empty list", __func__);
165
166 //TODO: get supported spatialization modes from FX engine or descriptor
167
168 sp<EffectHalInterface> effect;
169 status = effectsFactoryHal->createEffect(&descriptors[0].uuid, AUDIO_SESSION_OUTPUT_STAGE,
170 AUDIO_IO_HANDLE_NONE, AUDIO_PORT_HANDLE_NONE, &effect);
171 ALOGI("%s FX create status %d effect %p", __func__, status, effect.get());
172
173 if (status == NO_ERROR && effect != nullptr) {
174 spatializer = new Spatializer(descriptors[0], callback);
Eric Laurent2be8b292021-08-23 09:44:33 -0700175 if (spatializer->loadEngineConfiguration(effect) != NO_ERROR) {
176 spatializer.clear();
177 }
Eric Laurent6d607012021-07-05 11:54:40 +0200178 }
179
180 return spatializer;
181}
182
Eric Laurent2be8b292021-08-23 09:44:33 -0700183Spatializer::Spatializer(effect_descriptor_t engineDescriptor, SpatializerPolicyCallback* callback)
184 : mEngineDescriptor(engineDescriptor),
185 mPolicyCallback(callback) {
Eric Laurent6d607012021-07-05 11:54:40 +0200186 ALOGV("%s", __func__);
187}
188
Eric Laurent8a4259f2021-09-14 16:04:00 +0200189void Spatializer::onFirstRef() {
190 mLooper = new ALooper;
191 mLooper->setName("Spatializer-looper");
192 mLooper->start(
193 /*runOnCallingThread*/false,
194 /*canCallJava*/ false,
195 PRIORITY_AUDIO);
196
197 mHandler = new EngineCallbackHandler(this);
198 mLooper->registerHandler(mHandler);
199}
200
Eric Laurent6d607012021-07-05 11:54:40 +0200201Spatializer::~Spatializer() {
202 ALOGV("%s", __func__);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200203 if (mLooper != nullptr) {
204 mLooper->stop();
205 mLooper->unregisterHandler(mHandler->id());
206 }
207 mLooper.clear();
208 mHandler.clear();
Eric Laurent6d607012021-07-05 11:54:40 +0200209}
210
Eric Laurent2be8b292021-08-23 09:44:33 -0700211status_t Spatializer::loadEngineConfiguration(sp<EffectHalInterface> effect) {
212 ALOGV("%s", __func__);
213
214 std::vector<bool> supportsHeadTracking;
215 status_t status = getHalParameter<false>(effect, SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED,
216 &supportsHeadTracking);
217 if (status != NO_ERROR) {
218 return status;
219 }
220 mSupportsHeadTracking = supportsHeadTracking[0];
221
222 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_LEVELS, &mLevels);
223 if (status != NO_ERROR) {
224 return status;
225 }
226 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES,
227 &mSpatializationModes);
228 if (status != NO_ERROR) {
229 return status;
230 }
231 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS,
232 &mChannelMasks);
233 if (status != NO_ERROR) {
234 return status;
235 }
236 return NO_ERROR;
237}
238
239/** Gets the channel mask, sampling rate and format set for the spatializer input. */
240audio_config_base_t Spatializer::getAudioInConfig() const {
241 std::lock_guard lock(mLock);
242 audio_config_base_t config = AUDIO_CONFIG_BASE_INITIALIZER;
243 // For now use highest supported channel count
244 uint32_t maxCount = 0;
245 for ( auto mask : mChannelMasks) {
246 if (audio_channel_count_from_out_mask(mask) > maxCount) {
247 config.channel_mask = mask;
248 }
249 }
250 return config;
251}
252
Eric Laurent6d607012021-07-05 11:54:40 +0200253status_t Spatializer::registerCallback(
254 const sp<media::INativeSpatializerCallback>& callback) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700255 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200256 if (callback == nullptr) {
257 return BAD_VALUE;
258 }
259
260 sp<IBinder> binder = IInterface::asBinder(callback);
261 status_t status = binder->linkToDeath(this);
262 if (status == NO_ERROR) {
263 mSpatializerCallback = callback;
264 }
265 ALOGV("%s status %d", __func__, status);
266 return status;
267}
268
269// IBinder::DeathRecipient
270void Spatializer::binderDied(__unused const wp<IBinder> &who) {
271 {
Eric Laurent2be8b292021-08-23 09:44:33 -0700272 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200273 mLevel = SpatializationLevel::NONE;
274 mSpatializerCallback.clear();
275 }
276 ALOGV("%s", __func__);
277 mPolicyCallback->onCheckSpatializer();
278}
279
280// ISpatializer
281Status Spatializer::getSupportedLevels(std::vector<SpatializationLevel> *levels) {
282 ALOGV("%s", __func__);
283 if (levels == nullptr) {
284 return binderStatusFromStatusT(BAD_VALUE);
285 }
Eric Laurent6d607012021-07-05 11:54:40 +0200286 levels->push_back(SpatializationLevel::NONE);
Eric Laurent2be8b292021-08-23 09:44:33 -0700287 levels->insert(levels->end(), mLevels.begin(), mLevels.end());
Eric Laurent6d607012021-07-05 11:54:40 +0200288 return Status::ok();
289}
290
Eric Laurent2be8b292021-08-23 09:44:33 -0700291Status Spatializer::setLevel(SpatializationLevel level) {
Eric Laurent6d607012021-07-05 11:54:40 +0200292 ALOGV("%s level %d", __func__, (int)level);
293 if (level != SpatializationLevel::NONE
Eric Laurent2be8b292021-08-23 09:44:33 -0700294 && std::find(mLevels.begin(), mLevels.end(), level) == mLevels.end()) {
Eric Laurent6d607012021-07-05 11:54:40 +0200295 return binderStatusFromStatusT(BAD_VALUE);
296 }
297 sp<media::INativeSpatializerCallback> callback;
298 bool levelChanged = false;
299 {
Eric Laurent2be8b292021-08-23 09:44:33 -0700300 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200301 levelChanged = mLevel != level;
302 mLevel = level;
303 callback = mSpatializerCallback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700304
305 if (levelChanged && mEngine != nullptr) {
306 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL, std::vector<SpatializationLevel>{level});
307 }
Eric Laurent6d607012021-07-05 11:54:40 +0200308 }
309
310 if (levelChanged) {
311 mPolicyCallback->onCheckSpatializer();
312 if (callback != nullptr) {
313 callback->onLevelChanged(level);
314 }
315 }
316 return Status::ok();
317}
318
Eric Laurent2be8b292021-08-23 09:44:33 -0700319Status Spatializer::getLevel(SpatializationLevel *level) {
Eric Laurent6d607012021-07-05 11:54:40 +0200320 if (level == nullptr) {
321 return binderStatusFromStatusT(BAD_VALUE);
322 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700323 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200324 *level = mLevel;
325 ALOGV("%s level %d", __func__, (int)*level);
326 return Status::ok();
327}
328
Eric Laurentc87402b2021-09-17 16:49:42 +0200329Status Spatializer::isHeadTrackingSupported(bool *supports) {
330 ALOGV("%s mSupportsHeadTracking %d", __func__, mSupportsHeadTracking);
331 if (supports == nullptr) {
332 return binderStatusFromStatusT(BAD_VALUE);
333 }
334 std::lock_guard lock(mLock);
335 *supports = mSupportsHeadTracking;
336 return Status::ok();
337}
338
Eric Laurent6d607012021-07-05 11:54:40 +0200339Status Spatializer::getSupportedHeadTrackingModes(
Eric Laurent2be8b292021-08-23 09:44:33 -0700340 std::vector<SpatializerHeadTrackingMode>* modes) {
341 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200342 ALOGV("%s", __func__);
343 if (modes == nullptr) {
344 return binderStatusFromStatusT(BAD_VALUE);
345 }
Eric Laurent6d607012021-07-05 11:54:40 +0200346
Eric Laurent2be8b292021-08-23 09:44:33 -0700347 modes->push_back(SpatializerHeadTrackingMode::DISABLED);
348 if (mSupportsHeadTracking) {
349 if (mHeadSensor != nullptr) {
350 modes->push_back(SpatializerHeadTrackingMode::RELATIVE_WORLD);
351 if (mScreenSensor != nullptr) {
352 modes->push_back(SpatializerHeadTrackingMode::RELATIVE_SCREEN);
353 }
354 }
Eric Laurent6d607012021-07-05 11:54:40 +0200355 }
356 return Status::ok();
357}
358
Eric Laurent2be8b292021-08-23 09:44:33 -0700359Status Spatializer::setDesiredHeadTrackingMode(SpatializerHeadTrackingMode mode) {
360 ALOGV("%s mode %d", __func__, (int)mode);
361
362 if (!mSupportsHeadTracking) {
363 return binderStatusFromStatusT(INVALID_OPERATION);
364 }
365 std::lock_guard lock(mLock);
366 switch (mode) {
367 case SpatializerHeadTrackingMode::OTHER:
368 return binderStatusFromStatusT(BAD_VALUE);
369 case SpatializerHeadTrackingMode::DISABLED:
370 mDesiredHeadTrackingMode = HeadTrackingMode::STATIC;
371 break;
372 case SpatializerHeadTrackingMode::RELATIVE_WORLD:
373 mDesiredHeadTrackingMode = HeadTrackingMode::WORLD_RELATIVE;
374 break;
375 case SpatializerHeadTrackingMode::RELATIVE_SCREEN:
376 mDesiredHeadTrackingMode = HeadTrackingMode::SCREEN_RELATIVE;
377 break;
378 }
379
380 if (mPoseController != nullptr) {
381 mPoseController->setDesiredMode(mDesiredHeadTrackingMode);
382 }
383
384 return Status::ok();
385}
386
387Status Spatializer::getActualHeadTrackingMode(SpatializerHeadTrackingMode *mode) {
Eric Laurent6d607012021-07-05 11:54:40 +0200388 if (mode == nullptr) {
389 return binderStatusFromStatusT(BAD_VALUE);
390 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700391 std::lock_guard lock(mLock);
392 *mode = mActualHeadTrackingMode;
Eric Laurent6d607012021-07-05 11:54:40 +0200393 ALOGV("%s mode %d", __func__, (int)*mode);
394 return Status::ok();
395}
396
Ytai Ben-Tsvia16a9df2021-08-05 08:57:06 -0700397Status Spatializer::recenterHeadTracker() {
Eric Laurent780be4a2021-09-16 10:44:24 +0200398 if (!mSupportsHeadTracking) {
399 return binderStatusFromStatusT(INVALID_OPERATION);
400 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700401 std::lock_guard lock(mLock);
402 if (mPoseController != nullptr) {
403 mPoseController->recenter();
404 }
Eric Laurent6d607012021-07-05 11:54:40 +0200405 return Status::ok();
406}
407
408Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage) {
Eric Laurent6d607012021-07-05 11:54:40 +0200409 ALOGV("%s", __func__);
Eric Laurent780be4a2021-09-16 10:44:24 +0200410 if (!mSupportsHeadTracking) {
411 return binderStatusFromStatusT(INVALID_OPERATION);
412 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700413 std::optional<Pose3f> maybePose = Pose3f::fromVector(screenToStage);
414 if (!maybePose.has_value()) {
415 ALOGW("Invalid screenToStage vector.");
416 return binderStatusFromStatusT(BAD_VALUE);
417 }
418 std::lock_guard lock(mLock);
419 if (mPoseController != nullptr) {
420 mPoseController->setScreenToStagePose(maybePose.value());
421 }
Eric Laurent6d607012021-07-05 11:54:40 +0200422 return Status::ok();
423}
424
425Status Spatializer::release() {
426 ALOGV("%s", __func__);
427 bool levelChanged = false;
428 {
Eric Laurent2be8b292021-08-23 09:44:33 -0700429 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200430 if (mSpatializerCallback == nullptr) {
431 return binderStatusFromStatusT(INVALID_OPERATION);
432 }
433
434 sp<IBinder> binder = IInterface::asBinder(mSpatializerCallback);
435 binder->unlinkToDeath(this);
436 mSpatializerCallback.clear();
437
438 levelChanged = mLevel != SpatializationLevel::NONE;
439 mLevel = SpatializationLevel::NONE;
440 }
441
442 if (levelChanged) {
443 mPolicyCallback->onCheckSpatializer();
444 }
445 return Status::ok();
446}
447
Eric Laurent2be8b292021-08-23 09:44:33 -0700448Status Spatializer::setHeadSensor(int sensorHandle) {
449 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200450 if (!mSupportsHeadTracking) {
451 return binderStatusFromStatusT(INVALID_OPERATION);
452 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700453 std::lock_guard lock(mLock);
454 if (sensorHandle == ASENSOR_INVALID) {
455 mHeadSensor = nullptr;
456 } else {
457 mHeadSensor = VALUE_OR_RETURN_BINDER_STATUS(getSensorFromHandle(sensorHandle));
458 }
459 if (mPoseController != nullptr) {
460 mPoseController->setHeadSensor(mHeadSensor);
461 }
462 return Status::ok();
463}
464
465Status Spatializer::setScreenSensor(int sensorHandle) {
466 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200467 if (!mSupportsHeadTracking) {
468 return binderStatusFromStatusT(INVALID_OPERATION);
469 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700470 std::lock_guard lock(mLock);
471 if (sensorHandle == ASENSOR_INVALID) {
472 mScreenSensor = nullptr;
473 } else {
474 mScreenSensor = VALUE_OR_RETURN_BINDER_STATUS(getSensorFromHandle(sensorHandle));
475 }
476 if (mPoseController != nullptr) {
477 mPoseController->setScreenSensor(mScreenSensor);
478 }
479 return Status::ok();
480}
481
482Status Spatializer::setDisplayOrientation(float physicalToLogicalAngle) {
483 ALOGV("%s physicalToLogicalAngle %f", __func__, physicalToLogicalAngle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200484 if (!mSupportsHeadTracking) {
485 return binderStatusFromStatusT(INVALID_OPERATION);
486 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700487 std::lock_guard lock(mLock);
488 mDisplayOrientation = physicalToLogicalAngle;
489 if (mPoseController != nullptr) {
490 mPoseController->setDisplayOrientation(mDisplayOrientation);
491 }
Eric Laurent16ddaf42021-09-17 15:00:35 +0200492 if (mEngine != nullptr) {
493 setEffectParameter_l(
494 SPATIALIZER_PARAM_DISPLAY_ORIENTATION, std::vector<float>{physicalToLogicalAngle});
495 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700496 return Status::ok();
497}
498
499Status Spatializer::setHingeAngle(float hingeAngle) {
500 std::lock_guard lock(mLock);
501 ALOGV("%s hingeAngle %f", __func__, hingeAngle);
502 if (mEngine != nullptr) {
503 setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE, std::vector<float>{hingeAngle});
504 }
505 return Status::ok();
506}
507
508Status Spatializer::getSupportedModes(std::vector<SpatializationMode> *modes) {
509 ALOGV("%s", __func__);
510 if (modes == nullptr) {
511 return binderStatusFromStatusT(BAD_VALUE);
512 }
513 *modes = mSpatializationModes;
514 return Status::ok();
515}
516
Eric Laurent67816e32021-09-16 15:18:40 +0200517Status Spatializer::registerHeadTrackingCallback(
518 const sp<media::ISpatializerHeadTrackingCallback>& callback) {
519 ALOGV("%s callback %p", __func__, callback.get());
520 std::lock_guard lock(mLock);
521 if (!mSupportsHeadTracking) {
522 return binderStatusFromStatusT(INVALID_OPERATION);
523 }
524 mHeadTrackingCallback = callback;
525 return Status::ok();
526}
527
Eric Laurentc87402b2021-09-17 16:49:42 +0200528Status Spatializer::setParameter(int key, const std::vector<unsigned char>& value) {
529 ALOGV("%s key %d", __func__, key);
530 std::lock_guard lock(mLock);
531 status_t status = INVALID_OPERATION;
532 if (mEngine != nullptr) {
533 status = setEffectParameter_l(key, value);
534 }
535 return binderStatusFromStatusT(status);
536}
537
538Status Spatializer::getParameter(int key, std::vector<unsigned char> *value) {
Greg Kaiserf7249f82021-09-21 07:10:12 -0700539 ALOGV("%s key %d value size %d", __func__, key,
540 (value != nullptr ? (int)value->size() : -1));
Eric Laurentc87402b2021-09-17 16:49:42 +0200541 if (value == nullptr) {
George Burgess IV22386222021-09-22 12:09:31 -0700542 return binderStatusFromStatusT(BAD_VALUE);
Eric Laurentc87402b2021-09-17 16:49:42 +0200543 }
544 std::lock_guard lock(mLock);
545 status_t status = INVALID_OPERATION;
546 if (mEngine != nullptr) {
547 ALOGV("%s key %d mEngine %p", __func__, key, mEngine.get());
548 status = getEffectParameter_l(key, value);
549 }
550 return binderStatusFromStatusT(status);
551}
552
553Status Spatializer::getOutput(int *output) {
554 ALOGV("%s", __func__);
555 if (output == nullptr) {
556 binderStatusFromStatusT(BAD_VALUE);
557 }
558 std::lock_guard lock(mLock);
559 *output = VALUE_OR_RETURN_BINDER_STATUS(legacy2aidl_audio_io_handle_t_int32_t(mOutput));
560 ALOGV("%s got output %d", __func__, *output);
561 return Status::ok();
562}
563
Eric Laurent2be8b292021-08-23 09:44:33 -0700564// SpatializerPoseController::Listener
565void Spatializer::onHeadToStagePose(const Pose3f& headToStage) {
566 ALOGV("%s", __func__);
Eric Laurent780be4a2021-09-16 10:44:24 +0200567 LOG_ALWAYS_FATAL_IF(!mSupportsHeadTracking,
568 "onHeadToStagePose() called with no head tracking support!");
569
Eric Laurent2be8b292021-08-23 09:44:33 -0700570 auto vec = headToStage.toVector();
Eric Laurent8a4259f2021-09-14 16:04:00 +0200571 LOG_ALWAYS_FATAL_IF(vec.size() != sHeadPoseKeys.size(),
572 "%s invalid head to stage vector size %zu", __func__, vec.size());
573
574 sp<AMessage> msg =
575 new AMessage(EngineCallbackHandler::kWhatOnHeadToStagePose, mHandler);
576 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
577 msg->setFloat(sHeadPoseKeys[i], vec[i]);
578 }
579 msg->post();
580}
581
582void Spatializer::onHeadToStagePoseMsg(const std::vector<float>& headToStage) {
583 ALOGV("%s", __func__);
Eric Laurent67816e32021-09-16 15:18:40 +0200584 sp<media::ISpatializerHeadTrackingCallback> callback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700585 {
586 std::lock_guard lock(mLock);
Eric Laurent67816e32021-09-16 15:18:40 +0200587 callback = mHeadTrackingCallback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700588 if (mEngine != nullptr) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200589 setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
Eric Laurent2be8b292021-08-23 09:44:33 -0700590 }
591 }
592
593 if (callback != nullptr) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200594 callback->onHeadToSoundStagePoseUpdated(headToStage);
Eric Laurent2be8b292021-08-23 09:44:33 -0700595 }
596}
597
598void Spatializer::onActualModeChange(HeadTrackingMode mode) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200599 ALOGV("%s(%d)", __func__, (int)mode);
600 sp<AMessage> msg =
601 new AMessage(EngineCallbackHandler::kWhatOnActualModeChange, mHandler);
602 msg->setInt32(EngineCallbackHandler::kModeKey, static_cast<int>(mode));
603 msg->post();
604}
605
606void Spatializer::onActualModeChangeMsg(HeadTrackingMode mode) {
607 ALOGV("%s(%d)", __func__, (int) mode);
Eric Laurent67816e32021-09-16 15:18:40 +0200608 sp<media::ISpatializerHeadTrackingCallback> callback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700609 SpatializerHeadTrackingMode spatializerMode;
610 {
611 std::lock_guard lock(mLock);
612 if (!mSupportsHeadTracking) {
613 spatializerMode = SpatializerHeadTrackingMode::DISABLED;
614 } else {
615 switch (mode) {
616 case HeadTrackingMode::STATIC:
617 spatializerMode = SpatializerHeadTrackingMode::DISABLED;
618 break;
619 case HeadTrackingMode::WORLD_RELATIVE:
620 spatializerMode = SpatializerHeadTrackingMode::RELATIVE_WORLD;
621 break;
622 case HeadTrackingMode::SCREEN_RELATIVE:
623 spatializerMode = SpatializerHeadTrackingMode::RELATIVE_SCREEN;
624 break;
625 default:
626 LOG_ALWAYS_FATAL("Unknown mode: %d", mode);
627 }
628 }
629 mActualHeadTrackingMode = spatializerMode;
Eric Laurent67816e32021-09-16 15:18:40 +0200630 callback = mHeadTrackingCallback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700631 }
632 if (callback != nullptr) {
633 callback->onHeadTrackingModeChanged(spatializerMode);
634 }
635}
636
637/* static */
638ConversionResult<ASensorRef> Spatializer::getSensorFromHandle(int handle) {
639 ASensorManager* sensorManager =
640 ASensorManager_getInstanceForPackage("headtracker");
641 if (!sensorManager) {
642 ALOGE("Failed to get a sensor manager");
643 return base::unexpected(NO_INIT);
644 }
645 ASensorList sensorList;
646 int numSensors = ASensorManager_getSensorList(sensorManager, &sensorList);
647 for (int i = 0; i < numSensors; ++i) {
648 if (ASensor_getHandle(sensorList[i]) == handle) {
649 return sensorList[i];
650 }
651 }
652 return base::unexpected(BAD_VALUE);
653}
654
Eric Laurent6d607012021-07-05 11:54:40 +0200655status_t Spatializer::attachOutput(audio_io_handle_t output) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700656 std::shared_ptr<SpatializerPoseController> poseController;
657 {
658 std::lock_guard lock(mLock);
659 ALOGV("%s output %d mOutput %d", __func__, (int)output, (int)mOutput);
660 if (mOutput != AUDIO_IO_HANDLE_NONE) {
661 LOG_ALWAYS_FATAL_IF(mEngine == nullptr, "%s output set without FX engine", __func__);
662 // remove FX instance
663 mEngine->setEnabled(false);
664 mEngine.clear();
665 }
666 // create FX instance on output
667 AttributionSourceState attributionSource = AttributionSourceState();
668 mEngine = new AudioEffect(attributionSource);
669 mEngine->set(nullptr, &mEngineDescriptor.uuid, 0, Spatializer::engineCallback /* cbf */,
670 this /* user */, AUDIO_SESSION_OUTPUT_STAGE, output, {} /* device */,
671 false /* probe */, true /* notifyFramesProcessed */);
672 status_t status = mEngine->initCheck();
673 ALOGV("%s mEngine create status %d", __func__, (int)status);
674 if (status != NO_ERROR) {
675 return status;
676 }
677
678 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
679 std::vector<SpatializationLevel>{mLevel});
680 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
681 std::vector<SpatializerHeadTrackingMode>{mActualHeadTrackingMode});
682
683 mEngine->setEnabled(true);
684 mOutput = output;
685
Eric Laurent780be4a2021-09-16 10:44:24 +0200686 if (mSupportsHeadTracking) {
687 mPoseController = std::make_shared<SpatializerPoseController>(
688 static_cast<SpatializerPoseController::Listener*>(this), 10ms, 50ms);
689 LOG_ALWAYS_FATAL_IF(mPoseController == nullptr,
690 "%s could not allocate pose controller", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700691
Eric Laurent780be4a2021-09-16 10:44:24 +0200692 mPoseController->setDesiredMode(mDesiredHeadTrackingMode);
693 mPoseController->setHeadSensor(mHeadSensor);
694 mPoseController->setScreenSensor(mScreenSensor);
695 mPoseController->setDisplayOrientation(mDisplayOrientation);
696 poseController = mPoseController;
697 }
Eric Laurent6d607012021-07-05 11:54:40 +0200698 }
Eric Laurent780be4a2021-09-16 10:44:24 +0200699 if (poseController != nullptr) {
700 poseController->waitUntilCalculated();
701 }
Eric Laurent6d607012021-07-05 11:54:40 +0200702 return NO_ERROR;
703}
704
705audio_io_handle_t Spatializer::detachOutput() {
Eric Laurent2be8b292021-08-23 09:44:33 -0700706 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200707 ALOGV("%s mOutput %d", __func__, (int)mOutput);
Eric Laurent2be8b292021-08-23 09:44:33 -0700708 audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
Eric Laurent6d607012021-07-05 11:54:40 +0200709 if (mOutput == AUDIO_IO_HANDLE_NONE) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700710 return output;
Eric Laurent6d607012021-07-05 11:54:40 +0200711 }
712 // remove FX instance
713 mEngine->setEnabled(false);
714 mEngine.clear();
Eric Laurent2be8b292021-08-23 09:44:33 -0700715 output = mOutput;
Eric Laurent6d607012021-07-05 11:54:40 +0200716 mOutput = AUDIO_IO_HANDLE_NONE;
Eric Laurent2be8b292021-08-23 09:44:33 -0700717 mPoseController.reset();
Eric Laurent6d607012021-07-05 11:54:40 +0200718 return output;
719}
720
Eric Laurent2be8b292021-08-23 09:44:33 -0700721void Spatializer::calculateHeadPose() {
722 ALOGV("%s", __func__);
723 std::lock_guard lock(mLock);
724 if (mPoseController != nullptr) {
725 mPoseController->calculateAsync();
726 }
727}
Eric Laurent6d607012021-07-05 11:54:40 +0200728
Eric Laurent2be8b292021-08-23 09:44:33 -0700729void Spatializer::engineCallback(int32_t event, void *user, void *info) {
Eric Laurent6d607012021-07-05 11:54:40 +0200730 if (user == nullptr) {
731 return;
732 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700733 Spatializer* const me = reinterpret_cast<Spatializer *>(user);
Eric Laurent6d607012021-07-05 11:54:40 +0200734 switch (event) {
735 case AudioEffect::EVENT_FRAMES_PROCESSED: {
Eric Laurent2be8b292021-08-23 09:44:33 -0700736 int frames = info == nullptr ? 0 : *(int*)info;
Eric Laurent6d607012021-07-05 11:54:40 +0200737 ALOGD("%s frames processed %d for me %p", __func__, frames, me);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200738 me->postFramesProcessedMsg(frames);
Eric Laurent2be8b292021-08-23 09:44:33 -0700739 } break;
Eric Laurent6d607012021-07-05 11:54:40 +0200740 default:
741 ALOGD("%s event %d", __func__, event);
742 break;
743 }
744}
745
Eric Laurent8a4259f2021-09-14 16:04:00 +0200746void Spatializer::postFramesProcessedMsg(int frames) {
747 sp<AMessage> msg =
748 new AMessage(EngineCallbackHandler::kWhatOnFramesProcessed, mHandler);
749 msg->setInt32(EngineCallbackHandler::kNumFramesKey, frames);
750 msg->post();
751}
752
Eric Laurent6d607012021-07-05 11:54:40 +0200753} // namespace android