audio policy: concurrent capture
Implement concurrent capture in audio policy manager:
- Attach AudioRecord client to already opened input when possible
instead of systematically opening a new input for each client.
- Always allow inputs to start even in case of concurrency.
- Clients are selectively silenced based on their app state by audio
policy service.
- In case of concurrency on a given input stream, device and source is
chosen based app states and source priority.
Bug: 111438757
Test: Manual capture tests with solotester and Camera, Assistant and Duo
Test: CTS tests for AudioRecord
Change-Id: I302710ff545f67361d9aca89e81de40771ce7fb0
diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
index 59c8f10..c2ce754 100644
--- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
+++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
@@ -454,23 +454,6 @@
return rawbuffer;
}
-static std::string audioConcurrencyString(
- AudioPolicyInterface::concurrency_type__mask_t concurrency)
-{
- char buffer[64]; // oversized
- if (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_ALL) {
- snprintf(buffer, sizeof(buffer), "%s%s%s%s",
- (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_CALL)? ",call":"",
- (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_CAPTURE)? ",capture":"",
- (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_HOTWORD)? ",hotword":"",
- (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_PREEMPT)? ",preempt":"");
- } else {
- snprintf(buffer, sizeof(buffer), ",none");
- }
-
- return &buffer[1];
-}
-
std::string AudioPolicyService::getDeviceTypeStrForPortId(audio_port_handle_t portId) {
std::string typeStr;
struct audio_port port = {};
@@ -482,7 +465,7 @@
return typeStr;
}
-status_t AudioPolicyService::startInput(audio_port_handle_t portId, bool *silenced)
+status_t AudioPolicyService::startInput(audio_port_handle_t portId)
{
if (mAudioPolicyManager == NULL) {
return NO_INIT;
@@ -505,17 +488,16 @@
return PERMISSION_DENIED;
}
- // If UID inactive it records silence until becoming active
- *silenced = !mUidPolicy->isUidActive(client->uid) && !client->isVirtualDevice;
-
Mutex::Autolock _l(mLock);
- AudioPolicyInterface::concurrency_type__mask_t concurrency =
- AudioPolicyInterface::API_INPUT_CONCURRENCY_NONE;
+
+ client->active = true;
+ client->startTimeNs = systemTime();
+ updateUidStates_l();
status_t status;
{
AutoCallerClear acc;
- status = mAudioPolicyManager->startInput(portId, *silenced, &concurrency);
+ status = mAudioPolicyManager->startInput(portId);
}
@@ -524,7 +506,6 @@
static constexpr char kAudioPolicy[] = "audiopolicy";
- static constexpr char kAudioPolicyReason[] = "android.media.audiopolicy.reason";
static constexpr char kAudioPolicyStatus[] = "android.media.audiopolicy.status";
static constexpr char kAudioPolicyRqstSrc[] = "android.media.audiopolicy.rqst.src";
static constexpr char kAudioPolicyRqstPkg[] = "android.media.audiopolicy.rqst.pkg";
@@ -541,7 +522,6 @@
MediaAnalyticsItem *item = new MediaAnalyticsItem(kAudioPolicy);
if (item != NULL) {
- item->setCString(kAudioPolicyReason, audioConcurrencyString(concurrency).c_str());
item->setInt32(kAudioPolicyStatus, status);
item->setCString(kAudioPolicyRqstSrc,
@@ -556,54 +536,35 @@
item->setCString(
kAudioPolicyRqstDevice, getDeviceTypeStrForPortId(client->deviceId).c_str());
- // figure out who is active
- // NB: might the other party have given up the microphone since then? how sure.
- // perhaps could have given up on it.
- // we hold mLock, so perhaps we're safe for this looping
- if (concurrency != AudioPolicyInterface::API_INPUT_CONCURRENCY_NONE) {
- int count = mAudioRecordClients.size();
- for (int i = 0; i<count ; i++) {
- if (portId == mAudioRecordClients.keyAt(i)) {
- continue;
+ int count = mAudioRecordClients.size();
+ for (int i = 0; i < count ; i++) {
+ if (portId == mAudioRecordClients.keyAt(i)) {
+ continue;
+ }
+ sp<AudioRecordClient> other = mAudioRecordClients.valueAt(i);
+ if (other->active) {
+ // keeps the last of the clients marked active
+ item->setCString(kAudioPolicyActiveSrc,
+ audioSourceString(other->attributes.source).c_str());
+ item->setInt32(kAudioPolicyActiveSession, other->session);
+ if (other->opPackageName.size() != 0) {
+ item->setCString(kAudioPolicyActivePkg,
+ std::string(String8(other->opPackageName).string()).c_str());
+ } else {
+ item->setCString(kAudioPolicyRqstPkg,
+ std::to_string(other->uid).c_str());
}
- sp<AudioRecordClient> other = mAudioRecordClients.valueAt(i);
- if (other->active) {
- // keeps the last of the clients marked active
- item->setCString(kAudioPolicyActiveSrc,
- audioSourceString(other->attributes.source).c_str());
- item->setInt32(kAudioPolicyActiveSession, other->session);
- if (other->opPackageName.size() != 0) {
- item->setCString(kAudioPolicyActivePkg,
- std::string(String8(other->opPackageName).string()).c_str());
- } else {
- item->setCString(kAudioPolicyRqstPkg,
- std::to_string(other->uid).c_str());
- }
- item->setCString(kAudioPolicyActiveDevice,
- getDeviceTypeStrForPortId(other->deviceId).c_str());
- }
+ item->setCString(kAudioPolicyActiveDevice,
+ getDeviceTypeStrForPortId(other->deviceId).c_str());
}
}
item->selfrecord();
delete item;
item = NULL;
}
- }
-
- if (status == NO_ERROR) {
- LOG_ALWAYS_FATAL_IF(concurrency & ~AudioPolicyInterface::API_INPUT_CONCURRENCY_ALL,
- "startInput(): invalid concurrency type %d", (int)concurrency);
-
- // enforce permission (if any) required for each type of concurrency
- if (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_CALL) {
- //TODO: check incall capture permission
- }
- if (concurrency & AudioPolicyInterface::API_INPUT_CONCURRENCY_CAPTURE) {
- //TODO: check concurrent capture permission
- }
-
- client->active = true;
- } else {
+ client->active = false;
+ client->startTimeNs = 0;
+ updateUidStates_l();
finishRecording(client->opPackageName, client->uid);
}
@@ -615,6 +576,7 @@
if (mAudioPolicyManager == NULL) {
return NO_INIT;
}
+
Mutex::Autolock _l(mLock);
ssize_t index = mAudioRecordClients.indexOfKey(portId);
@@ -624,6 +586,9 @@
sp<AudioRecordClient> client = mAudioRecordClients.valueAt(index);
client->active = false;
+ client->startTimeNs = 0;
+
+ updateUidStates_l();
// finish the recording app op
finishRecording(client->opPackageName, client->uid);
@@ -646,6 +611,14 @@
return;
}
client = mAudioRecordClients.valueAt(index);
+
+ if (client->active) {
+ ALOGW("%s releasing active client portId %d", __FUNCTION__, portId);
+ client->active = false;
+ client->startTimeNs = 0;
+ updateUidStates_l();
+ }
+
mAudioRecordClients.removeItem(portId);
}
if (client == 0) {
diff --git a/services/audiopolicy/service/AudioPolicyService.cpp b/services/audiopolicy/service/AudioPolicyService.cpp
index 78dbf5f..2893872 100644
--- a/services/audiopolicy/service/AudioPolicyService.cpp
+++ b/services/audiopolicy/service/AudioPolicyService.cpp
@@ -348,11 +348,91 @@
void AudioPolicyService::updateUidStates_l()
{
- //TODO: implement real concurrent capture policy: for now just apply each app state directly
+// Go over all active clients and allow capture (does not force silence) in the
+// following cases:
+// - The client is the assistant AND
+// an accessibility service is on TOP AND the source is VOICE_RECOGNITION or HOTWORD
+// OR
+// is on TOP AND uses VOICE_RECOGNITION
+// OR uses HOTWORD AND there is no privacy sensitive active capture
+// - The client is an accessibility service AND
+// is on TOP AND the source is VOICE_RECOGNITION or HOTWORD
+// - Any other client AND
+// The assistant is not on TOP AND
+// is on TOP OR latest started AND
+// there is no privacy sensitive active capture
+//TODO: mamanage pre processing effects according to use case priority
+
+ sp<AudioRecordClient> topActive;
+ sp<AudioRecordClient> latestActive;
+ nsecs_t latestStartNs = 0;
+ sp<AudioRecordClient> latestSensitiveActive;
+ nsecs_t latestSensitiveStartNs = 0;
+ bool isA11yOnTop = mUidPolicy->isA11yOnTop();
+ bool isAssistantOnTop = false;
+ bool isSensitiveActive = false;
+
for (size_t i =0; i < mAudioRecordClients.size(); i++) {
sp<AudioRecordClient> current = mAudioRecordClients[i];
if (!current->active) continue;
- setAppState_l(current->uid, apmStatFromAmState(mUidPolicy->getUidState(current->uid)));
+ if (isPrivacySensitive(current->attributes.source)) {
+ if (current->startTimeNs > latestSensitiveStartNs) {
+ latestSensitiveActive = current;
+ latestSensitiveStartNs = current->startTimeNs;
+ }
+ isSensitiveActive = true;
+ }
+ if (mUidPolicy->getUidState(current->uid) == ActivityManager::PROCESS_STATE_TOP) {
+ topActive = current;
+ latestActive = nullptr;
+ if (mUidPolicy->isAssistantUid(current->uid)) {
+ isAssistantOnTop = true;
+ }
+ }
+ if (current->startTimeNs > latestStartNs) {
+ latestActive = current;
+ latestStartNs = current->startTimeNs;
+ }
+ }
+
+ if (topActive == nullptr && latestActive == nullptr) {
+ return;
+ }
+
+ for (size_t i =0; i < mAudioRecordClients.size(); i++) {
+ sp<AudioRecordClient> current = mAudioRecordClients[i];
+ if (!current->active) continue;
+
+ audio_source_t source = current->attributes.source;
+ bool isOnTop = mUidPolicy->getUidState(current->uid) == ActivityManager::PROCESS_STATE_TOP;
+ bool isLatest = current == latestActive;
+ bool isLatestSensitive = current == latestSensitiveActive;
+ bool forceIdle = true;
+ if (mUidPolicy->isAssistantUid(current->uid)) {
+ if (isA11yOnTop) {
+ if (source == AUDIO_SOURCE_HOTWORD || source == AUDIO_SOURCE_VOICE_RECOGNITION) {
+ forceIdle = false;
+ }
+ } else {
+ if (((isOnTop && source == AUDIO_SOURCE_VOICE_RECOGNITION) ||
+ source == AUDIO_SOURCE_HOTWORD) && !isSensitiveActive) {
+ forceIdle = false;
+ }
+ }
+ } else if (mUidPolicy->isA11yUid(current->uid)) {
+ if (isOnTop &&
+ (source == AUDIO_SOURCE_VOICE_RECOGNITION || source == AUDIO_SOURCE_HOTWORD)) {
+ forceIdle = false;
+ }
+ } else {
+ if (!isAssistantOnTop && (isOnTop || (topActive == nullptr && isLatest)) &&
+ (!isSensitiveActive || isLatestSensitive)) {
+ forceIdle = false;
+ }
+ }
+ setAppState_l(current->uid,
+ forceIdle ? APP_STATE_IDLE :
+ apmStatFromAmState(mUidPolicy->getUidState(current->uid)));
}
}
@@ -369,6 +449,22 @@
return APP_STATE_FOREGROUND;
}
+/* static */
+bool AudioPolicyService::isPrivacySensitive(audio_source_t source)
+{
+ switch (source) {
+ case AUDIO_SOURCE_VOICE_UPLINK:
+ case AUDIO_SOURCE_VOICE_DOWNLINK:
+ case AUDIO_SOURCE_VOICE_CALL:
+ case AUDIO_SOURCE_CAMCORDER:
+ case AUDIO_SOURCE_VOICE_COMMUNICATION:
+ return true;
+ default:
+ break;
+ }
+ return false;
+}
+
void AudioPolicyService::setAppState_l(uid_t uid, app_state_t state)
{
AutoCallerClear acc;
@@ -548,6 +644,7 @@
mObserverRegistered = true;
} else {
ALOGE("UidPolicy::registerSelf linkToDeath failed: %d", res);
+
am.unregisterUidObserver(this);
}
}
@@ -650,6 +747,7 @@
mCachedUids.insert(std::pair<uid_t,
std::pair<bool, int>>(uid, std::pair<bool, int>(active, state)));
}
+
return state;
}
@@ -730,6 +828,21 @@
}
}
+bool AudioPolicyService::UidPolicy::isA11yOnTop() {
+ for (const auto &uid : mCachedUids) {
+ std::vector<uid_t>::iterator it = find(mA11yUids.begin(), mA11yUids.end(), uid.first);
+ if (it == mA11yUids.end()) {
+ continue;
+ }
+ if (uid.second.second == ActivityManager::PROCESS_STATE_TOP ||
+ uid.second.second == ActivityManager::PROCESS_STATE_FOREGROUND_SERVICE ||
+ uid.second.second == ActivityManager::PROCESS_STATE_BOUND_FOREGROUND_SERVICE) {
+ return true;
+ }
+ }
+ return false;
+}
+
bool AudioPolicyService::UidPolicy::isA11yUid(uid_t uid)
{
std::vector<uid_t>::iterator it = find(mA11yUids.begin(), mA11yUids.end(), uid);
diff --git a/services/audiopolicy/service/AudioPolicyService.h b/services/audiopolicy/service/AudioPolicyService.h
index 4d7235f..dc5a36d 100644
--- a/services/audiopolicy/service/AudioPolicyService.h
+++ b/services/audiopolicy/service/AudioPolicyService.h
@@ -94,8 +94,7 @@
audio_input_flags_t flags,
audio_port_handle_t *selectedDeviceId = NULL,
audio_port_handle_t *portId = NULL);
- virtual status_t startInput(audio_port_handle_t portId,
- bool *silenced);
+ virtual status_t startInput(audio_port_handle_t portId);
virtual status_t stopInput(audio_port_handle_t portId);
virtual void releaseInput(audio_port_handle_t portId);
virtual status_t initStreamVolume(audio_stream_type_t stream,
@@ -276,6 +275,8 @@
void updateUidStates();
void updateUidStates_l();
+ static bool isPrivacySensitive(audio_source_t source);
+
// If recording we need to make sure the UID is allowed to do that. If the UID is idle
// then it cannot record and gets buffers with zeros - silence. As soon as the UID
// transitions to an active state we will start reporting buffers with data. This approach
@@ -299,6 +300,7 @@
bool isAssistantUid(uid_t uid) { return uid == mAssistantUid; }
void setA11yUids(const std::vector<uid_t>& uids) { mA11yUids.clear(); mA11yUids = uids; }
bool isA11yUid(uid_t uid);
+ bool isA11yOnTop();
// BnUidObserver implementation
void onUidActive(uid_t uid) override;
@@ -650,12 +652,11 @@
const audio_session_t session, const audio_port_handle_t deviceId,
const String16& opPackageName) :
AudioClient(attributes, io, uid, pid, session, deviceId),
- opPackageName(opPackageName), isConcurrent(false), isVirtualDevice(false) {}
+ opPackageName(opPackageName), startTimeNs(0) {}
~AudioRecordClient() override = default;
const String16 opPackageName; // client package name
- bool isConcurrent; // is allowed to concurrent capture
- bool isVirtualDevice; // uses virtual device: updated by APM::getInputForAttr()
+ nsecs_t startTimeNs;
};
// --- AudioPlaybackClient ---