Add playback rate to AudioTrack
Bug: 19196501
Change-Id: I6411e1d3ce652b711a71a6d9df020cb5f60d4714
diff --git a/include/media/AudioResamplerPublic.h b/include/media/AudioResamplerPublic.h
index 3c30148..07d946d 100644
--- a/include/media/AudioResamplerPublic.h
+++ b/include/media/AudioResamplerPublic.h
@@ -42,6 +42,8 @@
#define AUDIO_TIMESTRETCH_PITCH_MAX 2.0f
#define AUDIO_TIMESTRETCH_PITCH_NORMAL 1.0f
+// TODO: Consider putting these inlines into a class scope
+
// Returns the source frames needed to resample to destination frames. This is not a precise
// value and depends on the resampler (and possibly how it handles rounding internally).
// Nevertheless, this should be an upper bound on the requirements of the resampler.
@@ -66,4 +68,13 @@
return dstFrames > 2 ? dstFrames - 2 : 0;
}
+static inline size_t sourceFramesNeededWithTimestretch(
+ uint32_t srcSampleRate, size_t dstFramesRequired, uint32_t dstSampleRate,
+ float speed) {
+ // required is the number of input frames the resampler needs
+ size_t required = sourceFramesNeeded(srcSampleRate, dstFramesRequired, dstSampleRate);
+ // to deliver this, the time stretcher requires:
+ return required * (double)speed + 1 + 1; // accounting for rounding dependencies
+}
+
#endif // ANDROID_AUDIO_RESAMPLER_PUBLIC_H
diff --git a/include/media/AudioTrack.h b/include/media/AudioTrack.h
index e7e0703..a06197f 100644
--- a/include/media/AudioTrack.h
+++ b/include/media/AudioTrack.h
@@ -359,6 +359,21 @@
/* Return current source sample rate in Hz */
uint32_t getSampleRate() const;
+ /* Set source playback rate for timestretch
+ * 1.0 is normal speed: < 1.0 is slower, > 1.0 is faster
+ * 1.0 is normal pitch: < 1.0 is lower pitch, > 1.0 is higher pitch
+ *
+ * AUDIO_TIMESTRETCH_SPEED_MIN <= speed <= AUDIO_TIMESTRETCH_SPEED_MAX
+ * AUDIO_TIMESTRETCH_PITCH_MIN <= pitch <= AUDIO_TIMESTRETCH_PITCH_MAX
+ *
+ * Speed increases the playback rate of media, but does not alter pitch.
+ * Pitch increases the "tonal frequency" of media, but does not affect the playback rate.
+ */
+ status_t setPlaybackRate(float speed, float pitch);
+
+ /* Return current playback rate */
+ void getPlaybackRate(float *speed, float *pitch) const;
+
/* Enables looping and sets the start and end points of looping.
* Only supported for static buffer mode.
*
@@ -719,6 +734,9 @@
// increment mPosition by the delta of mServer, and return new value of mPosition
uint32_t updateAndGetPosition_l();
+ // check sample rate and speed is compatible with AudioTrack
+ bool isSampleRateSpeedAllowed_l(uint32_t sampleRate, float speed) const;
+
// Next 4 fields may be changed if IAudioTrack is re-created, but always != 0
sp<IAudioTrack> mAudioTrack;
sp<IMemory> mCblkMemory;
@@ -730,6 +748,8 @@
float mVolume[2];
float mSendLevel;
mutable uint32_t mSampleRate; // mutable because getSampleRate() can update it
+ float mSpeed; // timestretch: 1.0f for normal speed.
+ float mPitch; // timestretch: 1.0f for normal pitch.
size_t mFrameCount; // corresponds to current IAudioTrack, value is
// reported back by AudioFlinger to the client
size_t mReqFrameCount; // frame count to request the first or next time
diff --git a/include/private/media/AudioTrackShared.h b/include/private/media/AudioTrackShared.h
index 5644428..6cc2e2b 100644
--- a/include/private/media/AudioTrackShared.h
+++ b/include/private/media/AudioTrackShared.h
@@ -25,6 +25,7 @@
#include <utils/Log.h>
#include <utils/RefBase.h>
#include <audio_utils/roundup.h>
+#include <media/AudioResamplerPublic.h>
#include <media/SingleStateQueue.h>
namespace android {
@@ -113,6 +114,14 @@
mPosLoopQueue;
};
+
+struct AudioTrackPlaybackRate {
+ float mSpeed;
+ float mPitch;
+};
+
+typedef SingleStateQueue<AudioTrackPlaybackRate> AudioTrackPlaybackRateQueue;
+
// ----------------------------------------------------------------------------
// Important: do not add any virtual methods, including ~
@@ -159,6 +168,8 @@
uint32_t mSampleRate; // AudioTrack only: client's requested sample rate in Hz
// or 0 == default. Write-only client, read-only server.
+ AudioTrackPlaybackRateQueue::Shared mPlaybackRateQueue;
+
// client write-only, server read-only
uint16_t mSendLevel; // Fixed point U4.12 so 0x1000 means 1.0
@@ -313,7 +324,8 @@
AudioTrackClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount,
size_t frameSize, bool clientInServer = false)
: ClientProxy(cblk, buffers, frameCount, frameSize, true /*isOut*/,
- clientInServer) { }
+ clientInServer),
+ mPlaybackRateMutator(&cblk->mPlaybackRateQueue) { }
virtual ~AudioTrackClientProxy() { }
// No barriers on the following operations, so the ordering of loads/stores
@@ -333,6 +345,13 @@
mCblk->mSampleRate = sampleRate;
}
+ void setPlaybackRate(float speed, float pitch) {
+ AudioTrackPlaybackRate playbackRate;
+ playbackRate.mSpeed = speed;
+ playbackRate.mPitch = pitch;
+ mPlaybackRateMutator.push(playbackRate);
+ }
+
virtual void flush();
virtual uint32_t getUnderrunFrames() const {
@@ -344,6 +363,9 @@
bool getStreamEndDone() const;
status_t waitStreamEndDone(const struct timespec *requested);
+
+private:
+ AudioTrackPlaybackRateQueue::Mutator mPlaybackRateMutator;
};
class StaticAudioTrackClientProxy : public AudioTrackClientProxy {
@@ -458,8 +480,11 @@
public:
AudioTrackServerProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount,
size_t frameSize, bool clientInServer = false, uint32_t sampleRate = 0)
- : ServerProxy(cblk, buffers, frameCount, frameSize, true /*isOut*/, clientInServer) {
+ : ServerProxy(cblk, buffers, frameCount, frameSize, true /*isOut*/, clientInServer),
+ mPlaybackRateObserver(&cblk->mPlaybackRateQueue) {
mCblk->mSampleRate = sampleRate;
+ mPlaybackRate.mSpeed = AUDIO_TIMESTRETCH_SPEED_NORMAL;
+ mPlaybackRate.mPitch = AUDIO_TIMESTRETCH_PITCH_NORMAL;
}
protected:
virtual ~AudioTrackServerProxy() { }
@@ -493,6 +518,13 @@
// Return the total number of frames that AudioFlinger has obtained and released
virtual size_t framesReleased() const { return mCblk->mServer; }
+
+ // Return the playback speed and pitch read atomically. Not multi-thread safe on server side.
+ void getPlaybackRate(float *speed, float *pitch);
+
+private:
+ AudioTrackPlaybackRate mPlaybackRate; // last observed playback rate
+ AudioTrackPlaybackRateQueue::Observer mPlaybackRateObserver;
};
class StaticAudioTrackServerProxy : public AudioTrackServerProxy {
diff --git a/media/libmedia/AudioTrack.cpp b/media/libmedia/AudioTrack.cpp
index 9e9ec5b..89138e2 100644
--- a/media/libmedia/AudioTrack.cpp
+++ b/media/libmedia/AudioTrack.cpp
@@ -56,6 +56,24 @@
return convertTimespecToUs(tv);
}
+// Must match similar computation in createTrack_l in Threads.cpp.
+// TODO: Move to a common library
+static size_t calculateMinFrameCount(
+ uint32_t afLatencyMs, uint32_t afFrameCount, uint32_t afSampleRate,
+ uint32_t sampleRate, float speed)
+{
+ // Ensure that buffer depth covers at least audio hardware latency
+ uint32_t minBufCount = afLatencyMs / ((1000 * afFrameCount) / afSampleRate);
+ if (minBufCount < 2) {
+ minBufCount = 2;
+ }
+ ALOGV("calculateMinFrameCount afLatency %u afFrameCount %u afSampleRate %u "
+ "sampleRate %u speed %f minBufCount: %u",
+ afLatencyMs, afFrameCount, afSampleRate, sampleRate, speed, minBufCount);
+ return minBufCount * sourceFramesNeededWithTimestretch(
+ sampleRate, afFrameCount, afSampleRate, speed);
+}
+
// static
status_t AudioTrack::getMinFrameCount(
size_t* frameCount,
@@ -94,13 +112,10 @@
return status;
}
- // Ensure that buffer depth covers at least audio hardware latency
- uint32_t minBufCount = afLatency / ((1000 * afFrameCount) / afSampleRate);
- if (minBufCount < 2) {
- minBufCount = 2;
- }
+ // When called from createTrack, speed is 1.0f (normal speed).
+ // This is rechecked again on setting playback rate (TODO: on setting sample rate, too).
+ *frameCount = calculateMinFrameCount(afLatency, afFrameCount, afSampleRate, sampleRate, 1.0f);
- *frameCount = minBufCount * sourceFramesNeeded(sampleRate, afFrameCount, afSampleRate);
// The formula above should always produce a non-zero value under normal circumstances:
// AudioTrack.SAMPLE_RATE_HZ_MIN <= sampleRate <= AudioTrack.SAMPLE_RATE_HZ_MAX.
// Return error in the unlikely event that it does not, as that's part of the API contract.
@@ -109,8 +124,8 @@
streamType, sampleRate);
return BAD_VALUE;
}
- ALOGV("getMinFrameCount=%zu: afFrameCount=%zu, minBufCount=%u, afSampleRate=%u, afLatency=%u",
- *frameCount, afFrameCount, minBufCount, afSampleRate, afLatency);
+ ALOGV("getMinFrameCount=%zu: afFrameCount=%zu, afSampleRate=%u, afLatency=%u",
+ *frameCount, afFrameCount, afSampleRate, afLatency);
return NO_ERROR;
}
@@ -360,6 +375,8 @@
return BAD_VALUE;
}
mSampleRate = sampleRate;
+ mSpeed = AUDIO_TIMESTRETCH_SPEED_NORMAL;
+ mPitch = AUDIO_TIMESTRETCH_PITCH_NORMAL;
// Make copy of input parameter offloadInfo so that in the future:
// (a) createTrack_l doesn't need it as an input parameter
@@ -689,6 +706,7 @@
if (rate == 0 || rate > afSamplingRate * AUDIO_RESAMPLER_DOWN_RATIO_MAX) {
return BAD_VALUE;
}
+ // TODO: Should we also check if the buffer size is compatible?
mSampleRate = rate;
mProxy->setSampleRate(rate);
@@ -719,6 +737,42 @@
return mSampleRate;
}
+status_t AudioTrack::setPlaybackRate(float speed, float pitch)
+{
+ if (speed < AUDIO_TIMESTRETCH_SPEED_MIN
+ || speed > AUDIO_TIMESTRETCH_SPEED_MAX
+ || pitch < AUDIO_TIMESTRETCH_PITCH_MIN
+ || pitch > AUDIO_TIMESTRETCH_PITCH_MAX) {
+ return BAD_VALUE;
+ }
+ AutoMutex lock(mLock);
+ if (speed == mSpeed && pitch == mPitch) {
+ return NO_ERROR;
+ }
+ if (mIsTimed || isOffloadedOrDirect_l()) {
+ return INVALID_OPERATION;
+ }
+ if (mFlags & AUDIO_OUTPUT_FLAG_FAST) {
+ return INVALID_OPERATION;
+ }
+ // Check if the buffer size is compatible.
+ if (!isSampleRateSpeedAllowed_l(mSampleRate, speed)) {
+ ALOGV("setPlaybackRate(%f, %f) failed", speed, pitch);
+ return BAD_VALUE;
+ }
+ mSpeed = speed;
+ mPitch = pitch;
+ mProxy->setPlaybackRate(speed, pitch);
+ return NO_ERROR;
+}
+
+void AudioTrack::getPlaybackRate(float *speed, float *pitch) const
+{
+ AutoMutex lock(mLock);
+ *speed = mSpeed;
+ *pitch = mPitch;
+}
+
status_t AudioTrack::setLoop(uint32_t loopStart, uint32_t loopEnd, int loopCount)
{
if (mSharedBuffer == 0 || mIsTimed || isOffloadedOrDirect()) {
@@ -1086,8 +1140,16 @@
// there _is_ a frameCount parameter. We silently ignore it.
frameCount = mSharedBuffer->size() / mFrameSize;
} else {
- // For fast and normal streaming tracks,
- // the frame count calculations and checks are done by server
+ // For fast tracks the frame count calculations and checks are done by server
+
+ if ((mFlags & AUDIO_OUTPUT_FLAG_FAST) == 0) {
+ // for normal tracks precompute the frame count based on speed.
+ const size_t minFrameCount = calculateMinFrameCount(
+ afLatency, afFrameCount, afSampleRate, mSampleRate, mSpeed);
+ if (frameCount < minFrameCount) {
+ frameCount = minFrameCount;
+ }
+ }
}
IAudioFlinger::track_flags_t trackFlags = IAudioFlinger::TRACK_DEFAULT;
@@ -1230,6 +1292,7 @@
}
mAudioTrack->attachAuxEffect(mAuxEffectId);
+ // FIXME doesn't take into account speed or future sample rate changes (until restoreTrack)
// FIXME don't believe this lie
mLatency = afLatency + (1000*frameCount) / mSampleRate;
@@ -1255,6 +1318,7 @@
mProxy->setSendLevel(mSendLevel);
mProxy->setSampleRate(mSampleRate);
+ mProxy->setPlaybackRate(mSpeed, mPitch);
mProxy->setMinimum(mNotificationFramesAct);
mDeathNotifier = new DeathNotifier(this);
@@ -1617,6 +1681,7 @@
// Cache other fields that will be needed soon
uint32_t sampleRate = mSampleRate;
+ float speed = mSpeed;
uint32_t notificationFrames = mNotificationFramesAct;
if (mRefreshRemaining) {
mRefreshRemaining = false;
@@ -1745,7 +1810,7 @@
if (minFrames != (uint32_t) ~0) {
// This "fudge factor" avoids soaking CPU, and compensates for late progress by server
static const nsecs_t kFudgeNs = 10000000LL; // 10 ms
- ns = ((minFrames * 1000000000LL) / sampleRate) + kFudgeNs;
+ ns = ((double)minFrames * 1000000000) / ((double)sampleRate * speed) + kFudgeNs;
}
// If not supplying data by EVENT_MORE_DATA, then we're done
@@ -1786,7 +1851,8 @@
if (mRetryOnPartialBuffer && !isOffloaded()) {
mRetryOnPartialBuffer = false;
if (avail < mRemainingFrames) {
- int64_t myns = ((mRemainingFrames - avail) * 1100000000LL) / sampleRate;
+ int64_t myns = ((double)(mRemainingFrames - avail) * 1100000000)
+ / ((double)sampleRate * speed);
if (ns < 0 || myns < ns) {
ns = myns;
}
@@ -1841,7 +1907,7 @@
// that total to a sum == notificationFrames.
if (0 < misalignment && misalignment <= mRemainingFrames) {
mRemainingFrames = misalignment;
- return (mRemainingFrames * 1100000000LL) / sampleRate;
+ return ((double)mRemainingFrames * 1100000000) / ((double)sampleRate * speed);
}
#endif
@@ -1936,6 +2002,41 @@
return mPosition += (uint32_t) delta;
}
+bool AudioTrack::isSampleRateSpeedAllowed_l(uint32_t sampleRate, float speed) const
+{
+ // applicable for mixing tracks only (not offloaded or direct)
+ if (mStaticProxy != 0) {
+ return true; // static tracks do not have issues with buffer sizing.
+ }
+ status_t status;
+ uint32_t afLatency;
+ status = AudioSystem::getLatency(mOutput, &afLatency);
+ if (status != NO_ERROR) {
+ ALOGE("getLatency(%d) failed status %d", mOutput, status);
+ return false;
+ }
+
+ size_t afFrameCount;
+ status = AudioSystem::getFrameCount(mOutput, &afFrameCount);
+ if (status != NO_ERROR) {
+ ALOGE("getFrameCount(output=%d) status %d", mOutput, status);
+ return false;
+ }
+
+ uint32_t afSampleRate;
+ status = AudioSystem::getSamplingRate(mOutput, &afSampleRate);
+ if (status != NO_ERROR) {
+ ALOGE("getSamplingRate(output=%d) status %d", mOutput, status);
+ return false;
+ }
+
+ const size_t minFrameCount =
+ calculateMinFrameCount(afLatency, afFrameCount, afSampleRate, sampleRate, speed);
+ ALOGV("isSampleRateSpeedAllowed_l mFrameCount %zu minFrameCount %zu",
+ mFrameCount, minFrameCount);
+ return mFrameCount >= minFrameCount;
+}
+
status_t AudioTrack::setParameters(const String8& keyValuePairs)
{
AutoMutex lock(mLock);
@@ -2001,7 +2102,8 @@
return WOULD_BLOCK; // stale timestamp time, occurs before start.
}
const int64_t deltaTimeUs = timestampTimeUs - mStartUs;
- const int64_t deltaPositionByUs = timestamp.mPosition * 1000000LL / mSampleRate;
+ const int64_t deltaPositionByUs = (double)timestamp.mPosition * 1000000
+ / ((double)mSampleRate * mSpeed);
if (deltaPositionByUs > deltaTimeUs + kTimeJitterUs) {
// Verify that the counter can't count faster than the sample rate
@@ -2088,7 +2190,8 @@
snprintf(buffer, 255, " format(%d), channel count(%d), frame count(%zu)\n", mFormat,
mChannelCount, mFrameCount);
result.append(buffer);
- snprintf(buffer, 255, " sample rate(%u), status(%d)\n", mSampleRate, mStatus);
+ snprintf(buffer, 255, " sample rate(%u), speed(%f), status(%d)\n",
+ mSampleRate, mSpeed, mStatus);
result.append(buffer);
snprintf(buffer, 255, " state(%d), latency (%d)\n", mState, mLatency);
result.append(buffer);
diff --git a/media/libmedia/AudioTrackShared.cpp b/media/libmedia/AudioTrackShared.cpp
index 6d5f1af..ba67b40 100644
--- a/media/libmedia/AudioTrackShared.cpp
+++ b/media/libmedia/AudioTrackShared.cpp
@@ -793,6 +793,16 @@
(void) android_atomic_or(CBLK_UNDERRUN, &cblk->mFlags);
}
+void AudioTrackServerProxy::getPlaybackRate(float *speed, float *pitch)
+{ // do not call from multiple threads without holding lock
+ AudioTrackPlaybackRate playbackRate;
+ if (mPlaybackRateObserver.poll(playbackRate)) {
+ mPlaybackRate = playbackRate;
+ }
+ *speed = mPlaybackRate.mSpeed;
+ *pitch = mPlaybackRate.mPitch;
+}
+
// ---------------------------------------------------------------------------
StaticAudioTrackServerProxy::StaticAudioTrackServerProxy(audio_track_cblk_t* cblk, void *buffers,
diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp
index 1a20fae..b30fd20 100644
--- a/services/audioflinger/Threads.cpp
+++ b/services/audioflinger/Threads.cpp
@@ -1608,13 +1608,19 @@
// If you change this calculation, also review the start threshold which is related.
if (!(*flags & IAudioFlinger::TRACK_FAST)
&& audio_is_linear_pcm(format) && sharedBuffer == 0) {
+ // this must match AudioTrack.cpp calculateMinFrameCount().
+ // TODO: Move to a common library
uint32_t latencyMs = mOutput->stream->get_latency(mOutput->stream);
uint32_t minBufCount = latencyMs / ((1000 * mNormalFrameCount) / mSampleRate);
if (minBufCount < 2) {
minBufCount = 2;
}
+ // For normal mixing tracks, if speed is > 1.0f (normal), AudioTrack
+ // or the client should compute and pass in a larger buffer request.
size_t minFrameCount =
- minBufCount * sourceFramesNeeded(sampleRate, mNormalFrameCount, mSampleRate);
+ minBufCount * sourceFramesNeededWithTimestretch(
+ sampleRate, mNormalFrameCount,
+ mSampleRate, AUDIO_TIMESTRETCH_SPEED_NORMAL /*speed*/);
if (frameCount < minFrameCount) { // including frameCount == 0
frameCount = minFrameCount;
}
@@ -3592,21 +3598,17 @@
// hence the test on (mMixerStatus == MIXER_TRACKS_READY) meaning the track was mixed
// during last round
size_t desiredFrames;
- uint32_t sr = track->sampleRate();
- if (sr == mSampleRate) {
- desiredFrames = mNormalFrameCount;
- } else {
- desiredFrames = sourceFramesNeeded(sr, mNormalFrameCount, mSampleRate);
- // add frames already consumed but not yet released by the resampler
- // because mAudioTrackServerProxy->framesReady() will include these frames
- desiredFrames += mAudioMixer->getUnreleasedFrames(track->name());
-#if 0
- // the minimum track buffer size is normally twice the number of frames necessary
- // to fill one buffer and the resampler should not leave more than one buffer worth
- // of unreleased frames after each pass, but just in case...
- ALOG_ASSERT(desiredFrames <= cblk->frameCount_);
-#endif
- }
+ const uint32_t sampleRate = track->mAudioTrackServerProxy->getSampleRate();
+ float speed, pitch;
+ track->mAudioTrackServerProxy->getPlaybackRate(&speed, &pitch);
+
+ desiredFrames = sourceFramesNeededWithTimestretch(
+ sampleRate, mNormalFrameCount, mSampleRate, speed);
+ // TODO: ONLY USED FOR LEGACY RESAMPLERS, remove when they are removed.
+ // add frames already consumed but not yet released by the resampler
+ // because mAudioTrackServerProxy->framesReady() will include these frames
+ desiredFrames += mAudioMixer->getUnreleasedFrames(track->name());
+
uint32_t minFrames = 1;
if ((track->sharedBuffer() == 0) && !track->isStopped() && !track->isPausing() &&
(mMixerStatusIgnoringFastTracks == MIXER_TRACKS_READY)) {
@@ -3769,6 +3771,17 @@
AudioMixer::RESAMPLE,
AudioMixer::SAMPLE_RATE,
(void *)(uintptr_t)reqSampleRate);
+
+ // set the playback rate as an float array {speed, pitch}
+ float playbackRate[2];
+ track->mAudioTrackServerProxy->getPlaybackRate(
+ &playbackRate[0] /*speed*/, &playbackRate[1] /*pitch*/);
+ mAudioMixer->setParameter(
+ name,
+ AudioMixer::TIMESTRETCH,
+ AudioMixer::PLAYBACK_RATE,
+ playbackRate);
+
/*
* Select the appropriate output buffer for the track.
*
diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp
index 1566b1f..da2d634 100644
--- a/services/audioflinger/Tracks.cpp
+++ b/services/audioflinger/Tracks.cpp
@@ -903,9 +903,14 @@
mPreviousTimestampValid = false;
return INVALID_OPERATION;
}
+ // FIXME Not accurate under dynamic changes of sample rate and speed.
+ // Do not use track's mSampleRate as it is not current for mixer tracks.
+ uint32_t sampleRate = mAudioTrackServerProxy->getSampleRate();
+ float speed, pitch;
+ mAudioTrackServerProxy->getPlaybackRate(&speed, &pitch);
uint32_t unpresentedFrames =
- ((int64_t) playbackThread->mLatchQ.mUnpresentedFrames * mSampleRate) /
- playbackThread->mSampleRate;
+ ((double) playbackThread->mLatchQ.mUnpresentedFrames * sampleRate * speed)
+ / playbackThread->mSampleRate;
// FIXME Since we're using a raw pointer as the key, it is theoretically possible
// for a brand new track to share the same address as a recently destroyed
// track, and thus for us to get the frames released of the wrong track.