Add playback rate to AudioMixer
Bug: 19196501
Change-Id: I42d1f90e6297cf3f1304860d1691a5dfedd4c37d
diff --git a/services/audioflinger/AudioMixer.cpp b/services/audioflinger/AudioMixer.cpp
index cb90ece..c2c791f 100644
--- a/services/audioflinger/AudioMixer.cpp
+++ b/services/audioflinger/AudioMixer.cpp
@@ -123,6 +123,7 @@
t->resampler = NULL;
t->downmixerBufferProvider = NULL;
t->mReformatBufferProvider = NULL;
+ t->mTimestretchBufferProvider = NULL;
t++;
}
@@ -135,6 +136,7 @@
delete t->resampler;
delete t->downmixerBufferProvider;
delete t->mReformatBufferProvider;
+ delete t->mTimestretchBufferProvider;
t++;
}
delete [] mState.outputTemp;
@@ -213,6 +215,7 @@
t->mReformatBufferProvider = NULL;
t->downmixerBufferProvider = NULL;
t->mPostDownmixReformatBufferProvider = NULL;
+ t->mTimestretchBufferProvider = NULL;
t->mMixerFormat = AUDIO_FORMAT_PCM_16_BIT;
t->mFormat = format;
t->mMixerInFormat = selectMixerInFormat(format);
@@ -220,6 +223,8 @@
t->mMixerChannelMask = audio_channel_mask_from_representation_and_bits(
AUDIO_CHANNEL_REPRESENTATION_POSITION, AUDIO_CHANNEL_OUT_STEREO);
t->mMixerChannelCount = audio_channel_count_from_out_mask(t->mMixerChannelMask);
+ t->mSpeed = AUDIO_TIMESTRETCH_SPEED_NORMAL;
+ t->mPitch = AUDIO_TIMESTRETCH_PITCH_NORMAL;
// Check the downmixing (or upmixing) requirements.
status_t status = t->prepareForDownmix();
if (status != OK) {
@@ -412,6 +417,10 @@
mPostDownmixReformatBufferProvider->setBufferProvider(bufferProvider);
bufferProvider = mPostDownmixReformatBufferProvider;
}
+ if (mTimestretchBufferProvider) {
+ mTimestretchBufferProvider->setBufferProvider(bufferProvider);
+ bufferProvider = mTimestretchBufferProvider;
+ }
}
void AudioMixer::deleteTrackName(int name)
@@ -432,7 +441,9 @@
mState.tracks[name].unprepareForDownmix();
// delete the reformatter
mState.tracks[name].unprepareForReformat();
-
+ // delete the timestretch provider
+ delete track.mTimestretchBufferProvider;
+ track.mTimestretchBufferProvider = NULL;
mTrackNames &= ~(1<<name);
}
@@ -654,6 +665,26 @@
}
}
break;
+ case TIMESTRETCH:
+ switch (param) {
+ case PLAYBACK_RATE: {
+ const float speed = reinterpret_cast<float*>(value)[0];
+ const float pitch = reinterpret_cast<float*>(value)[1];
+ ALOG_ASSERT(AUDIO_TIMESTRETCH_SPEED_MIN <= speed
+ && speed <= AUDIO_TIMESTRETCH_SPEED_MAX,
+ "bad speed %f", speed);
+ ALOG_ASSERT(AUDIO_TIMESTRETCH_PITCH_MIN <= pitch
+ && pitch <= AUDIO_TIMESTRETCH_PITCH_MAX,
+ "bad pitch %f", pitch);
+ if (track.setPlaybackRate(speed, pitch)) {
+ ALOGV("setParameter(TIMESTRETCH, PLAYBACK_RATE, %f %f", speed, pitch);
+ // invalidateState(1 << name);
+ }
+ } break;
+ default:
+ LOG_ALWAYS_FATAL("setParameter timestretch: bad param %d", param);
+ }
+ break;
default:
LOG_ALWAYS_FATAL("setParameter: bad target %d", target);
@@ -699,6 +730,28 @@
return false;
}
+bool AudioMixer::track_t::setPlaybackRate(float speed, float pitch)
+{
+ if (speed == mSpeed && pitch == mPitch) {
+ return false;
+ }
+ mSpeed = speed;
+ mPitch = pitch;
+ if (mTimestretchBufferProvider == NULL) {
+ // TODO: Remove MONO_HACK. Resampler sees #channels after the downmixer
+ // but if none exists, it is the channel count (1 for mono).
+ const int timestretchChannelCount = downmixerBufferProvider != NULL
+ ? mMixerChannelCount : channelCount;
+ mTimestretchBufferProvider = new TimestretchBufferProvider(timestretchChannelCount,
+ mMixerInFormat, sampleRate, speed, pitch);
+ reconfigureBufferProviders();
+ } else {
+ reinterpret_cast<TimestretchBufferProvider*>(mTimestretchBufferProvider)
+ ->setPlaybackRate(speed, pitch);
+ }
+ return true;
+}
+
/* Checks to see if the volume ramp has completed and clears the increment
* variables appropriately.
*
@@ -777,6 +830,8 @@
mState.tracks[name].downmixerBufferProvider->reset();
} else if (mState.tracks[name].mPostDownmixReformatBufferProvider != NULL) {
mState.tracks[name].mPostDownmixReformatBufferProvider->reset();
+ } else if (mState.tracks[name].mTimestretchBufferProvider != NULL) {
+ mState.tracks[name].mTimestretchBufferProvider->reset();
}
mState.tracks[name].mInputBufferProvider = bufferProvider;
diff --git a/services/audioflinger/AudioMixer.h b/services/audioflinger/AudioMixer.h
index e283b83..e27a0d1 100644
--- a/services/audioflinger/AudioMixer.h
+++ b/services/audioflinger/AudioMixer.h
@@ -73,6 +73,7 @@
RESAMPLE = 0x3001,
RAMP_VOLUME = 0x3002, // ramp to new volume
VOLUME = 0x3003, // don't ramp
+ TIMESTRETCH = 0x3004,
// set Parameter names
// for target TRACK
@@ -100,6 +101,9 @@
VOLUME0 = 0x4200,
VOLUME1 = 0x4201,
AUXLEVEL = 0x4210,
+ // for target TIMESTRETCH
+ PLAYBACK_RATE = 0x4300, // Configure timestretch on this track name;
+ // parameter 'value' is a pointer to the new playback rate.
};
@@ -214,6 +218,9 @@
/* Buffer providers are constructed to translate the track input data as needed.
*
+ * TODO: perhaps make a single PlaybackConverterProvider class to move
+ * all pre-mixer track buffer conversions outside the AudioMixer class.
+ *
* 1) mInputBufferProvider: The AudioTrack buffer provider.
* 2) mReformatBufferProvider: If not NULL, performs the audio reformat to
* match either mMixerInFormat or mDownmixRequiresFormat, if the downmixer
@@ -223,11 +230,13 @@
* the number of channels required by the mixer sink.
* 4) mPostDownmixReformatBufferProvider: If not NULL, performs reformatting from
* the downmixer requirements to the mixer engine input requirements.
+ * 5) mTimestretchBufferProvider: Adds timestretching for playback rate
*/
AudioBufferProvider* mInputBufferProvider; // externally provided buffer provider.
PassthruBufferProvider* mReformatBufferProvider; // provider wrapper for reformatting.
PassthruBufferProvider* downmixerBufferProvider; // wrapper for channel conversion.
PassthruBufferProvider* mPostDownmixReformatBufferProvider;
+ PassthruBufferProvider* mTimestretchBufferProvider;
int32_t sessionId;
@@ -250,6 +259,9 @@
audio_channel_mask_t mMixerChannelMask;
uint32_t mMixerChannelCount;
+ float mSpeed;
+ float mPitch;
+
bool needsRamp() { return (volumeInc[0] | volumeInc[1] | auxInc) != 0; }
bool setResampler(uint32_t trackSampleRate, uint32_t devSampleRate);
bool doesResample() const { return resampler != NULL; }
@@ -262,6 +274,7 @@
void unprepareForDownmix();
status_t prepareForReformat();
void unprepareForReformat();
+ bool setPlaybackRate(float speed, float pitch);
void reconfigureBufferProviders();
};
diff --git a/services/audioflinger/BufferProviders.cpp b/services/audioflinger/BufferProviders.cpp
index e143805..e058e6c 100644
--- a/services/audioflinger/BufferProviders.cpp
+++ b/services/audioflinger/BufferProviders.cpp
@@ -20,7 +20,9 @@
#include <audio_effects/effect_downmix.h>
#include <audio_utils/primitives.h>
#include <audio_utils/format.h>
+#include <media/AudioResamplerPublic.h>
#include <media/EffectsFactoryApi.h>
+
#include <utils/Log.h>
#include "Configuration.h"
@@ -358,5 +360,165 @@
memcpy_by_audio_format(dst, mOutputFormat, src, mInputFormat, frames * mChannelCount);
}
+TimestretchBufferProvider::TimestretchBufferProvider(int32_t channelCount,
+ audio_format_t format, uint32_t sampleRate, float speed, float pitch) :
+ mChannelCount(channelCount),
+ mFormat(format),
+ mSampleRate(sampleRate),
+ mFrameSize(channelCount * audio_bytes_per_sample(format)),
+ mSpeed(speed),
+ mPitch(pitch),
+ mLocalBufferFrameCount(0),
+ mLocalBufferData(NULL),
+ mRemaining(0)
+{
+ ALOGV("TimestretchBufferProvider(%p)(%u, %#x, %u %f %f)",
+ this, channelCount, format, sampleRate, speed, pitch);
+ mBuffer.frameCount = 0;
+}
+
+TimestretchBufferProvider::~TimestretchBufferProvider()
+{
+ ALOGV("~TimestretchBufferProvider(%p)", this);
+ if (mBuffer.frameCount != 0) {
+ mTrackBufferProvider->releaseBuffer(&mBuffer);
+ }
+ free(mLocalBufferData);
+}
+
+status_t TimestretchBufferProvider::getNextBuffer(
+ AudioBufferProvider::Buffer *pBuffer, int64_t pts)
+{
+ ALOGV("TimestretchBufferProvider(%p)::getNextBuffer(%p (%zu), %lld)",
+ this, pBuffer, pBuffer->frameCount, pts);
+
+ // BYPASS
+ //return mTrackBufferProvider->getNextBuffer(pBuffer, pts);
+
+ // check if previously processed data is sufficient.
+ if (pBuffer->frameCount <= mRemaining) {
+ ALOGV("previous sufficient");
+ pBuffer->raw = mLocalBufferData;
+ return OK;
+ }
+
+ // do we need to resize our buffer?
+ if (pBuffer->frameCount > mLocalBufferFrameCount) {
+ void *newmem;
+ if (posix_memalign(&newmem, 32, pBuffer->frameCount * mFrameSize) == OK) {
+ if (mRemaining != 0) {
+ memcpy(newmem, mLocalBufferData, mRemaining * mFrameSize);
+ }
+ free(mLocalBufferData);
+ mLocalBufferData = newmem;
+ mLocalBufferFrameCount = pBuffer->frameCount;
+ }
+ }
+
+ // need to fetch more data
+ const size_t outputDesired = pBuffer->frameCount - mRemaining;
+ mBuffer.frameCount = mSpeed == AUDIO_TIMESTRETCH_SPEED_NORMAL
+ ? outputDesired : outputDesired * mSpeed + 1;
+
+ status_t res = mTrackBufferProvider->getNextBuffer(&mBuffer, pts);
+
+ ALOG_ASSERT(res == OK || mBuffer.frameCount == 0);
+ if (res != OK || mBuffer.frameCount == 0) { // not needed by API spec, but to be safe.
+ ALOGD("buffer error");
+ if (mRemaining == 0) {
+ pBuffer->raw = NULL;
+ pBuffer->frameCount = 0;
+ return res;
+ } else { // return partial count
+ pBuffer->raw = mLocalBufferData;
+ pBuffer->frameCount = mRemaining;
+ return OK;
+ }
+ }
+
+ // time-stretch the data
+ size_t dstAvailable = min(mLocalBufferFrameCount - mRemaining, outputDesired);
+ size_t srcAvailable = mBuffer.frameCount;
+ processFrames((uint8_t*)mLocalBufferData + mRemaining * mFrameSize, &dstAvailable,
+ mBuffer.raw, &srcAvailable);
+
+ // release all data consumed
+ mBuffer.frameCount = srcAvailable;
+ mTrackBufferProvider->releaseBuffer(&mBuffer);
+
+ // update buffer vars with the actual data processed and return with buffer
+ mRemaining += dstAvailable;
+
+ pBuffer->raw = mLocalBufferData;
+ pBuffer->frameCount = mRemaining;
+
+ return OK;
+}
+
+void TimestretchBufferProvider::releaseBuffer(AudioBufferProvider::Buffer *pBuffer)
+{
+ ALOGV("TimestretchBufferProvider(%p)::releaseBuffer(%p (%zu))",
+ this, pBuffer, pBuffer->frameCount);
+
+ // BYPASS
+ //return mTrackBufferProvider->releaseBuffer(pBuffer);
+
+ // LOG_ALWAYS_FATAL_IF(pBuffer->frameCount == 0, "Invalid framecount");
+ if (pBuffer->frameCount < mRemaining) {
+ memcpy(mLocalBufferData,
+ (uint8_t*)mLocalBufferData + pBuffer->frameCount * mFrameSize,
+ (mRemaining - pBuffer->frameCount) * mFrameSize);
+ mRemaining -= pBuffer->frameCount;
+ } else if (pBuffer->frameCount == mRemaining) {
+ mRemaining = 0;
+ } else {
+ LOG_ALWAYS_FATAL("Releasing more frames(%zu) than available(%zu)",
+ pBuffer->frameCount, mRemaining);
+ }
+
+ pBuffer->raw = NULL;
+ pBuffer->frameCount = 0;
+}
+
+void TimestretchBufferProvider::reset()
+{
+ mRemaining = 0;
+}
+
+status_t TimestretchBufferProvider::setPlaybackRate(float speed, float pitch)
+{
+ mSpeed = speed;
+ mPitch = pitch;
+ return OK;
+}
+
+void TimestretchBufferProvider::processFrames(void *dstBuffer, size_t *dstFrames,
+ const void *srcBuffer, size_t *srcFrames)
+{
+ ALOGV("processFrames(%zu %zu) remaining(%zu)", *dstFrames, *srcFrames, mRemaining);
+ // Note dstFrames is the required number of frames.
+
+ // Ensure consumption from src is as expected.
+ const size_t targetSrc = *dstFrames * mSpeed;
+ if (*srcFrames < targetSrc) { // limit dst frames to that possible
+ *dstFrames = *srcFrames / mSpeed;
+ } else if (*srcFrames > targetSrc + 1) {
+ *srcFrames = targetSrc + 1;
+ }
+
+ // Do the time stretch by memory copy without any local buffer.
+ if (*dstFrames <= *srcFrames) {
+ size_t copySize = mFrameSize * *dstFrames;
+ memcpy(dstBuffer, srcBuffer, copySize);
+ } else {
+ // cyclically repeat the source.
+ for (size_t count = 0; count < *dstFrames; count += *srcFrames) {
+ size_t remaining = min(*srcFrames, *dstFrames - count);
+ memcpy((uint8_t*)dstBuffer + mFrameSize * count,
+ srcBuffer, mFrameSize * *srcFrames);
+ }
+ }
+}
+
// ----------------------------------------------------------------------------
} // namespace android
diff --git a/services/audioflinger/BufferProviders.h b/services/audioflinger/BufferProviders.h
index 7145b80..2b6ea47 100644
--- a/services/audioflinger/BufferProviders.h
+++ b/services/audioflinger/BufferProviders.h
@@ -146,6 +146,45 @@
const audio_format_t mOutputFormat;
};
+// TimestretchBufferProvider derives from PassthruBufferProvider for time stretching
+class TimestretchBufferProvider : public PassthruBufferProvider {
+public:
+ TimestretchBufferProvider(int32_t channelCount,
+ audio_format_t format, uint32_t sampleRate, float speed, float pitch);
+ virtual ~TimestretchBufferProvider();
+
+ // Overrides AudioBufferProvider methods
+ virtual status_t getNextBuffer(Buffer* buffer, int64_t pts);
+ virtual void releaseBuffer(Buffer* buffer);
+
+ // Overrides PassthruBufferProvider
+ virtual void reset();
+
+ virtual status_t setPlaybackRate(float speed, float pitch);
+
+ // processes frames
+ // dstBuffer is where to place the data
+ // dstFrames [in/out] is the desired frames (return with actual placed in buffer)
+ // srcBuffer is the source data
+ // srcFrames [in/out] is the available source frames (return with consumed)
+ virtual void processFrames(void *dstBuffer, size_t *dstFrames,
+ const void *srcBuffer, size_t *srcFrames);
+
+protected:
+ const uint32_t mChannelCount;
+ const audio_format_t mFormat;
+ const uint32_t mSampleRate; // const for now (TODO change this)
+ const size_t mFrameSize;
+ float mSpeed;
+ float mPitch;
+
+private:
+ AudioBufferProvider::Buffer mBuffer;
+ size_t mLocalBufferFrameCount;
+ void *mLocalBufferData;
+ size_t mRemaining;
+};
+
// ----------------------------------------------------------------------------
} // namespace android