Merge "Fix bug for direct track with PCM != 16-bit"
diff --git a/include/media/stagefright/ClockEstimator.h b/include/media/stagefright/ClockEstimator.h
new file mode 100644
index 0000000..2fd6e75
--- /dev/null
+++ b/include/media/stagefright/ClockEstimator.h
@@ -0,0 +1,110 @@
+/*
+**
+** Copyright 2014, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+
+#ifndef CLOCK_ESTIMATOR_H_
+
+#define CLOCK_ESTIMATOR_H_
+
+
+#include <utils/RefBase.h>
+#include <utils/Vector.h>
+
+namespace android {
+// ---------------------------------------------------------------------------
+
+struct ClockEstimator : RefBase {
+ virtual double estimate(double x, double y) = 0;
+ virtual void reset() = 0;
+};
+
+struct WindowedLinearFitEstimator : ClockEstimator {
+ struct LinearFit {
+ /**
+ * Fit y = a * x + b, where each input has a weight
+ */
+ double mX; // sum(w_i * x_i)
+ double mXX; // sum(w_i * x_i^2)
+ double mY; // sum(w_i * y_i)
+ double mYY; // sum(w_i * y_i^2)
+ double mXY; // sum(w_i * x_i * y_i)
+ double mW; // sum(w_i)
+
+ LinearFit();
+ void reset();
+ void combine(const LinearFit &lf);
+ void add(double x, double y, double w);
+ void scale(double w);
+ double interpolate(double x);
+ double size() const;
+
+ DISALLOW_EVIL_CONSTRUCTORS(LinearFit);
+ };
+
+ /**
+ * Estimator for f(x) = y' where input y' is noisy, but
+ * theoretically linear:
+ *
+ * y' =~ y = a * x + b
+ *
+ * It uses linear fit regression over a tapering rolling window
+ * to get an estimate for y (from the current and past inputs
+ * (x, y')).
+ *
+ * ____________
+ * /| |\
+ * / | | \
+ * / | | \ <--- new data (x, y')
+ * / | main | \
+ * <--><----------><-->
+ * tail head
+ *
+ * weight is 1 under the main window, tapers exponentially by
+ * the factors given in the head and the tail.
+ *
+ * Assuming that x and y' are monotonic, that x is somewhat
+ * evenly sampled, and that a =~ 1, the estimated y is also
+ * going to be monotonic.
+ */
+ WindowedLinearFitEstimator(
+ size_t headLength = 5, double headFactor = 0.5,
+ size_t mainLength = 0, double tailFactor = 0.99);
+
+ virtual void reset();
+
+ // add a new sample (x -> y') and return an estimated value for the true y
+ virtual double estimate(double x, double y);
+
+private:
+ Vector<double> mXHistory; // circular buffer
+ Vector<double> mYHistory; // circular buffer
+ LinearFit mHead;
+ LinearFit mMain;
+ LinearFit mTail;
+ double mHeadFactorInv;
+ double mTailFactor;
+ double mFirstWeight;
+ size_t mHistoryLength;
+ size_t mHeadLength;
+ size_t mNumSamples;
+ size_t mSampleIx;
+
+ DISALLOW_EVIL_CONSTRUCTORS(WindowedLinearFitEstimator);
+};
+
+}; // namespace android
+
+#endif
diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk
index 714b5e0..d9e39ff 100644
--- a/media/libstagefright/Android.mk
+++ b/media/libstagefright/Android.mk
@@ -14,6 +14,7 @@
AwesomePlayer.cpp \
CameraSource.cpp \
CameraSourceTimeLapse.cpp \
+ ClockEstimator.cpp \
DataSource.cpp \
DataURISource.cpp \
DRMExtractor.cpp \
diff --git a/media/libstagefright/AwesomePlayer.cpp b/media/libstagefright/AwesomePlayer.cpp
index 6e5003f..8d3032b 100644
--- a/media/libstagefright/AwesomePlayer.cpp
+++ b/media/libstagefright/AwesomePlayer.cpp
@@ -42,6 +42,7 @@
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/timedtext/TimedTextDriver.h>
#include <media/stagefright/AudioPlayer.h>
+#include <media/stagefright/ClockEstimator.h>
#include <media/stagefright/DataSource.h>
#include <media/stagefright/FileSource.h>
#include <media/stagefright/MediaBuffer.h>
@@ -231,6 +232,8 @@
&AwesomePlayer::onAudioTearDownEvent);
mAudioTearDownEventPending = false;
+ mClockEstimator = new WindowedLinearFitEstimator();
+
reset();
}
@@ -1866,21 +1869,28 @@
TimeSource *ts =
((mFlags & AUDIO_AT_EOS) || !(mFlags & AUDIOPLAYER_STARTED))
? &mSystemTimeSource : mTimeSource;
+ int64_t systemTimeUs = mSystemTimeSource.getRealTimeUs();
+ int64_t looperTimeUs = ALooper::GetNowUs();
if (mFlags & FIRST_FRAME) {
modifyFlags(FIRST_FRAME, CLEAR);
mSinceLastDropped = 0;
- mTimeSourceDeltaUs = ts->getRealTimeUs() - timeUs;
+ mClockEstimator->reset();
+ mTimeSourceDeltaUs = estimateRealTimeUs(ts, systemTimeUs) - timeUs;
}
int64_t realTimeUs, mediaTimeUs;
if (!(mFlags & AUDIO_AT_EOS) && mAudioPlayer != NULL
&& mAudioPlayer->getMediaTimeMapping(&realTimeUs, &mediaTimeUs)) {
+ ALOGV("updating TSdelta (%" PRId64 " => %" PRId64 " change %" PRId64 ")",
+ mTimeSourceDeltaUs, realTimeUs - mediaTimeUs,
+ mTimeSourceDeltaUs - (realTimeUs - mediaTimeUs));
+ ATRACE_INT("TS delta change (ms)", (mTimeSourceDeltaUs - (realTimeUs - mediaTimeUs)) / 1E3);
mTimeSourceDeltaUs = realTimeUs - mediaTimeUs;
}
if (wasSeeking == SEEK_VIDEO_ONLY) {
- int64_t nowUs = ts->getRealTimeUs() - mTimeSourceDeltaUs;
+ int64_t nowUs = estimateRealTimeUs(ts, systemTimeUs) - mTimeSourceDeltaUs;
int64_t latenessUs = nowUs - timeUs;
@@ -1891,12 +1901,13 @@
}
}
+ int64_t latenessUs = 0;
if (wasSeeking == NO_SEEK) {
// Let's display the first frame after seeking right away.
- int64_t nowUs = ts->getRealTimeUs() - mTimeSourceDeltaUs;
+ int64_t nowUs = estimateRealTimeUs(ts, systemTimeUs) - mTimeSourceDeltaUs;
- int64_t latenessUs = nowUs - timeUs;
+ latenessUs = nowUs - timeUs;
ATRACE_INT("Video Lateness (ms)", latenessUs / 1E3);
@@ -1950,9 +1961,9 @@
}
}
- if (latenessUs < -10000) {
- // We're more than 10ms early.
- postVideoEvent_l(10000);
+ if (latenessUs < -30000) {
+ // We're more than 30ms early, schedule at most 20 ms before time due
+ postVideoEvent_l(latenessUs < -60000 ? 30000 : -latenessUs - 20000);
return;
}
}
@@ -1966,6 +1977,8 @@
if (mVideoRenderer != NULL) {
mSinceLastDropped++;
+ mVideoBuffer->meta_data()->setInt64(kKeyTime, looperTimeUs - latenessUs);
+
mVideoRenderer->render(mVideoBuffer);
if (!mVideoRenderingStarted) {
mVideoRenderingStarted = true;
@@ -2015,14 +2028,26 @@
int64_t nextTimeUs;
CHECK(mVideoBuffer->meta_data()->findInt64(kKeyTime, &nextTimeUs));
- int64_t delayUs = nextTimeUs - ts->getRealTimeUs() + mTimeSourceDeltaUs;
- postVideoEvent_l(delayUs > 10000 ? 10000 : delayUs < 0 ? 0 : delayUs);
+ systemTimeUs = mSystemTimeSource.getRealTimeUs();
+ int64_t delayUs = nextTimeUs - estimateRealTimeUs(ts, systemTimeUs) + mTimeSourceDeltaUs;
+ ATRACE_INT("Frame delta (ms)", (nextTimeUs - timeUs) / 1E3);
+ ALOGV("next frame in %" PRId64, delayUs);
+ // try to schedule 30ms before time due
+ postVideoEvent_l(delayUs > 60000 ? 30000 : (delayUs < 30000 ? 0 : delayUs - 30000));
return;
}
postVideoEvent_l();
}
+int64_t AwesomePlayer::estimateRealTimeUs(TimeSource *ts, int64_t systemTimeUs) {
+ if (ts == &mSystemTimeSource) {
+ return systemTimeUs;
+ } else {
+ return (int64_t)mClockEstimator->estimate(systemTimeUs, ts->getRealTimeUs());
+ }
+}
+
void AwesomePlayer::postVideoEvent_l(int64_t delayUs) {
ATRACE_CALL();
diff --git a/media/libstagefright/ClockEstimator.cpp b/media/libstagefright/ClockEstimator.cpp
new file mode 100644
index 0000000..34d1e42
--- /dev/null
+++ b/media/libstagefright/ClockEstimator.cpp
@@ -0,0 +1,177 @@
+/*
+**
+** Copyright 2014, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "ClockEstimator"
+#include <utils/Log.h>
+
+#include <math.h>
+#include <media/stagefright/ClockEstimator.h>
+
+#include <media/stagefright/foundation/ADebug.h>
+
+namespace android {
+
+WindowedLinearFitEstimator::WindowedLinearFitEstimator(
+ size_t headLength, double headFactor, size_t mainLength, double tailFactor)
+ : mHeadFactorInv(1. / headFactor),
+ mTailFactor(tailFactor),
+ mHistoryLength(mainLength + headLength),
+ mHeadLength(headLength) {
+ reset();
+ mXHistory.resize(mHistoryLength);
+ mYHistory.resize(mHistoryLength);
+ mFirstWeight = pow(headFactor, mHeadLength);
+}
+
+WindowedLinearFitEstimator::LinearFit::LinearFit() {
+ reset();
+}
+
+void WindowedLinearFitEstimator::LinearFit::reset() {
+ mX = mXX = mY = mYY = mXY = mW = 0.;
+}
+
+double WindowedLinearFitEstimator::LinearFit::size() const {
+ double s = mW * mW + mX * mX + mY * mY + mXX * mXX + mXY * mXY + mYY * mYY;
+ if (s > 1e72) {
+ // 1e72 corresponds to clock monotonic time of about 8 years
+ ALOGW("estimator is overflowing: w=%g x=%g y=%g xx=%g xy=%g yy=%g",
+ mW, mX, mY, mXX, mXY, mYY);
+ }
+ return s;
+}
+
+void WindowedLinearFitEstimator::LinearFit::add(double x, double y, double w) {
+ mW += w;
+ mX += w * x;
+ mY += w * y;
+ mXX += w * x * x;
+ mXY += w * x * y;
+ mYY += w * y * y;
+}
+
+void WindowedLinearFitEstimator::LinearFit::combine(const LinearFit &lf) {
+ mW += lf.mW;
+ mX += lf.mX;
+ mY += lf.mY;
+ mXX += lf.mXX;
+ mXY += lf.mXY;
+ mYY += lf.mYY;
+}
+
+void WindowedLinearFitEstimator::LinearFit::scale(double w) {
+ mW *= w;
+ mX *= w;
+ mY *= w;
+ mXX *= w;
+ mXY *= w;
+ mYY *= w;
+}
+
+double WindowedLinearFitEstimator::LinearFit::interpolate(double x) {
+ double div = mW * mXX - mX * mX;
+ if (fabs(div) < 1e-5 * mW * mW) {
+ // this only should happen on the first value
+ return x;
+ // assuming a = 1, we could also return x + (mY - mX) / mW;
+ }
+ double a_div = (mW * mXY - mX * mY);
+ double b_div = (mXX * mY - mX * mXY);
+ ALOGV("a=%.4g b=%.4g in=%g out=%g",
+ a_div / div, b_div / div, x, (a_div * x + b_div) / div);
+ return (a_div * x + b_div) / div;
+}
+
+double WindowedLinearFitEstimator::estimate(double x, double y) {
+ /*
+ * TODO: We could update the head by adding the new sample to it
+ * and amplifying it, but this approach can lead to unbounded
+ * error. Instead, we recalculate the head at each step, which
+ * is computationally more expensive. We could balance the two
+ * methods by recalculating just before the error becomes
+ * significant.
+ */
+ const bool update_head = false;
+ if (update_head) {
+ // add new sample to the head
+ mHead.scale(mHeadFactorInv); // amplify head
+ mHead.add(x, y, mFirstWeight);
+ }
+
+ /*
+ * TRICKY: place elements into the circular buffer at decreasing
+ * indices, so that we can access past elements by addition
+ * (thereby avoiding potentially negative indices.)
+ */
+ if (mNumSamples >= mHeadLength) {
+ // move last head sample from head to the main window
+ size_t lastHeadIx = (mSampleIx + mHeadLength) % mHistoryLength;
+ if (update_head) {
+ mHead.add(mXHistory[lastHeadIx], mYHistory[lastHeadIx], -1.); // remove
+ }
+ mMain.add(mXHistory[lastHeadIx], mYHistory[lastHeadIx], 1.);
+ if (mNumSamples >= mHistoryLength) {
+ // move last main sample from main window to tail
+ mMain.add(mXHistory[mSampleIx], mYHistory[mSampleIx], -1.); // remove
+ mTail.add(mXHistory[mSampleIx], mYHistory[mSampleIx], 1.);
+ mTail.scale(mTailFactor); // attenuate tail
+ }
+ }
+
+ mXHistory.editItemAt(mSampleIx) = x;
+ mYHistory.editItemAt(mSampleIx) = y;
+ if (mNumSamples < mHistoryLength) {
+ ++mNumSamples;
+ }
+
+ // recalculate head unless we were using the update method
+ if (!update_head) {
+ mHead.reset();
+ double w = mFirstWeight;
+ for (size_t headIx = 0; headIx < mHeadLength && headIx < mNumSamples; ++headIx) {
+ size_t ix = (mSampleIx + headIx) % mHistoryLength;
+ mHead.add(mXHistory[ix], mYHistory[ix], w);
+ w *= mHeadFactorInv;
+ }
+ }
+
+ if (mSampleIx > 0) {
+ --mSampleIx;
+ } else {
+ mSampleIx = mHistoryLength - 1;
+ }
+
+ // return estimation result
+ LinearFit total;
+ total.combine(mHead);
+ total.combine(mMain);
+ total.combine(mTail);
+ return total.interpolate(x);
+}
+
+void WindowedLinearFitEstimator::reset() {
+ mHead.reset();
+ mMain.reset();
+ mTail.reset();
+ mNumSamples = 0;
+ mSampleIx = mHistoryLength - 1;
+}
+
+}; // namespace android
+
+
diff --git a/media/libstagefright/include/AwesomePlayer.h b/media/libstagefright/include/AwesomePlayer.h
index a81bbba..77d65e0 100644
--- a/media/libstagefright/include/AwesomePlayer.h
+++ b/media/libstagefright/include/AwesomePlayer.h
@@ -32,6 +32,7 @@
namespace android {
struct AudioPlayer;
+struct ClockEstimator;
struct DataSource;
struct MediaBuffer;
struct MediaExtractor;
@@ -236,6 +237,7 @@
MediaBuffer *mVideoBuffer;
+ sp<ClockEstimator> mClockEstimator;
sp<HTTPBase> mConnectingDataSource;
sp<NuCachedSource2> mCachedSource;
@@ -296,6 +298,7 @@
bool getBitrate(int64_t *bitrate);
+ int64_t estimateRealTimeUs(TimeSource *ts, int64_t systemTimeUs);
void finishSeekIfNecessary(int64_t videoTimeUs);
void ensureCacheIsFetching_l();
diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp
index 2f874f5..16d6f42 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.cpp
+++ b/services/camera/libcameraservice/device3/Camera3Device.cpp
@@ -1936,9 +1936,6 @@
if (msg->message.error.error_code == CAMERA3_MSG_ERROR_DEVICE) {
listener->notifyError(ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE,
resultExtras);
- } else {
- listener->notifyError(ICameraDeviceCallbacks::ERROR_CAMERA_SERVICE,
- resultExtras);
}
} else {
ALOGE("Camera %d: %s: no listener available", mId, __FUNCTION__);