1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "VideoFrameSchedulerBase"
19 #include <utils/Log.h>
20 #define ATRACE_TAG ATRACE_TAG_VIDEO
21 #include <utils/Trace.h>
22 #include <utils/Vector.h>
23
24 #include <media/stagefright/foundation/ADebug.h>
25 #include <media/stagefright/foundation/AUtils.h>
26 #include <media/stagefright/VideoFrameSchedulerBase.h>
27
28 namespace android {
29
30 template<class T>
compare(const T * lhs,const T * rhs)31 static int compare(const T *lhs, const T *rhs) {
32 if (*lhs < *rhs) {
33 return -1;
34 } else if (*lhs > *rhs) {
35 return 1;
36 } else {
37 return 0;
38 }
39 }
40
41 /* ======================================================================= */
42 /* PLL */
43 /* ======================================================================= */
44
45 static const size_t kMinSamplesToStartPrime = 3;
46 static const size_t kMinSamplesToStopPrime = VideoFrameSchedulerBase::kHistorySize;
47 static const size_t kMinSamplesToEstimatePeriod = 3;
48 static const size_t kMaxSamplesToEstimatePeriod = VideoFrameSchedulerBase::kHistorySize;
49
50 static const size_t kPrecision = 12;
51 static const int64_t kErrorThreshold = (1 << (kPrecision * 2)) / 10;
52 static const int64_t kMultiplesThresholdDiv = 4; // 25%
53 static const int64_t kReFitThresholdDiv = 100; // 1%
54 static const nsecs_t kMaxAllowedFrameSkip = VideoFrameSchedulerBase::kNanosIn1s; // 1 sec
55 static const nsecs_t kMinPeriod = VideoFrameSchedulerBase::kNanosIn1s / 120; // 120Hz
56 static const nsecs_t kRefitRefreshPeriod = 10 * VideoFrameSchedulerBase::kNanosIn1s; // 10 sec
57
PLL()58 VideoFrameSchedulerBase::PLL::PLL()
59 : mPeriod(-1),
60 mPhase(0),
61 mPrimed(false),
62 mSamplesUsedForPriming(0),
63 mLastTime(-1),
64 mNumSamples(0) {
65 }
66
reset(float fps)67 void VideoFrameSchedulerBase::PLL::reset(float fps) {
68 //test();
69
70 mSamplesUsedForPriming = 0;
71 mLastTime = -1;
72
73 // set up or reset video PLL
74 if (fps <= 0.f) {
75 mPeriod = -1;
76 mPrimed = false;
77 } else {
78 ALOGV("reset at %.1f fps", fps);
79 mPeriod = (nsecs_t)(1e9 / fps + 0.5);
80 mPrimed = true;
81 }
82
83 restart();
84 }
85
86 // reset PLL but keep previous period estimate
restart()87 void VideoFrameSchedulerBase::PLL::restart() {
88 mNumSamples = 0;
89 mPhase = -1;
90 }
91
92 #if 0
93
94 void VideoFrameSchedulerBase::PLL::test() {
95 nsecs_t period = VideoFrameSchedulerBase::kNanosIn1s / 60;
96 mTimes[0] = 0;
97 mTimes[1] = period;
98 mTimes[2] = period * 3;
99 mTimes[3] = period * 4;
100 mTimes[4] = period * 7;
101 mTimes[5] = period * 8;
102 mTimes[6] = period * 10;
103 mTimes[7] = period * 12;
104 mNumSamples = 8;
105 int64_t a, b, err;
106 fit(0, period * 12 / 7, 8, &a, &b, &err);
107 // a = 0.8(5)+
108 // b = -0.14097(2)+
109 // err = 0.2750578(703)+
110 ALOGD("a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)",
111 (long long)a, (a / (float)(1 << kPrecision)),
112 (long long)b, (b / (float)(1 << kPrecision)),
113 (long long)err, (err / (float)(1 << (kPrecision * 2))));
114 }
115
116 #endif
117
118 // If overflow happens, the value is already incorrect, and no mater what value we get is OK.
119 // And this part of calculation is not important, so it's OK to simply disable overflow check
120 // instead of using double which makes code more complicated.
121 __attribute__((no_sanitize("integer")))
fit(nsecs_t phase,nsecs_t period,size_t numSamplesToUse,int64_t * a,int64_t * b,int64_t * err)122 bool VideoFrameSchedulerBase::PLL::fit(
123 nsecs_t phase, nsecs_t period, size_t numSamplesToUse,
124 int64_t *a, int64_t *b, int64_t *err) {
125 if (numSamplesToUse > mNumSamples) {
126 numSamplesToUse = mNumSamples;
127 }
128
129 if ((period >> kPrecision) == 0 ) {
130 ALOGW("Period is 0, or after including precision is 0 - would cause div0, returning");
131 return false;
132 }
133
134 int64_t sumX = 0;
135 int64_t sumXX = 0;
136 int64_t sumXY = 0;
137 int64_t sumYY = 0;
138 int64_t sumY = 0;
139
140 int64_t x = 0; // x usually is in [0..numSamplesToUse)
141 nsecs_t lastTime;
142 for (size_t i = 0; i < numSamplesToUse; i++) {
143 size_t ix = (mNumSamples - numSamplesToUse + i) % kHistorySize;
144 nsecs_t time = mTimes[ix];
145 if (i > 0) {
146 x += divRound(time - lastTime, period);
147 }
148 // y is usually in [-numSamplesToUse..numSamplesToUse+kRefitRefreshPeriod/kMinPeriod) << kPrecision
149 // ideally in [0..numSamplesToUse), but shifted by -numSamplesToUse during
150 // priming, and possibly shifted by up to kRefitRefreshPeriod/kMinPeriod
151 // while we are not refitting.
152 int64_t y = divRound(time - phase, period >> kPrecision);
153 sumX += x;
154 sumY += y;
155 sumXX += x * x;
156 sumXY += x * y;
157 sumYY += y * y;
158 lastTime = time;
159 }
160
161 int64_t div = (int64_t)numSamplesToUse * sumXX - sumX * sumX;
162 if (div == 0) {
163 return false;
164 }
165
166 int64_t a_nom = (int64_t)numSamplesToUse * sumXY - sumX * sumY;
167 int64_t b_nom = sumXX * sumY - sumX * sumXY;
168 *a = divRound(a_nom, div);
169 *b = divRound(b_nom, div);
170 // don't use a and b directly as the rounding error is significant
171 *err = sumYY - divRound(a_nom * sumXY + b_nom * sumY, div);
172 ALOGV("fitting[%zu] a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)",
173 numSamplesToUse,
174 (long long)*a, (*a / (float)(1 << kPrecision)),
175 (long long)*b, (*b / (float)(1 << kPrecision)),
176 (long long)*err, (*err / (float)(1 << (kPrecision * 2))));
177 return true;
178 }
179
prime(size_t numSamplesToUse)180 void VideoFrameSchedulerBase::PLL::prime(size_t numSamplesToUse) {
181 if (numSamplesToUse > mNumSamples) {
182 numSamplesToUse = mNumSamples;
183 }
184 CHECK(numSamplesToUse >= 3); // must have at least 3 samples
185
186 // estimate video framerate from deltas between timestamps, and
187 // 2nd order deltas
188 Vector<nsecs_t> deltas;
189 nsecs_t lastTime, firstTime;
190 for (size_t i = 0; i < numSamplesToUse; ++i) {
191 size_t index = (mNumSamples - numSamplesToUse + i) % kHistorySize;
192 nsecs_t time = mTimes[index];
193 if (i > 0) {
194 if (time - lastTime > kMinPeriod) {
195 //ALOGV("delta: %lld", (long long)(time - lastTime));
196 deltas.push(time - lastTime);
197 }
198 } else {
199 firstTime = time;
200 }
201 lastTime = time;
202 }
203 deltas.sort(compare<nsecs_t>);
204 size_t numDeltas = deltas.size();
205 if (numDeltas > 1) {
206 nsecs_t deltaMinLimit = max(deltas[0] / kMultiplesThresholdDiv, kMinPeriod);
207 nsecs_t deltaMaxLimit = deltas[numDeltas / 2] * kMultiplesThresholdDiv;
208 for (size_t i = numDeltas / 2 + 1; i < numDeltas; ++i) {
209 if (deltas[i] > deltaMaxLimit) {
210 deltas.resize(i);
211 numDeltas = i;
212 break;
213 }
214 }
215 for (size_t i = 1; i < numDeltas; ++i) {
216 nsecs_t delta2nd = deltas[i] - deltas[i - 1];
217 if (delta2nd >= deltaMinLimit) {
218 //ALOGV("delta2: %lld", (long long)(delta2nd));
219 deltas.push(delta2nd);
220 }
221 }
222 }
223
224 // use the one that yields the best match
225 int64_t bestScore;
226 for (size_t i = 0; i < deltas.size(); ++i) {
227 nsecs_t delta = deltas[i];
228 int64_t score = 0;
229 #if 1
230 // simplest score: number of deltas that are near multiples
231 size_t matches = 0;
232 for (size_t j = 0; j < deltas.size(); ++j) {
233 nsecs_t err = periodicError(deltas[j], delta);
234 if (err < delta / kMultiplesThresholdDiv) {
235 ++matches;
236 }
237 }
238 score = matches;
239 #if 0
240 // could be weighed by the (1 - normalized error)
241 if (numSamplesToUse >= kMinSamplesToEstimatePeriod) {
242 int64_t a, b, err;
243 fit(firstTime, delta, numSamplesToUse, &a, &b, &err);
244 err = (1 << (2 * kPrecision)) - err;
245 score *= max(0, err);
246 }
247 #endif
248 #else
249 // or use the error as a negative score
250 if (numSamplesToUse >= kMinSamplesToEstimatePeriod) {
251 int64_t a, b, err;
252 fit(firstTime, delta, numSamplesToUse, &a, &b, &err);
253 score = -delta * err;
254 }
255 #endif
256 if (i == 0 || score > bestScore) {
257 bestScore = score;
258 mPeriod = delta;
259 mPhase = firstTime;
260 }
261 }
262 ALOGV("priming[%zu] phase:%lld period:%lld",
263 numSamplesToUse, (long long)mPhase, (long long)mPeriod);
264 }
265
addSample(nsecs_t time)266 nsecs_t VideoFrameSchedulerBase::PLL::addSample(nsecs_t time) {
267 if (mLastTime >= 0
268 // if time goes backward, or we skipped rendering
269 && (time > mLastTime + kMaxAllowedFrameSkip || time < mLastTime)) {
270 restart();
271 }
272
273 mLastTime = time;
274 mTimes[mNumSamples % kHistorySize] = time;
275 ++mNumSamples;
276
277 bool doFit = time > mRefitAt;
278 if ((mPeriod <= 0 || !mPrimed) && mNumSamples >= kMinSamplesToStartPrime) {
279 prime(kMinSamplesToStopPrime);
280 ++mSamplesUsedForPriming;
281 doFit = true;
282 }
283 if (mPeriod > 0 && mNumSamples >= kMinSamplesToEstimatePeriod) {
284 if (mPhase < 0) {
285 // initialize phase to the current render time
286 mPhase = time;
287 doFit = true;
288 } else if (!doFit) {
289 int64_t err = periodicError(time - mPhase, mPeriod);
290 doFit = err > mPeriod / kReFitThresholdDiv;
291 }
292
293 if (doFit) {
294 int64_t a, b, err;
295 if (!fit(mPhase, mPeriod, kMaxSamplesToEstimatePeriod, &a, &b, &err)) {
296 // samples are not suitable for fitting. this means they are
297 // also not suitable for priming.
298 ALOGV("could not fit - keeping old period:%lld", (long long)mPeriod);
299 return mPeriod;
300 }
301
302 mRefitAt = time + kRefitRefreshPeriod;
303
304 mPhase += (mPeriod * b) >> kPrecision;
305 mPeriod = (mPeriod * a) >> kPrecision;
306 ALOGV("new phase:%lld period:%lld", (long long)mPhase, (long long)mPeriod);
307
308 if (err < kErrorThreshold) {
309 if (!mPrimed && mSamplesUsedForPriming >= kMinSamplesToStopPrime) {
310 mPrimed = true;
311 }
312 } else {
313 mPrimed = false;
314 mSamplesUsedForPriming = 0;
315 }
316 }
317 }
318 return mPeriod;
319 }
320
getPeriod() const321 nsecs_t VideoFrameSchedulerBase::PLL::getPeriod() const {
322 return mPrimed ? mPeriod : 0;
323 }
324
325 /* ======================================================================= */
326 /* Frame Scheduler */
327 /* ======================================================================= */
328
VideoFrameSchedulerBase()329 VideoFrameSchedulerBase::VideoFrameSchedulerBase()
330 : mVsyncTime(0),
331 mVsyncPeriod(0),
332 mVsyncRefreshAt(0),
333 mLastVsyncTime(-1),
334 mTimeCorrection(0) {
335 }
336
init(float videoFps)337 void VideoFrameSchedulerBase::init(float videoFps) {
338 updateVsync();
339
340 mLastVsyncTime = -1;
341 mTimeCorrection = 0;
342
343 mPll.reset(videoFps);
344 }
345
restart()346 void VideoFrameSchedulerBase::restart() {
347 mLastVsyncTime = -1;
348 mTimeCorrection = 0;
349
350 mPll.restart();
351 }
352
getVsyncPeriod()353 nsecs_t VideoFrameSchedulerBase::getVsyncPeriod() {
354 if (mVsyncPeriod > 0) {
355 return mVsyncPeriod;
356 }
357 return kDefaultVsyncPeriod;
358 }
359
getFrameRate()360 float VideoFrameSchedulerBase::getFrameRate() {
361 nsecs_t videoPeriod = mPll.getPeriod();
362 if (videoPeriod > 0) {
363 return 1e9 / videoPeriod;
364 }
365 return 0.f;
366 }
367
schedule(nsecs_t renderTime)368 nsecs_t VideoFrameSchedulerBase::schedule(nsecs_t renderTime) {
369 nsecs_t origRenderTime = renderTime;
370
371 nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC);
372 if (now >= mVsyncRefreshAt) {
373 updateVsync();
374 }
375
376 // without VSYNC info, there is nothing to do
377 if (mVsyncPeriod == 0) {
378 ALOGV("no vsync: render=%lld", (long long)renderTime);
379 return renderTime;
380 }
381
382 // ensure vsync time is well before (corrected) render time
383 if (mVsyncTime > renderTime - 4 * mVsyncPeriod) {
384 mVsyncTime -=
385 ((mVsyncTime - renderTime) / mVsyncPeriod + 5) * mVsyncPeriod;
386 }
387
388 // Video presentation takes place at the VSYNC _after_ renderTime. Adjust renderTime
389 // so this effectively becomes a rounding operation (to the _closest_ VSYNC.)
390 renderTime -= mVsyncPeriod / 2;
391
392 const nsecs_t videoPeriod = mPll.addSample(origRenderTime);
393 if (videoPeriod > 0) {
394 // Smooth out rendering
395 size_t N = 12;
396 nsecs_t fiveSixthDev =
397 abs(((videoPeriod * 5 + mVsyncPeriod) % (mVsyncPeriod * 6)) - mVsyncPeriod)
398 / (mVsyncPeriod / 100);
399 // use 20 samples if we are doing 5:6 ratio +- 1% (e.g. playing 50Hz on 60Hz)
400 if (fiveSixthDev < 12) { /* 12% / 6 = 2% */
401 N = 20;
402 }
403
404 nsecs_t offset = 0;
405 nsecs_t edgeRemainder = 0;
406 for (size_t i = 1; i <= N; i++) {
407 offset +=
408 (renderTime + mTimeCorrection + videoPeriod * i - mVsyncTime) % mVsyncPeriod;
409 edgeRemainder += (videoPeriod * i) % mVsyncPeriod;
410 }
411 mTimeCorrection += mVsyncPeriod / 2 - offset / (nsecs_t)N;
412 renderTime += mTimeCorrection;
413 nsecs_t correctionLimit = mVsyncPeriod * 3 / 5;
414 edgeRemainder = abs(edgeRemainder / (nsecs_t)N - mVsyncPeriod / 2);
415 if (edgeRemainder <= mVsyncPeriod / 3) {
416 correctionLimit /= 2;
417 }
418
419 // estimate how many VSYNCs a frame will spend on the display
420 nsecs_t nextVsyncTime =
421 renderTime + mVsyncPeriod - ((renderTime - mVsyncTime) % mVsyncPeriod);
422 if (mLastVsyncTime >= 0) {
423 size_t minVsyncsPerFrame = videoPeriod / mVsyncPeriod;
424 size_t vsyncsForLastFrame = divRound(nextVsyncTime - mLastVsyncTime, mVsyncPeriod);
425 bool vsyncsPerFrameAreNearlyConstant =
426 periodicError(videoPeriod, mVsyncPeriod) / (mVsyncPeriod / 20) == 0;
427
428 if (mTimeCorrection > correctionLimit &&
429 (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame > minVsyncsPerFrame)) {
430 // remove a VSYNC
431 mTimeCorrection -= mVsyncPeriod / 2;
432 renderTime -= mVsyncPeriod / 2;
433 nextVsyncTime -= mVsyncPeriod;
434 if (vsyncsForLastFrame > 0)
435 --vsyncsForLastFrame;
436 } else if (mTimeCorrection < -correctionLimit &&
437 (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame == minVsyncsPerFrame)) {
438 // add a VSYNC
439 mTimeCorrection += mVsyncPeriod / 2;
440 renderTime += mVsyncPeriod / 2;
441 nextVsyncTime += mVsyncPeriod;
442 if (vsyncsForLastFrame < ULONG_MAX)
443 ++vsyncsForLastFrame;
444 } else if (mTimeCorrection < -correctionLimit * 2
445 || mTimeCorrection > correctionLimit * 2) {
446 ALOGW("correction beyond limit: %lld vs %lld (vsyncs for last frame: %zu, min: %zu)"
447 " restarting. render=%lld",
448 (long long)mTimeCorrection, (long long)correctionLimit,
449 vsyncsForLastFrame, minVsyncsPerFrame, (long long)origRenderTime);
450 restart();
451 return origRenderTime;
452 }
453
454 ATRACE_INT("FRAME_VSYNCS", vsyncsForLastFrame);
455 }
456 mLastVsyncTime = nextVsyncTime;
457 }
458
459 // align rendertime to the center between VSYNC edges
460 renderTime -= (renderTime - mVsyncTime) % mVsyncPeriod;
461 renderTime += mVsyncPeriod / 2;
462 ALOGV("adjusting render: %lld => %lld", (long long)origRenderTime, (long long)renderTime);
463 ATRACE_INT("FRAME_FLIP_IN(ms)", (renderTime - now) / 1000000);
464 return renderTime;
465 }
466
~VideoFrameSchedulerBase()467 VideoFrameSchedulerBase::~VideoFrameSchedulerBase() {}
468
469 } // namespace android
470