1 /*
2  * Copyright 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define ATRACE_TAG ATRACE_TAG_GRAPHICS
19 #undef LOG_TAG
20 #define LOG_TAG "RegionSamplingThread"
21 
22 #include "RegionSamplingThread.h"
23 
24 #include <cutils/properties.h>
25 #include <gui/IRegionSamplingListener.h>
26 #include <utils/Trace.h>
27 #include <string>
28 
29 #include <compositionengine/Display.h>
30 #include <compositionengine/impl/OutputCompositionState.h>
31 #include "DisplayDevice.h"
32 #include "Layer.h"
33 #include "SurfaceFlinger.h"
34 
35 namespace android {
36 using namespace std::chrono_literals;
37 
38 template <typename T>
39 struct SpHash {
operator ()android::SpHash40     size_t operator()(const sp<T>& p) const { return std::hash<T*>()(p.get()); }
41 };
42 
43 constexpr auto lumaSamplingStepTag = "LumaSamplingStep";
44 enum class samplingStep {
45     noWorkNeeded,
46     idleTimerWaiting,
47     waitForQuietFrame,
48     waitForZeroPhase,
49     waitForSamplePhase,
50     sample
51 };
52 
53 constexpr auto timeForRegionSampling = 5000000ns;
54 constexpr auto maxRegionSamplingSkips = 10;
55 constexpr auto defaultRegionSamplingOffset = -3ms;
56 constexpr auto defaultRegionSamplingPeriod = 100ms;
57 constexpr auto defaultRegionSamplingTimerTimeout = 100ms;
58 // TODO: (b/127403193) duration to string conversion could probably be constexpr
59 template <typename Rep, typename Per>
toNsString(std::chrono::duration<Rep,Per> t)60 inline std::string toNsString(std::chrono::duration<Rep, Per> t) {
61     return std::to_string(std::chrono::duration_cast<std::chrono::nanoseconds>(t).count());
62 }
63 
EnvironmentTimingTunables()64 RegionSamplingThread::EnvironmentTimingTunables::EnvironmentTimingTunables() {
65     char value[PROPERTY_VALUE_MAX] = {};
66 
67     property_get("debug.sf.region_sampling_offset_ns", value,
68                  toNsString(defaultRegionSamplingOffset).c_str());
69     int const samplingOffsetNsRaw = atoi(value);
70 
71     property_get("debug.sf.region_sampling_period_ns", value,
72                  toNsString(defaultRegionSamplingPeriod).c_str());
73     int const samplingPeriodNsRaw = atoi(value);
74 
75     property_get("debug.sf.region_sampling_timer_timeout_ns", value,
76                  toNsString(defaultRegionSamplingTimerTimeout).c_str());
77     int const samplingTimerTimeoutNsRaw = atoi(value);
78 
79     if ((samplingPeriodNsRaw < 0) || (samplingTimerTimeoutNsRaw < 0)) {
80         ALOGW("User-specified sampling tuning options nonsensical. Using defaults");
81         mSamplingOffset = defaultRegionSamplingOffset;
82         mSamplingPeriod = defaultRegionSamplingPeriod;
83         mSamplingTimerTimeout = defaultRegionSamplingTimerTimeout;
84     } else {
85         mSamplingOffset = std::chrono::nanoseconds(samplingOffsetNsRaw);
86         mSamplingPeriod = std::chrono::nanoseconds(samplingPeriodNsRaw);
87         mSamplingTimerTimeout = std::chrono::nanoseconds(samplingTimerTimeoutNsRaw);
88     }
89 }
90 
91 struct SamplingOffsetCallback : DispSync::Callback {
SamplingOffsetCallbackandroid::SamplingOffsetCallback92     SamplingOffsetCallback(RegionSamplingThread& samplingThread, Scheduler& scheduler,
93                            std::chrono::nanoseconds targetSamplingOffset)
94           : mRegionSamplingThread(samplingThread),
95             mScheduler(scheduler),
96             mTargetSamplingOffset(targetSamplingOffset) {}
97 
~SamplingOffsetCallbackandroid::SamplingOffsetCallback98     ~SamplingOffsetCallback() { stopVsyncListener(); }
99 
100     SamplingOffsetCallback(const SamplingOffsetCallback&) = delete;
101     SamplingOffsetCallback& operator=(const SamplingOffsetCallback&) = delete;
102 
startVsyncListenerandroid::SamplingOffsetCallback103     void startVsyncListener() {
104         std::lock_guard lock(mMutex);
105         if (mVsyncListening) return;
106 
107         mPhaseIntervalSetting = Phase::ZERO;
108         mScheduler.withPrimaryDispSync([this](android::DispSync& sync) {
109             sync.addEventListener("SamplingThreadDispSyncListener", 0, this, mLastCallbackTime);
110         });
111         mVsyncListening = true;
112     }
113 
stopVsyncListenerandroid::SamplingOffsetCallback114     void stopVsyncListener() {
115         std::lock_guard lock(mMutex);
116         stopVsyncListenerLocked();
117     }
118 
119 private:
stopVsyncListenerLockedandroid::SamplingOffsetCallback120     void stopVsyncListenerLocked() /*REQUIRES(mMutex)*/ {
121         if (!mVsyncListening) return;
122 
123         mScheduler.withPrimaryDispSync([this](android::DispSync& sync) {
124             sync.removeEventListener(this, &mLastCallbackTime);
125         });
126         mVsyncListening = false;
127     }
128 
onDispSyncEventandroid::SamplingOffsetCallback129     void onDispSyncEvent(nsecs_t /* when */) final {
130         std::unique_lock<decltype(mMutex)> lock(mMutex);
131 
132         if (mPhaseIntervalSetting == Phase::ZERO) {
133             ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForSamplePhase));
134             mPhaseIntervalSetting = Phase::SAMPLING;
135             mScheduler.withPrimaryDispSync([this](android::DispSync& sync) {
136                 sync.changePhaseOffset(this, mTargetSamplingOffset.count());
137             });
138             return;
139         }
140 
141         if (mPhaseIntervalSetting == Phase::SAMPLING) {
142             mPhaseIntervalSetting = Phase::ZERO;
143             mScheduler.withPrimaryDispSync(
144                     [this](android::DispSync& sync) { sync.changePhaseOffset(this, 0); });
145             stopVsyncListenerLocked();
146             lock.unlock();
147             mRegionSamplingThread.notifySamplingOffset();
148             return;
149         }
150     }
151 
152     RegionSamplingThread& mRegionSamplingThread;
153     Scheduler& mScheduler;
154     const std::chrono::nanoseconds mTargetSamplingOffset;
155     mutable std::mutex mMutex;
156     nsecs_t mLastCallbackTime = 0;
157     enum class Phase {
158         ZERO,
159         SAMPLING
160     } mPhaseIntervalSetting /*GUARDED_BY(mMutex) macro doesnt work with unique_lock?*/
161             = Phase::ZERO;
162     bool mVsyncListening /*GUARDED_BY(mMutex)*/ = false;
163 };
164 
RegionSamplingThread(SurfaceFlinger & flinger,Scheduler & scheduler,const TimingTunables & tunables)165 RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger, Scheduler& scheduler,
166                                            const TimingTunables& tunables)
167       : mFlinger(flinger),
168         mScheduler(scheduler),
169         mTunables(tunables),
170         mIdleTimer(std::chrono::duration_cast<std::chrono::milliseconds>(
171                            mTunables.mSamplingTimerTimeout),
172                    [] {}, [this] { checkForStaleLuma(); }),
173         mPhaseCallback(std::make_unique<SamplingOffsetCallback>(*this, mScheduler,
174                                                                 tunables.mSamplingOffset)),
175         lastSampleTime(0ns) {
__anonedeb02ff0702() 176     mThread = std::thread([this]() { threadMain(); });
177     pthread_setname_np(mThread.native_handle(), "RegionSamplingThread");
178     mIdleTimer.start();
179 }
180 
RegionSamplingThread(SurfaceFlinger & flinger,Scheduler & scheduler)181 RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger, Scheduler& scheduler)
182       : RegionSamplingThread(flinger, scheduler,
183                              TimingTunables{defaultRegionSamplingOffset,
184                                             defaultRegionSamplingPeriod,
185                                             defaultRegionSamplingTimerTimeout}) {}
186 
~RegionSamplingThread()187 RegionSamplingThread::~RegionSamplingThread() {
188     mIdleTimer.stop();
189 
190     {
191         std::lock_guard lock(mThreadControlMutex);
192         mRunning = false;
193         mCondition.notify_one();
194     }
195 
196     if (mThread.joinable()) {
197         mThread.join();
198     }
199 }
200 
addListener(const Rect & samplingArea,const sp<IBinder> & stopLayerHandle,const sp<IRegionSamplingListener> & listener)201 void RegionSamplingThread::addListener(const Rect& samplingArea, const sp<IBinder>& stopLayerHandle,
202                                        const sp<IRegionSamplingListener>& listener) {
203     wp<Layer> stopLayer;
204     if (stopLayerHandle != nullptr && stopLayerHandle->localBinder() != nullptr) {
205         stopLayer = static_cast<Layer::Handle*>(stopLayerHandle.get())->owner;
206     }
207 
208     sp<IBinder> asBinder = IInterface::asBinder(listener);
209     asBinder->linkToDeath(this);
210     std::lock_guard lock(mSamplingMutex);
211     mDescriptors.emplace(wp<IBinder>(asBinder), Descriptor{samplingArea, stopLayer, listener});
212 }
213 
removeListener(const sp<IRegionSamplingListener> & listener)214 void RegionSamplingThread::removeListener(const sp<IRegionSamplingListener>& listener) {
215     std::lock_guard lock(mSamplingMutex);
216     mDescriptors.erase(wp<IBinder>(IInterface::asBinder(listener)));
217 }
218 
checkForStaleLuma()219 void RegionSamplingThread::checkForStaleLuma() {
220     std::lock_guard lock(mThreadControlMutex);
221 
222     if (mDiscardedFrames > 0) {
223         ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForZeroPhase));
224         mDiscardedFrames = 0;
225         mPhaseCallback->startVsyncListener();
226     }
227 }
228 
notifyNewContent()229 void RegionSamplingThread::notifyNewContent() {
230     doSample();
231 }
232 
notifySamplingOffset()233 void RegionSamplingThread::notifySamplingOffset() {
234     doSample();
235 }
236 
doSample()237 void RegionSamplingThread::doSample() {
238     std::lock_guard lock(mThreadControlMutex);
239     auto now = std::chrono::nanoseconds(systemTime(SYSTEM_TIME_MONOTONIC));
240     if (lastSampleTime + mTunables.mSamplingPeriod > now) {
241         ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::idleTimerWaiting));
242         if (mDiscardedFrames == 0) mDiscardedFrames++;
243         return;
244     }
245     if (mDiscardedFrames < maxRegionSamplingSkips) {
246         // If there is relatively little time left for surfaceflinger
247         // until the next vsync deadline, defer this sampling work
248         // to a later frame, when hopefully there will be more time.
249         DisplayStatInfo stats;
250         mScheduler.getDisplayStatInfo(&stats);
251         if (std::chrono::nanoseconds(stats.vsyncTime) - now < timeForRegionSampling) {
252             ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForQuietFrame));
253             mDiscardedFrames++;
254             return;
255         }
256     }
257 
258     ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::sample));
259 
260     mDiscardedFrames = 0;
261     lastSampleTime = now;
262 
263     mIdleTimer.reset();
264     mPhaseCallback->stopVsyncListener();
265 
266     mSampleRequested = true;
267     mCondition.notify_one();
268 }
269 
binderDied(const wp<IBinder> & who)270 void RegionSamplingThread::binderDied(const wp<IBinder>& who) {
271     std::lock_guard lock(mSamplingMutex);
272     mDescriptors.erase(who);
273 }
274 
275 namespace {
276 // Using Rec. 709 primaries
getLuma(float r,float g,float b)277 inline float getLuma(float r, float g, float b) {
278     constexpr auto rec709_red_primary = 0.2126f;
279     constexpr auto rec709_green_primary = 0.7152f;
280     constexpr auto rec709_blue_primary = 0.0722f;
281     return rec709_red_primary * r + rec709_green_primary * g + rec709_blue_primary * b;
282 }
283 } // anonymous namespace
284 
sampleArea(const uint32_t * data,int32_t width,int32_t height,int32_t stride,uint32_t orientation,const Rect & sample_area)285 float sampleArea(const uint32_t* data, int32_t width, int32_t height, int32_t stride,
286                  uint32_t orientation, const Rect& sample_area) {
287     if (!sample_area.isValid() || (sample_area.getWidth() > width) ||
288         (sample_area.getHeight() > height)) {
289         ALOGE("invalid sampling region requested");
290         return 0.0f;
291     }
292 
293     // (b/133849373) ROT_90 screencap images produced upside down
294     auto area = sample_area;
295     if (orientation & ui::Transform::ROT_90) {
296         area.top = height - area.top;
297         area.bottom = height - area.bottom;
298         std::swap(area.top, area.bottom);
299 
300         area.left = width - area.left;
301         area.right = width - area.right;
302         std::swap(area.left, area.right);
303     }
304 
305     std::array<int32_t, 256> brightnessBuckets = {};
306     const int32_t majoritySampleNum = area.getWidth() * area.getHeight() / 2;
307 
308     for (int32_t row = area.top; row < area.bottom; ++row) {
309         const uint32_t* rowBase = data + row * stride;
310         for (int32_t column = area.left; column < area.right; ++column) {
311             uint32_t pixel = rowBase[column];
312             const float r = pixel & 0xFF;
313             const float g = (pixel >> 8) & 0xFF;
314             const float b = (pixel >> 16) & 0xFF;
315             const uint8_t luma = std::round(getLuma(r, g, b));
316             ++brightnessBuckets[luma];
317             if (brightnessBuckets[luma] > majoritySampleNum) return luma / 255.0f;
318         }
319     }
320 
321     int32_t accumulated = 0;
322     size_t bucket = 0;
323     for (; bucket < brightnessBuckets.size(); bucket++) {
324         accumulated += brightnessBuckets[bucket];
325         if (accumulated > majoritySampleNum) break;
326     }
327 
328     return bucket / 255.0f;
329 }
330 
sampleBuffer(const sp<GraphicBuffer> & buffer,const Point & leftTop,const std::vector<RegionSamplingThread::Descriptor> & descriptors,uint32_t orientation)331 std::vector<float> RegionSamplingThread::sampleBuffer(
332         const sp<GraphicBuffer>& buffer, const Point& leftTop,
333         const std::vector<RegionSamplingThread::Descriptor>& descriptors, uint32_t orientation) {
334     void* data_raw = nullptr;
335     buffer->lock(GRALLOC_USAGE_SW_READ_OFTEN, &data_raw);
336     std::shared_ptr<uint32_t> data(reinterpret_cast<uint32_t*>(data_raw),
337                                    [&buffer](auto) { buffer->unlock(); });
338     if (!data) return {};
339 
340     const int32_t width = buffer->getWidth();
341     const int32_t height = buffer->getHeight();
342     const int32_t stride = buffer->getStride();
343     std::vector<float> lumas(descriptors.size());
344     std::transform(descriptors.begin(), descriptors.end(), lumas.begin(),
345                    [&](auto const& descriptor) {
346                        return sampleArea(data.get(), width, height, stride, orientation,
347                                          descriptor.area - leftTop);
348                    });
349     return lumas;
350 }
351 
captureSample()352 void RegionSamplingThread::captureSample() {
353     ATRACE_CALL();
354     std::lock_guard lock(mSamplingMutex);
355 
356     if (mDescriptors.empty()) {
357         return;
358     }
359 
360     const auto device = mFlinger.getDefaultDisplayDevice();
361     const auto orientation = [](uint32_t orientation) {
362         switch (orientation) {
363             default:
364             case DisplayState::eOrientationDefault:
365                 return ui::Transform::ROT_0;
366             case DisplayState::eOrientation90:
367                 return ui::Transform::ROT_90;
368             case DisplayState::eOrientation180:
369                 return ui::Transform::ROT_180;
370             case DisplayState::eOrientation270:
371                 return ui::Transform::ROT_270;
372         }
373     }(device->getOrientation());
374 
375     std::vector<RegionSamplingThread::Descriptor> descriptors;
376     Region sampleRegion;
377     for (const auto& [listener, descriptor] : mDescriptors) {
378         sampleRegion.orSelf(descriptor.area);
379         descriptors.emplace_back(descriptor);
380     }
381 
382     const Rect sampledArea = sampleRegion.bounds();
383 
384     auto dx = 0;
385     auto dy = 0;
386     switch (orientation) {
387         case ui::Transform::ROT_90:
388             dx = device->getWidth();
389             break;
390         case ui::Transform::ROT_180:
391             dx = device->getWidth();
392             dy = device->getHeight();
393             break;
394         case ui::Transform::ROT_270:
395             dy = device->getHeight();
396             break;
397         default:
398             break;
399     }
400 
401     ui::Transform t(orientation);
402     auto screencapRegion = t.transform(sampleRegion);
403     screencapRegion = screencapRegion.translate(dx, dy);
404     DisplayRenderArea renderArea(device, screencapRegion.bounds(), sampledArea.getWidth(),
405                                  sampledArea.getHeight(), ui::Dataspace::V0_SRGB, orientation);
406 
407     std::unordered_set<sp<IRegionSamplingListener>, SpHash<IRegionSamplingListener>> listeners;
408 
409     auto traverseLayers = [&](const LayerVector::Visitor& visitor) {
410         bool stopLayerFound = false;
411         auto filterVisitor = [&](Layer* layer) {
412             // We don't want to capture any layers beyond the stop layer
413             if (stopLayerFound) return;
414 
415             // Likewise if we just found a stop layer, set the flag and abort
416             for (const auto& [area, stopLayer, listener] : descriptors) {
417                 if (layer == stopLayer.promote().get()) {
418                     stopLayerFound = true;
419                     return;
420                 }
421             }
422 
423             // Compute the layer's position on the screen
424             const Rect bounds = Rect(layer->getBounds());
425             const ui::Transform transform = layer->getTransform();
426             constexpr bool roundOutwards = true;
427             Rect transformed = transform.transform(bounds, roundOutwards);
428 
429             // If this layer doesn't intersect with the larger sampledArea, skip capturing it
430             Rect ignore;
431             if (!transformed.intersect(sampledArea, &ignore)) return;
432 
433             // If the layer doesn't intersect a sampling area, skip capturing it
434             bool intersectsAnyArea = false;
435             for (const auto& [area, stopLayer, listener] : descriptors) {
436                 if (transformed.intersect(area, &ignore)) {
437                     intersectsAnyArea = true;
438                     listeners.insert(listener);
439                 }
440             }
441             if (!intersectsAnyArea) return;
442 
443             ALOGV("Traversing [%s] [%d, %d, %d, %d]", layer->getName().string(), bounds.left,
444                   bounds.top, bounds.right, bounds.bottom);
445             visitor(layer);
446         };
447         mFlinger.traverseLayersInDisplay(device, filterVisitor);
448     };
449 
450     sp<GraphicBuffer> buffer = nullptr;
451     if (mCachedBuffer && mCachedBuffer->getWidth() == sampledArea.getWidth() &&
452         mCachedBuffer->getHeight() == sampledArea.getHeight()) {
453         buffer = mCachedBuffer;
454     } else {
455         const uint32_t usage = GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_HW_RENDER;
456         buffer = new GraphicBuffer(sampledArea.getWidth(), sampledArea.getHeight(),
457                                    PIXEL_FORMAT_RGBA_8888, 1, usage, "RegionSamplingThread");
458     }
459 
460     bool ignored;
461     mFlinger.captureScreenCommon(renderArea, traverseLayers, buffer, false, ignored);
462 
463     std::vector<Descriptor> activeDescriptors;
464     for (const auto& descriptor : descriptors) {
465         if (listeners.count(descriptor.listener) != 0) {
466             activeDescriptors.emplace_back(descriptor);
467         }
468     }
469 
470     ALOGV("Sampling %zu descriptors", activeDescriptors.size());
471     std::vector<float> lumas =
472             sampleBuffer(buffer, sampledArea.leftTop(), activeDescriptors, orientation);
473     if (lumas.size() != activeDescriptors.size()) {
474         ALOGW("collected %zu median luma values for %zu descriptors", lumas.size(),
475               activeDescriptors.size());
476         return;
477     }
478 
479     for (size_t d = 0; d < activeDescriptors.size(); ++d) {
480         activeDescriptors[d].listener->onSampleCollected(lumas[d]);
481     }
482 
483     // Extend the lifetime of mCachedBuffer from the previous frame to here to ensure that:
484     // 1) The region sampling thread is the last owner of the buffer, and the freeing of the buffer
485     // happens in this thread, as opposed to the main thread.
486     // 2) The listener(s) receive their notifications prior to freeing the buffer.
487     mCachedBuffer = buffer;
488     ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::noWorkNeeded));
489 }
490 
491 // NO_THREAD_SAFETY_ANALYSIS is because std::unique_lock presently lacks thread safety annotations.
threadMain()492 void RegionSamplingThread::threadMain() NO_THREAD_SAFETY_ANALYSIS {
493     std::unique_lock<std::mutex> lock(mThreadControlMutex);
494     while (mRunning) {
495         if (mSampleRequested) {
496             mSampleRequested = false;
497             lock.unlock();
498             captureSample();
499             lock.lock();
500         }
501         mCondition.wait(lock, [this]() REQUIRES(mThreadControlMutex) {
502             return mSampleRequested || !mRunning;
503         });
504     }
505 }
506 
507 } // namespace android
508