1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <gtest/gtest_prod.h> 20 #include "anomaly/AnomalyTracker.h" 21 #include "condition/ConditionTimer.h" 22 #include "condition/ConditionTracker.h" 23 #include "external/PullDataReceiver.h" 24 #include "external/StatsPullerManager.h" 25 #include "matchers/EventMatcherWizard.h" 26 #include "stats_log_util.h" 27 #include "MetricProducer.h" 28 #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" 29 30 namespace android { 31 namespace os { 32 namespace statsd { 33 34 struct ValueBucket { 35 int64_t mBucketStartNs; 36 int64_t mBucketEndNs; 37 std::vector<int> valueIndex; 38 std::vector<Value> values; 39 // If the metric has no condition, then this field is just wasted. 40 // When we tune statsd memory usage in the future, this is a candidate to optimize. 41 int64_t mConditionTrueNs; 42 }; 43 44 45 // Aggregates values within buckets. 46 // 47 // There are different events that might complete a bucket 48 // - a condition change 49 // - an app upgrade 50 // - an alarm set to the end of the bucket 51 class ValueMetricProducer : public virtual MetricProducer, public virtual PullDataReceiver { 52 public: 53 ValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric, 54 const int conditionIndex, const sp<ConditionWizard>& conditionWizard, 55 const int whatMatcherIndex, 56 const sp<EventMatcherWizard>& matcherWizard, 57 const int pullTagId, const int64_t timeBaseNs, const int64_t startTimeNs, 58 const sp<StatsPullerManager>& pullerManager); 59 60 virtual ~ValueMetricProducer(); 61 62 // Process data pulled on bucket boundary. 63 void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data, 64 bool pullSuccess, int64_t originalPullTimeNs) override; 65 66 // ValueMetric needs special logic if it's a pulled atom. notifyAppUpgrade(const int64_t & eventTimeNs,const string & apk,const int uid,const int64_t version)67 void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, 68 const int64_t version) override { 69 std::lock_guard<std::mutex> lock(mMutex); 70 if (!mSplitBucketForAppUpgrade) { 71 return; 72 } 73 if (mIsPulled && mCondition) { 74 pullAndMatchEventsLocked(eventTimeNs, mCondition); 75 } 76 flushCurrentBucketLocked(eventTimeNs, eventTimeNs); 77 }; 78 79 protected: 80 void onMatchedLogEventInternalLocked( 81 const size_t matcherIndex, const MetricDimensionKey& eventKey, 82 const ConditionKey& conditionKey, bool condition, 83 const LogEvent& event) override; 84 85 private: 86 void onDumpReportLocked(const int64_t dumpTimeNs, 87 const bool include_current_partial_bucket, 88 const bool erase_data, 89 const DumpLatency dumpLatency, 90 std::set<string> *str_set, 91 android::util::ProtoOutputStream* protoOutput) override; 92 void clearPastBucketsLocked(const int64_t dumpTimeNs) override; 93 94 // Internal interface to handle active state change. 95 void onActiveStateChangedLocked(const int64_t& eventTimeNs) override; 96 97 // Internal interface to handle condition change. 98 void onConditionChangedLocked(const bool conditionMet, const int64_t eventTime) override; 99 100 // Internal interface to handle sliced condition change. 101 void onSlicedConditionMayChangeLocked(bool overallCondition, const int64_t eventTime) override; 102 103 // Internal function to calculate the current used bytes. 104 size_t byteSizeLocked() const override; 105 106 void dumpStatesLocked(FILE* out, bool verbose) const override; 107 108 // For pulled metrics, this method should only be called if a pull has be done. Else we will 109 // not have complete data for the bucket. 110 void flushIfNeededLocked(const int64_t& eventTime) override; 111 112 // For pulled metrics, this method should only be called if a pulled have be done. Else we will 113 // not have complete data for the bucket. 114 void flushCurrentBucketLocked(const int64_t& eventTimeNs, 115 const int64_t& nextBucketStartTimeNs) override; 116 117 void prepareFirstBucketLocked() override; 118 119 void dropDataLocked(const int64_t dropTimeNs) override; 120 121 // Calculate previous bucket end time based on current time. 122 int64_t calcPreviousBucketEndTime(const int64_t currentTimeNs); 123 124 // Calculate how many buckets are present between the current bucket and eventTimeNs. 125 int64_t calcBucketsForwardCount(const int64_t& eventTimeNs) const; 126 127 // Mark the data as invalid. 128 void invalidateCurrentBucket(); 129 void invalidateCurrentBucketWithoutResetBase(); 130 131 const int mWhatMatcherIndex; 132 133 sp<EventMatcherWizard> mEventMatcherWizard; 134 135 sp<StatsPullerManager> mPullerManager; 136 137 // Value fields for matching. 138 std::vector<Matcher> mFieldMatchers; 139 140 // Value fields for matching. 141 std::set<MetricDimensionKey> mMatchedMetricDimensionKeys; 142 143 // tagId for pulled data. -1 if this is not pulled 144 const int mPullTagId; 145 146 // if this is pulled metric 147 const bool mIsPulled; 148 149 // internal state of an ongoing aggregation bucket. 150 typedef struct { 151 // Index in multi value aggregation. 152 int valueIndex; 153 // Holds current base value of the dimension. Take diff and update if necessary. 154 Value base; 155 // Whether there is a base to diff to. 156 bool hasBase; 157 // Current value, depending on the aggregation type. 158 Value value; 159 // Number of samples collected. 160 int sampleSize; 161 // If this dimension has any non-tainted value. If not, don't report the 162 // dimension. 163 bool hasValue = false; 164 // Whether new data is seen in the bucket. 165 bool seenNewData = false; 166 } Interval; 167 168 std::unordered_map<MetricDimensionKey, std::vector<Interval>> mCurrentSlicedBucket; 169 170 std::unordered_map<MetricDimensionKey, int64_t> mCurrentFullBucket; 171 172 // Save the past buckets and we can clear when the StatsLogReport is dumped. 173 std::unordered_map<MetricDimensionKey, std::vector<ValueBucket>> mPastBuckets; 174 175 // Pairs of (elapsed start, elapsed end) denoting buckets that were skipped. 176 std::list<std::pair<int64_t, int64_t>> mSkippedBuckets; 177 178 const int64_t mMinBucketSizeNs; 179 180 // Util function to check whether the specified dimension hits the guardrail. 181 bool hitGuardRailLocked(const MetricDimensionKey& newKey); 182 bool hasReachedGuardRailLimit() const; 183 184 bool hitFullBucketGuardRailLocked(const MetricDimensionKey& newKey); 185 186 void pullAndMatchEventsLocked(const int64_t timestampNs, ConditionState condition); 187 188 void accumulateEvents(const std::vector<std::shared_ptr<LogEvent>>& allData, 189 int64_t originalPullTimeNs, int64_t eventElapsedTimeNs, 190 ConditionState condition); 191 192 ValueBucket buildPartialBucket(int64_t bucketEndTime, 193 const std::vector<Interval>& intervals); 194 void initCurrentSlicedBucket(int64_t nextBucketStartTimeNs); 195 void appendToFullBucket(int64_t eventTimeNs, int64_t fullBucketEndTimeNs); 196 197 // Reset diff base and mHasGlobalBase 198 void resetBase(); 199 200 static const size_t kBucketSize = sizeof(ValueBucket{}); 201 202 const size_t mDimensionSoftLimit; 203 204 const size_t mDimensionHardLimit; 205 206 const bool mUseAbsoluteValueOnReset; 207 208 const ValueMetric::AggregationType mAggregationType; 209 210 const bool mUseDiff; 211 212 const ValueMetric::ValueDirection mValueDirection; 213 214 const bool mSkipZeroDiffOutput; 215 216 // If true, use a zero value as base to compute the diff. 217 // This is used for new keys which are present in the new data but was not 218 // present in the base data. 219 // The default base will only be used if we have a global base. 220 const bool mUseZeroDefaultBase; 221 222 // For pulled metrics, this is always set to true whenever a pull succeeds. 223 // It is set to false when a pull fails, or upon condition change to false. 224 // This is used to decide if we have the right base data to compute the 225 // diff against. 226 bool mHasGlobalBase; 227 228 // Invalid bucket. There was a problem in collecting data in the current bucket so we cannot 229 // trust any of the data in this bucket. 230 // 231 // For instance, one pull failed. 232 bool mCurrentBucketIsInvalid; 233 234 const int64_t mMaxPullDelayNs; 235 236 const bool mSplitBucketForAppUpgrade; 237 238 ConditionTimer mConditionTimer; 239 240 FRIEND_TEST(ValueMetricProducerTest, TestAnomalyDetection); 241 FRIEND_TEST(ValueMetricProducerTest, TestBaseSetOnConditionChange); 242 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundariesOnAppUpgrade); 243 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundariesOnConditionChange); 244 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryNoCondition); 245 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition); 246 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2); 247 FRIEND_TEST(ValueMetricProducerTest, TestBucketIncludingUnknownConditionIsInvalid); 248 FRIEND_TEST(ValueMetricProducerTest, TestBucketInvalidIfGlobalBaseIsNotSet); 249 FRIEND_TEST(ValueMetricProducerTest, TestCalcPreviousBucketEndTime); 250 FRIEND_TEST(ValueMetricProducerTest, TestDataIsNotUpdatedWhenNoConditionChanged); 251 FRIEND_TEST(ValueMetricProducerTest, TestEmptyDataResetsBase_onBucketBoundary); 252 FRIEND_TEST(ValueMetricProducerTest, TestEmptyDataResetsBase_onConditionChanged); 253 FRIEND_TEST(ValueMetricProducerTest, TestEmptyDataResetsBase_onDataPulled); 254 FRIEND_TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition); 255 FRIEND_TEST(ValueMetricProducerTest, TestFirstBucket); 256 FRIEND_TEST(ValueMetricProducerTest, TestFullBucketResetWhenLastBucketInvalid); 257 FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenGuardRailHit); 258 FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenInitialPullFailed); 259 FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenLastPullFailed); 260 FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenOneConditionFailed); 261 FRIEND_TEST(ValueMetricProducerTest, TestLateOnDataPulledWithDiff); 262 FRIEND_TEST(ValueMetricProducerTest, TestLateOnDataPulledWithoutDiff); 263 FRIEND_TEST(ValueMetricProducerTest, TestPartialBucketCreated); 264 FRIEND_TEST(ValueMetricProducerTest, TestPartialResetOnBucketBoundaries); 265 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryFalse); 266 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryTrue); 267 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_withFailure); 268 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_withMultipleConditionChanges); 269 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_withoutCondition); 270 FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsNoCondition); 271 FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset); 272 FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsTakeZeroOnReset); 273 FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsWithFiltering); 274 FRIEND_TEST(ValueMetricProducerTest, TestPulledValueWithUpgrade); 275 FRIEND_TEST(ValueMetricProducerTest, TestPulledValueWithUpgradeWhileConditionFalse); 276 FRIEND_TEST(ValueMetricProducerTest, TestPulledWithAppUpgradeDisabled); 277 FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateAvg); 278 FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateMax); 279 FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateMin); 280 FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateSum); 281 FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithCondition); 282 FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithUpgrade); 283 FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithoutCondition); 284 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullDelayExceeded); 285 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange); 286 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange_EndOfBucket); 287 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullFailBeforeConditionChange); 288 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullTooLate); 289 FRIEND_TEST(ValueMetricProducerTest, TestSkipZeroDiffOutput); 290 FRIEND_TEST(ValueMetricProducerTest, TestSkipZeroDiffOutputMultiValue); 291 FRIEND_TEST(ValueMetricProducerTest, TestTrimUnusedDimensionKey); 292 FRIEND_TEST(ValueMetricProducerTest, TestUseZeroDefaultBase); 293 FRIEND_TEST(ValueMetricProducerTest, TestUseZeroDefaultBaseWithPullFailures); 294 friend class ValueMetricProducerTestHelper; 295 }; 296 297 } // namespace statsd 298 } // namespace os 299 } // namespace android 300