1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include "anomaly/AlarmMonitor.h"
20 #include "anomaly/AlarmTracker.h"
21 #include "anomaly/AnomalyTracker.h"
22 #include "condition/ConditionTracker.h"
23 #include "config/ConfigKey.h"
24 #include "external/StatsPullerManager.h"
25 #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"
26 #include "logd/LogEvent.h"
27 #include "matchers/LogMatchingTracker.h"
28 #include "metrics/MetricProducer.h"
29 #include "packages/UidMap.h"
30 
31 #include <unordered_map>
32 
33 namespace android {
34 namespace os {
35 namespace statsd {
36 
37 // A MetricsManager is responsible for managing metrics from one single config source.
38 class MetricsManager : public virtual android::RefBase {
39 public:
40     MetricsManager(const ConfigKey& configKey, const StatsdConfig& config, const int64_t timeBaseNs,
41                    const int64_t currentTimeNs, const sp<UidMap>& uidMap,
42                    const sp<StatsPullerManager>& pullerManager,
43                    const sp<AlarmMonitor>& anomalyAlarmMonitor,
44                    const sp<AlarmMonitor>& periodicAlarmMonitor);
45 
46     virtual ~MetricsManager();
47 
48     // Return whether the configuration is valid.
49     bool isConfigValid() const;
50 
51     bool checkLogCredentials(const LogEvent& event);
52 
53     bool eventSanityCheck(const LogEvent& event);
54 
55     void onLogEvent(const LogEvent& event);
56 
57     void onAnomalyAlarmFired(
58         const int64_t& timestampNs,
59         unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>>& alarmSet);
60 
61     void onPeriodicAlarmFired(
62         const int64_t& timestampNs,
63         unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>>& alarmSet);
64 
65     void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid,
66                           const int64_t version);
67 
68     void notifyAppRemoved(const int64_t& eventTimeNs, const string& apk, const int uid);
69 
70     void onUidMapReceived(const int64_t& eventTimeNs);
71 
shouldWriteToDisk()72     bool shouldWriteToDisk() const {
73         return mNoReportMetricIds.size() != mAllMetricProducers.size();
74     }
75 
shouldPersistLocalHistory()76     bool shouldPersistLocalHistory() const {
77         return mShouldPersistHistory;
78     }
79 
80     void dumpStates(FILE* out, bool verbose);
81 
isInTtl(const int64_t timestampNs)82     inline bool isInTtl(const int64_t timestampNs) const {
83         return mTtlNs <= 0 || timestampNs < mTtlEndNs;
84     };
85 
hashStringInReport()86     inline bool hashStringInReport() const {
87         return mHashStringsInReport;
88     };
89 
versionStringsInReport()90     inline bool versionStringsInReport() const {
91         return mVersionStringsInReport;
92     };
93 
installerInReport()94     inline bool installerInReport() const {
95         return mInstallerInReport;
96     };
97 
refreshTtl(const int64_t currentTimestampNs)98     void refreshTtl(const int64_t currentTimestampNs) {
99         if (mTtlNs > 0) {
100             mTtlEndNs = currentTimestampNs + mTtlNs;
101         }
102     };
103 
104     // Returns the elapsed realtime when this metric manager last reported metrics. If this config
105     // has not yet dumped any reports, this is the time the metricsmanager was initialized.
getLastReportTimeNs()106     inline int64_t getLastReportTimeNs() const {
107         return mLastReportTimeNs;
108     };
109 
getLastReportWallClockNs()110     inline int64_t getLastReportWallClockNs() const {
111         return mLastReportWallClockNs;
112     };
113 
getNumMetrics()114     inline size_t getNumMetrics() const {
115         return mAllMetricProducers.size();
116     }
117 
118     virtual void dropData(const int64_t dropTimeNs);
119 
120     virtual void onDumpReport(const int64_t dumpTimeNs,
121                               const bool include_current_partial_bucket,
122                               const bool erase_data,
123                               const DumpLatency dumpLatency,
124                               std::set<string> *str_set,
125                               android::util::ProtoOutputStream* protoOutput);
126 
127     // Computes the total byte size of all metrics managed by a single config source.
128     // Does not change the state.
129     virtual size_t byteSize();
130 
131     // Returns whether or not this config is active.
132     // The config is active if any metric in the config is active.
isActive()133     inline bool isActive() const {
134         return mIsActive;
135     }
136 
137     void loadActiveConfig(const ActiveConfig& config, int64_t currentTimeNs);
138 
139     void writeActiveConfigToProtoOutputStream(
140             int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto);
141 
142 private:
143     // For test only.
getTtlEndNs()144     inline int64_t getTtlEndNs() const { return mTtlEndNs; }
145 
146     const ConfigKey mConfigKey;
147 
148     sp<UidMap> mUidMap;
149 
150     bool mConfigValid = false;
151 
152     bool mHashStringsInReport = false;
153     bool mVersionStringsInReport = false;
154     bool mInstallerInReport = false;
155 
156     const int64_t mTtlNs;
157     int64_t mTtlEndNs;
158 
159     int64_t mLastReportTimeNs;
160     int64_t mLastReportWallClockNs;
161 
162     // The uid log sources from StatsdConfig.
163     std::vector<int32_t> mAllowedUid;
164 
165     // The pkg log sources from StatsdConfig.
166     std::vector<std::string> mAllowedPkg;
167 
168     // The combined uid sources (after translating pkg name to uid).
169     // Logs from uids that are not in the list will be ignored to avoid spamming.
170     std::set<int32_t> mAllowedLogSources;
171 
172     // Contains the annotations passed in with StatsdConfig.
173     std::list<std::pair<const int64_t, const int32_t>> mAnnotations;
174 
175     const bool mShouldPersistHistory;
176 
177     // To guard access to mAllowedLogSources
178     mutable std::mutex mAllowedLogSourcesMutex;
179 
180     // All event tags that are interesting to my metrics.
181     std::set<int> mTagIds;
182 
183     // We only store the sp of LogMatchingTracker, MetricProducer, and ConditionTracker in
184     // MetricsManager. There are relationships between them, and the relationships are denoted by
185     // index instead of pointers. The reasons for this are: (1) the relationship between them are
186     // complicated, so storing index instead of pointers reduces the risk that A holds B's sp, and B
187     // holds A's sp. (2) When we evaluate matcher results, or condition results, we can quickly get
188     // the related results from a cache using the index.
189 
190     // Hold all the atom matchers from the config.
191     std::vector<sp<LogMatchingTracker>> mAllAtomMatchers;
192 
193     // Hold all the conditions from the config.
194     std::vector<sp<ConditionTracker>> mAllConditionTrackers;
195 
196     // Hold all metrics from the config.
197     std::vector<sp<MetricProducer>> mAllMetricProducers;
198 
199     // Hold all alert trackers.
200     std::vector<sp<AnomalyTracker>> mAllAnomalyTrackers;
201 
202     // Hold all periodic alarm trackers.
203     std::vector<sp<AlarmTracker>> mAllPeriodicAlarmTrackers;
204 
205     // To make the log processing more efficient, we want to do as much filtering as possible
206     // before we go into individual trackers and conditions to match.
207 
208     // 1st filter: check if the event tag id is in mTagIds.
209     // 2nd filter: if it is, we parse the event because there is at least one member is interested.
210     //             then pass to all LogMatchingTrackers (itself also filter events by ids).
211     // 3nd filter: for LogMatchingTrackers that matched this event, we pass this event to the
212     //             ConditionTrackers and MetricProducers that use this matcher.
213     // 4th filter: for ConditionTrackers that changed value due to this event, we pass
214     //             new conditions to  metrics that use this condition.
215 
216     // The following map is initialized from the statsd_config.
217 
218     // Maps from the index of the LogMatchingTracker to index of MetricProducer.
219     std::unordered_map<int, std::vector<int>> mTrackerToMetricMap;
220 
221     // Maps from LogMatchingTracker to ConditionTracker
222     std::unordered_map<int, std::vector<int>> mTrackerToConditionMap;
223 
224     // Maps from ConditionTracker to MetricProducer
225     std::unordered_map<int, std::vector<int>> mConditionToMetricMap;
226 
227     // Maps from life span triggering event to MetricProducers.
228     std::unordered_map<int, std::vector<int>> mActivationAtomTrackerToMetricMap;
229 
230     // Maps deactivation triggering event to MetricProducers.
231     std::unordered_map<int, std::vector<int>> mDeactivationAtomTrackerToMetricMap;
232 
233     std::vector<int> mMetricIndexesWithActivation;
234 
235     void initLogSourceWhiteList();
236 
237     // The metrics that don't need to be uploaded or even reported.
238     std::set<int64_t> mNoReportMetricIds;
239 
240    // The config is active if any metric in the config is active.
241     bool mIsActive;
242 
243     // The config is always active if any metric in the config does not have an activation signal.
244     bool mIsAlwaysActive;
245 
246     FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensions);
247     FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks);
248     FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByFirstUid);
249     FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByChain);
250     FRIEND_TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent);
251     FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents);
252     FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm);
253     FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEventsWithActivation);
254     FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEventsNoCondition);
255     FRIEND_TEST(GaugeMetricE2eTest, TestConditionChangeToTrueSamplePulledEvents);
256     FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents);
257     FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm);
258     FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_WithActivation);
259     FRIEND_TEST(DimensionInConditionE2eTest, TestCreateCountMetric_NoLink_OR_CombinationCondition);
260     FRIEND_TEST(DimensionInConditionE2eTest, TestCreateCountMetric_Link_OR_CombinationCondition);
261     FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_OR_CombinationCondition);
262     FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_Link_OR_CombinationCondition);
263 
264     FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_SimpleCondition);
265     FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_Link_SimpleCondition);
266     FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_PartialLink_SimpleCondition);
267     FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_AND_CombinationCondition);
268     FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_Link_AND_CombinationCondition);
269     FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_PartialLink_AND_CombinationCondition);
270 
271     FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket);
272     FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets);
273     FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket);
274     FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets);
275     FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period);
276 
277     FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms);
278     FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric);
279     FRIEND_TEST(MetricActivationE2eTest, TestCountMetric);
280     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithOneDeactivation);
281     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoDeactivations);
282     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithSameDeactivation);
283     FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoMetricsTwoDeactivations);
284 
285     FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead);
286     FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBoot);
287     FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBootMultipleActivations);
288     FRIEND_TEST(StatsLogProcessorTest,
289             TestActivationOnBootMultipleActivationsDifferentActivationTypes);
290     FRIEND_TEST(StatsLogProcessorTest, TestActivationsPersistAcrossSystemServerRestart);
291 
292     FRIEND_TEST(DurationMetricE2eTest, TestOneBucket);
293     FRIEND_TEST(DurationMetricE2eTest, TestTwoBuckets);
294     FRIEND_TEST(DurationMetricE2eTest, TestWithActivation);
295     FRIEND_TEST(DurationMetricE2eTest, TestWithCondition);
296     FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition);
297     FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition);
298 };
299 
300 }  // namespace statsd
301 }  // namespace os
302 }  // namespace android
303