1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.compatibility.common.tradefed.result;
18 
19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
20 import com.android.json.stream.JsonWriter;
21 import com.android.tradefed.build.IBuildInfo;
22 import com.android.tradefed.config.Option;
23 import com.android.tradefed.config.OptionCopier;
24 import com.android.tradefed.invoker.IInvocationContext;
25 import com.android.tradefed.log.LogUtil.CLog;
26 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
27 import com.android.tradefed.result.IShardableListener;
28 import com.android.tradefed.result.TestDescription;
29 import com.android.tradefed.util.proto.TfMetricProtoUtil;
30 
31 import java.io.File;
32 import java.io.FileNotFoundException;
33 import java.io.IOException;
34 import java.io.PrintWriter;
35 import java.util.Collection;
36 import java.util.Collections;
37 import java.util.HashMap;
38 import java.util.LinkedList;
39 import java.util.Map;
40 
41 /**
42  * Write test metadata to the result/metadata folder.
43  */
44 public class MetadataReporter implements IShardableListener {
45 
46     @Option(name = "include-failure-time", description = "Include timing about tests that failed.")
47     private boolean mIncludeFailures = false;
48 
49     @Option(name = "min-test-duration", description = "Ignore test durations less than this.",
50             isTimeVal = true)
51     private long mMinTestDuration = 2 * 1000;
52 
53     private static final String METADATA_DIR = "metadata";
54     private CompatibilityBuildHelper mBuildHelper;
55     private File mMetadataDir;
56     private long mStartTime;
57     private String mCurrentModule;
58     private boolean mTestFailed;
59     private Collection<TestMetadata> mTestMetadata = new LinkedList<>();
60 
61     /**
62      * {@inheritDoc}
63      */
64     @Override
clone()65     public IShardableListener clone() {
66         MetadataReporter clone = new MetadataReporter();
67         OptionCopier.copyOptionsNoThrow(this, clone);
68         return clone;
69     }
70 
71     /**
72      * {@inheritDoc}
73      */
74     @Override
invocationStarted(IInvocationContext context)75     public void invocationStarted(IInvocationContext context) {
76         IBuildInfo buildInfo = context.getBuildInfos().get(0);
77         synchronized(this) {
78             if (mBuildHelper == null) {
79                 mBuildHelper = new CompatibilityBuildHelper(buildInfo);
80                 try {
81                     mMetadataDir = new File(mBuildHelper.getResultDir(), METADATA_DIR);
82                 } catch (FileNotFoundException e) {
83                     throw new RuntimeException("Metadata Directory was not created: " +
84                             mMetadataDir.getAbsolutePath());
85                 }
86             }
87         }
88     }
89 
90     /**
91      * {@inheritDoc}
92      */
93     @Override
testRunStarted(String id, int numTests)94     public void testRunStarted(String id, int numTests) {
95         this.mCurrentModule = id;
96     }
97 
98     /**
99      * {@inheritDoc}
100      */
101     @Override
testStarted(TestDescription test)102     public void testStarted(TestDescription test) {
103         mStartTime = System.currentTimeMillis();
104         mTestFailed = false;
105     }
106 
107     /**
108      * {@inheritDoc}
109      */
110     @Override
testFailed(TestDescription test, String trace)111     public void testFailed(TestDescription test, String trace) {
112         mTestFailed = true;
113     }
114 
115     /**
116      * {@inheritDoc}
117      */
118     @Override
testIgnored(TestDescription test)119     public void testIgnored(TestDescription test) {
120         mTestFailed = true;
121     }
122 
123     /**
124      * {@inheritDoc}
125      */
126     @Override
testAssumptionFailure(TestDescription test, String trace)127     public void testAssumptionFailure(TestDescription test, String trace) {
128         mTestFailed = true;
129     }
130 
131     /**
132      * {@inheritDoc}
133      */
134     @Override
testEnded(TestDescription test, HashMap<String, Metric> testMetrics)135     public void testEnded(TestDescription test, HashMap<String, Metric> testMetrics) {
136         long duration = System.currentTimeMillis() - mStartTime;
137         if (mTestFailed && !mIncludeFailures) {
138             return;
139         }
140         if (duration < mMinTestDuration) {
141             return;
142         }
143 
144         TestMetadata metadata = new TestMetadata();
145         metadata.testId = buildTestId(test);
146         metadata.seconds = duration / 1000; // convert to second for reporting
147         mTestMetadata.add(metadata);
148     }
149 
150     /**
151      * {@inheritDoc}
152      */
153     @Override
testRunEnded(long elapsedTime, Map<String, String> metrics)154     public void testRunEnded(long elapsedTime, Map<String, String> metrics) {
155         testRunEnded(elapsedTime, TfMetricProtoUtil.upgradeConvert(metrics));
156     }
157 
158     /**
159      * {@inheritDoc}
160      */
161     @Override
testRunEnded(long elapsedTime, HashMap<String, Metric> metrics)162     public void testRunEnded(long elapsedTime, HashMap<String, Metric> metrics) {
163         if (!mTestMetadata.isEmpty()) {
164             tryWriteToFile(mBuildHelper, mCurrentModule, mMetadataDir, mTestMetadata);
165         }
166         mTestMetadata.clear();
167     }
168 
169     /** Information about a test's execution. */
170     public static class TestMetadata {
171         // The id of the test
172         String testId;
173         // The duration of the test.
174         long seconds;
175     }
176 
buildTestId(TestDescription test)177     private static String buildTestId(TestDescription test) {
178         return String.format("%s.%s", test.getClassName(), test.getTestName());
179     }
180 
tryWriteToFile( CompatibilityBuildHelper compatibilityBuildHelper, String moduleName, File metadataDir, Collection<TestMetadata> metadatas)181     private static void tryWriteToFile(
182             CompatibilityBuildHelper compatibilityBuildHelper,
183             String moduleName,
184             File metadataDir,
185             Collection<TestMetadata> metadatas) {
186 
187         metadataDir.mkdirs();
188 
189         String moduleFileName = moduleName + "." + System.currentTimeMillis() + ".json";
190         File metadataFile = new File(metadataDir, moduleFileName);
191         Map<String, String> buildAttributes =
192                 compatibilityBuildHelper.getBuildInfo().getBuildAttributes();
193         try (JsonWriter writer = new JsonWriter(new PrintWriter(metadataFile))) {
194             writer.beginObject();
195 
196             writer.name("fingerprint");
197             writer.value(buildAttributes.get("cts:build_fingerprint"));
198 
199             writer.name("product");
200             writer.value(buildAttributes.get("cts:build_product"));
201 
202             writer.name("build_id");
203             writer.value(buildAttributes.get("cts:build_id"));
204 
205             writer.name("suite_version");
206             writer.value(compatibilityBuildHelper.getSuiteVersion());
207 
208             writer.name("suite_name");
209             writer.value(compatibilityBuildHelper.getSuiteName());
210 
211             writer.name("suite_build");
212             writer.value(compatibilityBuildHelper.getSuiteBuild());
213 
214             writer.name("module_id");
215             writer.value(moduleName);
216 
217             writer.name("test");
218             writer.beginArray();
219             for (TestMetadata metadata : metadatas) {
220                 writer.beginObject();
221                 writer.name("id");
222                 writer.value(metadata.testId);
223                 writer.name("sec");
224                 writer.value(metadata.seconds);
225                 writer.endObject();
226             }
227             writer.endArray();
228 
229             writer.endObject();
230         } catch (IOException e) {
231             CLog.e("[%s] While saving metadata.", metadataFile.getAbsolutePath());
232             CLog.e(e);
233         }
234     }
235 
getTestMetadata()236     protected Collection<TestMetadata> getTestMetadata() {
237         return Collections.unmodifiableCollection(mTestMetadata);
238     }
239 }
240