1# Copyright 2019, The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""ATest execution info generator."""
16
17# pylint: disable=line-too-long
18
19from __future__ import print_function
20
21import glob
22import logging
23import json
24import os
25import sys
26
27import atest_utils as au
28import constants
29
30from metrics import metrics_utils
31
32_ARGS_KEY = 'args'
33_STATUS_PASSED_KEY = 'PASSED'
34_STATUS_FAILED_KEY = 'FAILED'
35_STATUS_IGNORED_KEY = 'IGNORED'
36_SUMMARY_KEY = 'summary'
37_TOTAL_SUMMARY_KEY = 'total_summary'
38_TEST_RUNNER_KEY = 'test_runner'
39_TEST_NAME_KEY = 'test_name'
40_TEST_TIME_KEY = 'test_time'
41_TEST_DETAILS_KEY = 'details'
42_TEST_RESULT_NAME = 'test_result'
43_EXIT_CODE_ATTR = 'EXIT_CODE'
44_MAIN_MODULE_KEY = '__main__'
45_UUID_LEN = 30
46_RESULT_LEN = 35
47_COMMAND_LEN = 50
48_LOGCAT_FMT = '{}/log/invocation_*/{}*logcat-on-failure*'
49
50_SUMMARY_MAP_TEMPLATE = {_STATUS_PASSED_KEY : 0,
51                         _STATUS_FAILED_KEY : 0,
52                         _STATUS_IGNORED_KEY : 0,}
53
54PREPARE_END_TIME = None
55
56
57def preparation_time(start_time):
58    """Return the preparation time.
59
60    Args:
61        start_time: The time.
62
63    Returns:
64        The preparation time if PREPARE_END_TIME is set, None otherwise.
65    """
66    return PREPARE_END_TIME - start_time if PREPARE_END_TIME else None
67
68
69def symlink_latest_result(test_result_dir):
70    """Make the symbolic link to latest result.
71
72    Args:
73        test_result_dir: A string of the dir path.
74    """
75    symlink = os.path.join(constants.ATEST_RESULT_ROOT, 'LATEST')
76    if os.path.exists(symlink) or os.path.islink(symlink):
77        os.remove(symlink)
78    os.symlink(test_result_dir, symlink)
79
80
81def print_test_result(root, history_arg):
82    """Make a list of latest n test result.
83
84    Args:
85        root: A string of the test result root path.
86        history_arg: A string of an integer or uuid. If it's an integer string,
87                     the number of lines of test result will be given; else it
88                     will be treated a uuid and print test result accordingly
89                     in detail.
90    """
91    if not history_arg.isdigit():
92        path = os.path.join(constants.ATEST_RESULT_ROOT, history_arg,
93                            'test_result')
94        print_test_result_by_path(path)
95        return
96    target = '%s/20*_*_*' % root
97    paths = glob.glob(target)
98    paths.sort(reverse=True)
99    print('{:-^{uuid_len}} {:-^{result_len}} {:-^{command_len}}'
100          .format('uuid', 'result', 'command',
101                  uuid_len=_UUID_LEN,
102                  result_len=_RESULT_LEN,
103                  command_len=_COMMAND_LEN))
104    for path in paths[0: int(history_arg)+1]:
105        result_path = os.path.join(path, 'test_result')
106        if os.path.isfile(result_path):
107            try:
108                with open(result_path) as json_file:
109                    result = json.load(json_file)
110                    total_summary = result.get(_TOTAL_SUMMARY_KEY, {})
111                    summary_str = ', '.join([k+':'+str(v)
112                                             for k, v in total_summary.items()])
113                    print('{:<{uuid_len}} {:<{result_len}} atest {:<{command_len}}'
114                          .format(os.path.basename(path),
115                                  summary_str,
116                                  result.get(_ARGS_KEY, ''),
117                                  uuid_len=_UUID_LEN,
118                                  result_len=_RESULT_LEN,
119                                  command_len=_COMMAND_LEN))
120            except ValueError:
121                pass
122
123
124def print_test_result_by_path(path):
125    """Print latest test result.
126
127    Args:
128        path: A string of test result path.
129    """
130    if os.path.isfile(path):
131        with open(path) as json_file:
132            result = json.load(json_file)
133            print("\natest {}".format(result.get(_ARGS_KEY, '')))
134            print('\nTotal Summary:\n{}'.format(au.delimiter('-')))
135            total_summary = result.get(_TOTAL_SUMMARY_KEY, {})
136            print(', '.join([(k+':'+str(v))
137                             for k, v in total_summary.items()]))
138            fail_num = total_summary.get(_STATUS_FAILED_KEY)
139            if fail_num > 0:
140                message = '%d test failed' % fail_num
141                print('\n')
142                print(au.colorize(message, constants.RED))
143                print('-' * len(message))
144                test_runner = result.get(_TEST_RUNNER_KEY, {})
145                for runner_name in test_runner.keys():
146                    test_dict = test_runner.get(runner_name, {})
147                    for test_name in test_dict:
148                        test_details = test_dict.get(test_name, {})
149                        for fail in test_details.get(_STATUS_FAILED_KEY):
150                            print(au.colorize('{}'.format(
151                                fail.get(_TEST_NAME_KEY)), constants.RED))
152                            failure_files = glob.glob(_LOGCAT_FMT.format(
153                                os.path.dirname(path), fail.get(_TEST_NAME_KEY)
154                                ))
155                            if failure_files:
156                                print('{} {}'.format(
157                                    au.colorize('LOGCAT-ON-FAILURES:',
158                                                constants.CYAN),
159                                    failure_files[0]))
160                            print('{} {}'.format(
161                                au.colorize('STACKTRACE:\n', constants.CYAN),
162                                fail.get(_TEST_DETAILS_KEY)))
163
164
165def has_non_test_options(args):
166    """
167    check whether non-test option in the args.
168
169    Args:
170        args: An argspace.Namespace class instance holding parsed args.
171
172    Returns:
173        True, if args has at least one non-test option.
174        False, otherwise.
175    """
176    return (args.collect_tests_only
177            or args.dry_run
178            or args.help
179            or args.history
180            or args.info
181            or args.version
182            or args.latest_result)
183
184
185class AtestExecutionInfo:
186    """Class that stores the whole test progress information in JSON format.
187
188    ----
189    For example, running command
190        atest hello_world_test HelloWorldTest
191
192    will result in storing the execution detail in JSON:
193    {
194      "args": "hello_world_test HelloWorldTest",
195      "test_runner": {
196          "AtestTradefedTestRunner": {
197              "hello_world_test": {
198                  "FAILED": [
199                      {"test_time": "(5ms)",
200                       "details": "Hello, Wor...",
201                       "test_name": "HelloWorldTest#PrintHelloWorld"}
202                      ],
203                  "summary": {"FAILED": 1, "PASSED": 0, "IGNORED": 0}
204              },
205              "HelloWorldTests": {
206                  "PASSED": [
207                      {"test_time": "(27ms)",
208                       "details": null,
209                       "test_name": "...HelloWorldTest#testHalloWelt"},
210                      {"test_time": "(1ms)",
211                       "details": null,
212                       "test_name": "....HelloWorldTest#testHelloWorld"}
213                      ],
214                  "summary": {"FAILED": 0, "PASSED": 2, "IGNORED": 0}
215              }
216          }
217      },
218      "total_summary": {"FAILED": 1, "PASSED": 2, "IGNORED": 0}
219    }
220    """
221
222    result_reporters = []
223
224    def __init__(self, args, work_dir, args_ns):
225        """Initialise an AtestExecutionInfo instance.
226
227        Args:
228            args: Command line parameters.
229            work_dir: The directory for saving information.
230            args_ns: An argspace.Namespace class instance holding parsed args.
231
232        Returns:
233               A json format string.
234        """
235        self.args = args
236        self.work_dir = work_dir
237        self.result_file = None
238        self.args_ns = args_ns
239
240    def __enter__(self):
241        """Create and return information file object."""
242        full_file_name = os.path.join(self.work_dir, _TEST_RESULT_NAME)
243        try:
244            self.result_file = open(full_file_name, 'w')
245        except IOError:
246            logging.error('Cannot open file %s', full_file_name)
247        return self.result_file
248
249    def __exit__(self, exit_type, value, traceback):
250        """Write execution information and close information file."""
251        if self.result_file:
252            self.result_file.write(AtestExecutionInfo.
253                                   _generate_execution_detail(self.args))
254            self.result_file.close()
255            if not has_non_test_options(self.args_ns):
256                symlink_latest_result(self.work_dir)
257        main_module = sys.modules.get(_MAIN_MODULE_KEY)
258        main_exit_code = getattr(main_module, _EXIT_CODE_ATTR,
259                                 constants.EXIT_CODE_ERROR)
260        if main_exit_code == constants.EXIT_CODE_SUCCESS:
261            metrics_utils.send_exit_event(main_exit_code)
262        else:
263            metrics_utils.handle_exc_and_send_exit_event(main_exit_code)
264
265    @staticmethod
266    def _generate_execution_detail(args):
267        """Generate execution detail.
268
269        Args:
270            args: Command line parameters that you want to save.
271
272        Returns:
273            A json format string.
274        """
275        info_dict = {_ARGS_KEY: ' '.join(args)}
276        try:
277            AtestExecutionInfo._arrange_test_result(
278                info_dict,
279                AtestExecutionInfo.result_reporters)
280            return json.dumps(info_dict)
281        except ValueError as err:
282            logging.warning('Parsing test result failed due to : %s', err)
283
284    @staticmethod
285    def _arrange_test_result(info_dict, reporters):
286        """Append test result information in given dict.
287
288        Arrange test information to below
289        "test_runner": {
290            "test runner name": {
291                "test name": {
292                    "FAILED": [
293                        {"test time": "",
294                         "details": "",
295                         "test name": ""}
296                    ],
297                "summary": {"FAILED": 0, "PASSED": 0, "IGNORED": 0}
298                },
299            },
300        "total_summary": {"FAILED": 0, "PASSED": 0, "IGNORED": 0}
301
302        Args:
303            info_dict: A dict you want to add result information in.
304            reporters: A list of result_reporter.
305
306        Returns:
307            A dict contains test result information data.
308        """
309        info_dict[_TEST_RUNNER_KEY] = {}
310        for reporter in reporters:
311            for test in reporter.all_test_results:
312                runner = info_dict[_TEST_RUNNER_KEY].setdefault(
313                    test.runner_name, {})
314                group = runner.setdefault(test.group_name, {})
315                result_dict = {_TEST_NAME_KEY : test.test_name,
316                               _TEST_TIME_KEY : test.test_time,
317                               _TEST_DETAILS_KEY : test.details}
318                group.setdefault(test.status, []).append(result_dict)
319
320        total_test_group_summary = _SUMMARY_MAP_TEMPLATE.copy()
321        for runner in info_dict[_TEST_RUNNER_KEY]:
322            for group in info_dict[_TEST_RUNNER_KEY][runner]:
323                group_summary = _SUMMARY_MAP_TEMPLATE.copy()
324                for status in info_dict[_TEST_RUNNER_KEY][runner][group]:
325                    count = len(info_dict[_TEST_RUNNER_KEY][runner][group][status])
326                    if status in _SUMMARY_MAP_TEMPLATE:
327                        group_summary[status] = count
328                        total_test_group_summary[status] += count
329                info_dict[_TEST_RUNNER_KEY][runner][group][_SUMMARY_KEY] = group_summary
330        info_dict[_TOTAL_SUMMARY_KEY] = total_test_group_summary
331        return info_dict
332