1#
2# Copyright (C) 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16
17import logging
18import os
19import re
20import tempfile
21import xml.etree.ElementTree
22
23from vts.runners.host import asserts
24from vts.runners.host import const
25from vts.runners.host import keys
26from vts.runners.host import test_runner
27
28from vts.testcases.template.binary_test import binary_test
29from vts.testcases.template.binary_test import binary_test_case
30from vts.testcases.template.gtest_binary_test import gtest_test_case
31
32_GTEST_RESULT_ATTRIBUTE_ALLOW_LIST = ('properties',)
33
34
35class GtestBinaryTest(binary_test.BinaryTest):
36    '''Base class to run gtests binary on target.
37
38    Attributes:
39        DEVICE_TEST_DIR: string, temp location for storing binary
40        TAG_PATH_SEPARATOR: string, separator used to separate tag and path
41        shell: ShellMirrorObject, shell mirror
42        tags: all the tags that appeared in binary list
43        testcases: list of GtestTestCase objects, list of test cases to run
44        _dut: AndroidDevice, the device under test as config
45        _gtest_results: list of GtestResult objects, used during batch mode
46                        for result storage and parsing
47    '''
48
49    # @Override
50    def setUpClass(self):
51        '''Prepare class, push binaries, set permission, create test cases.'''
52        self.collect_tests_only = self.getUserParam(
53            keys.ConfigKeys.IKEY_COLLECT_TESTS_ONLY, default_value=False)
54        self.batch_mode = self.getUserParam(
55            keys.ConfigKeys.IKEY_GTEST_BATCH_MODE, default_value=False)
56
57        if self.batch_mode:
58            if self.collect_tests_only:
59                self.batch_mode = False
60                logging.debug("Disable batch mode when collecting tests.")
61            else:
62                self._gtest_results = []
63
64        super(GtestBinaryTest, self).setUpClass()
65
66    # @Override
67    def CreateTestCase(self, path, tag=''):
68        '''Create a list of GtestTestCase objects from a binary path.
69
70        Args:
71            path: string, absolute path of a gtest binary on device
72            tag: string, a tag that will be appended to the end of test name
73
74        Returns:
75            A list of GtestTestCase objects on success; an empty list otherwise.
76            In non-batch mode, each object respresents a test case in the
77            gtest binary located at the provided path. Usually there are more
78            than one object returned.
79            In batch mode, each object represents a gtest binary located at
80            the provided path; the returned list will always be a one object
81            list in batch mode. Test case names are stored in full_name
82            property in the object, delimited by ':' according to gtest
83            documentation, after being filtered and processed according to
84            host configuration.
85        '''
86        working_directory = self.working_directory[
87            tag] if tag in self.working_directory else None
88        envp = self.envp[tag] if tag in self.envp else ''
89        args = self.args[tag] if tag in self.args else ''
90        ld_library_path = self.ld_library_path[
91            tag] if tag in self.ld_library_path else None
92        profiling_library_path = self.profiling_library_path[
93            tag] if tag in self.profiling_library_path else None
94
95        gtest_list_args = args + " --gtest_list_tests"
96        list_test_case = binary_test_case.BinaryTestCase(
97            'gtest_list_tests',
98            path,
99            path,
100            tag,
101            self.PutTag,
102            working_directory,
103            ld_library_path,
104            profiling_library_path,
105            envp=envp,
106            args=gtest_list_args)
107        cmd = ['chmod 755 %s' % path, list_test_case.GetRunCommand()]
108        cmd_results = self.shell.Execute(cmd)
109        test_cases = []
110        asserts.assertFalse(any(cmd_results[const.EXIT_CODE]),
111                'Failed to list test cases from %s. Command: %s, Result: %s.' %
112                (path, cmd, cmd_results))
113
114        test_suite = ''
115        for line in cmd_results[const.STDOUT][1].split('\n'):
116            line = str(line)
117            if not len(line.strip()):
118                continue
119            elif line.startswith(' '):  # Test case name
120                test_name = line.split('#')[0].strip()
121                # Skip any test that doesn't instantiate the parameterized gtest
122                if re.match('UninstantiatedParamaterizedTestSuite<(.*)>', test_name):
123                    continue
124                test_case = gtest_test_case.GtestTestCase(
125                    test_suite, test_name, path, tag, self.PutTag,
126                    working_directory, ld_library_path, profiling_library_path,
127                    envp=envp, args=args)
128                logging.debug('Gtest test case: %s' % test_case)
129                test_cases.append(test_case)
130            else:  # Test suite name
131                test_suite = line.strip()
132                if test_suite.endswith('.'):
133                    test_suite = test_suite[:-1]
134
135        if not self.batch_mode:
136            return test_cases
137
138        # Gtest batch mode
139        test_names = map(lambda test: test.full_name, test_cases)
140
141        gtest_batch = gtest_test_case.GtestTestCase(
142            path, '', path, tag, self.PutTag, working_directory,
143            ld_library_path, profiling_library_path, envp=envp)
144        gtest_batch.full_name = ':'.join(test_names)
145        return [gtest_batch]
146
147    # @Override
148    def VerifyTestResult(self, test_case, command_results):
149        '''Parse Gtest xml result output.
150
151        Sample
152        <testsuites tests="1" failures="1" disabled="0" errors="0"
153         timestamp="2017-05-24T18:32:10" time="0.012" name="AllTests">
154          <testsuite name="ConsumerIrHidlTest"
155           tests="1" failures="1" disabled="0" errors="0" time="0.01">
156            <testcase name="TransmitTest" status="run" time="0.01"
157             classname="ConsumerIrHidlTest">
158              <failure message="hardware/interfaces..." type="">
159                <![CDATA[hardware/interfaces...]]>
160              </failure>
161            </testcase>
162          </testsuite>
163        </testsuites>
164
165        Args:
166            test_case: GtestTestCase object, the test being run. This param
167                       is not currently used in this method.
168            command_results: dict of lists, shell command result
169        '''
170        asserts.assertTrue(command_results, 'Empty command response.')
171        asserts.assertEqual(
172            len(command_results), 3, 'Abnormal command response.')
173        for item in command_results.values():
174            asserts.assertEqual(
175                len(item), 2,
176                'Abnormal command result length: %s' % command_results)
177
178        for stderr in command_results[const.STDERR]:
179            if stderr and stderr.strip():
180                for line in stderr.split('\n'):
181                    logging.error(line)
182
183        xml_str = command_results[const.STDOUT][1]
184
185        if self.batch_mode:
186            self._ParseBatchResults(test_case, xml_str)
187            return
188
189        asserts.assertFalse(
190            command_results[const.EXIT_CODE][1],
191            'Failed to show Gtest XML output: %s' % command_results)
192
193        root = self._ParseResultXmlString(xml_str)
194        asserts.assertEqual(root.get('tests'), '1', 'No tests available')
195        success = True
196        if root.get('errors') != '0' or root.get('failures') != '0':
197            messages = [x.get('message') for x in root.findall('.//failure')]
198            success = False
199
200        for stdout in command_results[const.STDOUT]:
201            if stdout and stdout.strip():
202                for line in stdout.split('\n'):
203                    if success:
204                        logging.debug(line)
205                    else:
206                        logging.error(line)
207
208        if not success:
209            asserts.fail('\n'.join([x for x in messages if x]))
210
211        asserts.skipIf(root.get('disabled') == '1', 'Gtest test case disabled')
212
213    def _ParseResultXmlString(self, xml_str):
214        """Parses the xml result string into elements.
215
216        Args:
217            xml_str: string, result xml text content.
218
219        Returns:
220            xml.etree.ElementTree, parsed xml content.
221
222        Raises:
223            assertion failure if xml format is not expected.
224        """
225        asserts.assertTrue(xml_str is not None, 'Test command result not received.')
226        xml_str = xml_str.strip()
227        asserts.assertTrue(xml_str, 'Test command result is empty.')
228
229        try:
230            return xml.etree.ElementTree.fromstring(xml_str)
231        except:
232            asserts.fail('Result xml content is corrupted.')
233
234    def _ParseBatchResults(self, test_case_original, xml_str):
235        '''Parse batch mode gtest results
236
237        Args:
238            test_case_original: GtestTestCase object, original batch test case object
239            xml_str: string, result xml output content
240        '''
241        root = self._ParseResultXmlString(xml_str)
242
243        for test_suite in root:
244            logging.debug('Test tag: %s, attribute: %s',
245                          test_suite.tag,
246                          test_suite.attrib)
247            for test_case in test_suite:
248                result = gtest_test_case.GtestTestCase(
249                    test_suite.get('name'),
250                    test_case.get('name'), '', test_case_original.tag,
251                    self.PutTag, name_appendix=test_case_original.name_appendix)
252
253                failure_message = None
254                for sub in test_case:
255                    if sub.tag == 'failure':
256                        failure_message = sub.get('message')
257
258                test_case_filtered = filter(
259                    lambda sub: sub.tag not in _GTEST_RESULT_ATTRIBUTE_ALLOW_LIST, test_case)
260                if len(test_case_filtered) and not failure_message:
261                    failure_message = 'Error: %s\n' % test_case.attrib
262                    for sub in test_case_filtered:
263                        failure_message += '%s: %s\n' % (sub.tag, sub.attrib)
264
265                result.failure_message = failure_message
266
267                self._gtest_results.append(result)
268
269    def _VerifyBatchResult(self, gtest_result):
270        '''Check a gtest test case result in batch mode
271
272        Args:
273            gtest_result: GtestTestCase object, representing gtest result
274        '''
275        asserts.assertFalse(gtest_result.failure_message,
276                            gtest_result.failure_message)
277
278    # @Override
279    def generateAllTests(self):
280        '''Runs all binary tests.
281
282        If the test cases should run in batch mode, this method executes the
283        gtest commands without adding test records, and then parses the XML
284        reports to records.
285        If the test cases should run in batch mode but be skipped (e.g., HAL is
286        not implemented), this method applies the filters in base_test, skips
287        the batch test cases, and adds one record for each of them.
288        '''
289        if self.batch_mode and not self.isSkipAllTests():
290            # TODO(b/126412742): Convert filters to --gtest_filter.
291            for test_case in self.testcases:
292                logging.info('Running %s test cases in batch.',
293                             len(test_case.full_name.split(':')))
294                gtest_filter_flag=('--gtest_filter={test}').format(test = test_case)
295                dst = '/data/local/tmp/filter_file'
296                temp = tempfile.NamedTemporaryFile()
297                try:
298                    temp.write(gtest_filter_flag)
299                    self._dut.adb.push('{src} {dst}'.format(src=temp.name, dst=dst))
300                finally:
301                    temp.close()
302                test_case.filter_file = dst
303                self.RunTestCase(test_case)
304
305                self.shell.Execute('rm %s' % dst)
306                self.runGeneratedTests(
307                    test_func=self._VerifyBatchResult,
308                    settings=self._gtest_results,
309                    name_func=str)
310
311                self._gtest_results = []
312            return
313
314        self.runGeneratedTests(
315            test_func=self.RunTestCase, settings=self.testcases, name_func=str)
316
317
318if __name__ == "__main__":
319    test_runner.main()
320