1#!/usr/bin/env python3
2# Copyright 2019, The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Atest tool functions."""
17
18# pylint: disable=line-too-long
19
20from __future__ import print_function
21
22import json
23import logging
24import os
25import pickle
26import shutil
27import subprocess
28import sys
29
30import atest_utils as au
31import constants
32import module_info
33
34from metrics import metrics_utils
35
36MAC_UPDB_SRC = os.path.join(os.path.dirname(__file__), 'updatedb_darwin.sh')
37MAC_UPDB_DST = os.path.join(os.getenv(constants.ANDROID_HOST_OUT, ''), 'bin')
38UPDATEDB = 'updatedb'
39LOCATE = 'locate'
40SEARCH_TOP = os.getenv(constants.ANDROID_BUILD_TOP, '')
41MACOSX = 'Darwin'
42OSNAME = os.uname()[0]
43# When adding new index, remember to append constants to below tuple.
44INDEXES = (constants.CC_CLASS_INDEX,
45           constants.CLASS_INDEX,
46           constants.LOCATE_CACHE,
47           constants.MODULE_INDEX,
48           constants.PACKAGE_INDEX,
49           constants.QCLASS_INDEX)
50
51# The list was generated by command:
52# find `gettop` -type d -wholename `gettop`/out -prune  -o -type d -name '.*'
53# -print | awk -F/ '{{print $NF}}'| sort -u
54PRUNENAMES = ['.abc', '.appveyor', '.azure-pipelines',
55              '.bazelci', '.buildscript',
56              '.cache', '.ci', '.circleci', '.conan', '.config',
57              '.externalToolBuilders',
58              '.git', '.github', '.gitlab-ci', '.google', '.gradle',
59              '.idea', '.intermediates',
60              '.jenkins',
61              '.kokoro',
62              '.libs_cffi_backend',
63              '.mvn',
64              '.prebuilt_info', '.private', '__pycache__',
65              '.repo',
66              '.semaphore', '.settings', '.static', '.svn',
67              '.test', '.travis', '.travis_scripts', '.tx',
68              '.vscode']
69
70def _mkdir_when_inexists(dirname):
71    if not os.path.isdir(dirname):
72        os.makedirs(dirname)
73
74def _install_updatedb():
75    """Install a customized updatedb for MacOS and ensure it is executable."""
76    _mkdir_when_inexists(MAC_UPDB_DST)
77    _mkdir_when_inexists(constants.INDEX_DIR)
78    if OSNAME == MACOSX:
79        shutil.copy2(MAC_UPDB_SRC, os.path.join(MAC_UPDB_DST, UPDATEDB))
80        os.chmod(os.path.join(MAC_UPDB_DST, UPDATEDB), 0o0755)
81
82def _delete_indexes():
83    """Delete all available index files."""
84    for index in INDEXES:
85        if os.path.isfile(index):
86            os.remove(index)
87
88def get_report_file(results_dir, acloud_args):
89    """Get the acloud report file path.
90
91    This method can parse either string:
92        --acloud-create '--report-file=/tmp/acloud.json'
93        --acloud-create '--report-file /tmp/acloud.json'
94    and return '/tmp/acloud.json' as the report file. Otherwise returning the
95    default path(/tmp/atest_result/<hashed_dir>/acloud_status.json).
96
97    Args:
98        results_dir: string of directory to store atest results.
99        acloud_args: string of acloud create.
100
101    Returns:
102        A string path of acloud report file.
103    """
104    match = constants.ACLOUD_REPORT_FILE_RE.match(acloud_args)
105    if match:
106        return match.group('report_file')
107    return os.path.join(results_dir, 'acloud_status.json')
108
109def has_command(cmd):
110    """Detect if the command is available in PATH.
111
112    Args:
113        cmd: A string of the tested command.
114
115    Returns:
116        True if found, False otherwise.
117    """
118    return bool(shutil.which(cmd))
119
120def run_updatedb(search_root=SEARCH_TOP, output_cache=constants.LOCATE_CACHE,
121                 **kwargs):
122    """Run updatedb and generate cache in $ANDROID_HOST_OUT/indexes/mlocate.db
123
124    Args:
125        search_root: The path of the search root(-U).
126        output_cache: The filename of the updatedb cache(-o).
127        kwargs: (optional)
128            prunepaths: A list of paths unwanted to be searched(-e).
129            prunenames: A list of dirname that won't be cached(-n).
130    """
131    prunenames = kwargs.pop('prunenames', ' '.join(PRUNENAMES))
132    prunepaths = kwargs.pop('prunepaths', os.path.join(search_root, 'out'))
133    if kwargs:
134        raise TypeError('Unexpected **kwargs: %r' % kwargs)
135    updatedb_cmd = [UPDATEDB, '-l0']
136    updatedb_cmd.append('-U%s' % search_root)
137    updatedb_cmd.append('-e%s' % prunepaths)
138    updatedb_cmd.append('-n%s' % prunenames)
139    updatedb_cmd.append('-o%s' % output_cache)
140    try:
141        _install_updatedb()
142    except IOError as e:
143        logging.error('Error installing updatedb: %s', e)
144
145    if not has_command(UPDATEDB):
146        return
147    logging.debug('Running updatedb... ')
148    try:
149        full_env_vars = os.environ.copy()
150        logging.debug('Executing: %s', updatedb_cmd)
151        subprocess.check_call(updatedb_cmd, env=full_env_vars)
152    except (KeyboardInterrupt, SystemExit):
153        logging.error('Process interrupted or failure.')
154
155def _dump_index(dump_file, output, output_re, key, value):
156    """Dump indexed data with pickle.
157
158    Args:
159        dump_file: A string of absolute path of the index file.
160        output: A string generated by locate and grep.
161        output_re: An regex which is used for grouping patterns.
162        key: A string for dictionary key, e.g. classname, package,
163             cc_class, etc.
164        value: A set of path.
165
166    The data structure will be like:
167    {
168      'Foo': {'/path/to/Foo.java', '/path2/to/Foo.kt'},
169      'Boo': {'/path3/to/Boo.java'}
170    }
171    """
172    _dict = {}
173    with open(dump_file, 'wb') as cache_file:
174        if isinstance(output, bytes):
175            output = output.decode()
176        for entry in output.splitlines():
177            match = output_re.match(entry)
178            if match:
179                _dict.setdefault(match.group(key), set()).add(
180                    match.group(value))
181        try:
182            pickle.dump(_dict, cache_file, protocol=2)
183            logging.debug('Done')
184        except IOError:
185            os.remove(dump_file)
186            logging.error('Failed in dumping %s', dump_file)
187
188def _get_cc_result(locatedb=None):
189    """Search all testable cc/cpp and grep TEST(), TEST_F() or TEST_P().
190
191    Returns:
192        A string object generated by subprocess.
193    """
194    if not locatedb:
195        locatedb = constants.LOCATE_CACHE
196    cc_grep_re = r'^\s*TEST(_P|_F)?\s*\(\w+,'
197    if OSNAME == MACOSX:
198        find_cmd = (r"locate -d {0} '*.cpp' '*.cc' | grep -i test "
199                    "| xargs egrep -sH '{1}' || true")
200    else:
201        find_cmd = (r"locate -d {0} / | egrep -i '/*.test.*\.(cc|cpp)$' "
202                    "| xargs egrep -sH '{1}' || true")
203    find_cc_cmd = find_cmd.format(locatedb, cc_grep_re)
204    logging.debug('Probing CC classes:\n %s', find_cc_cmd)
205    return subprocess.check_output(find_cc_cmd, shell=True)
206
207def _get_java_result(locatedb=None):
208    """Search all testable java/kt and grep package.
209
210    Returns:
211        A string object generated by subprocess.
212    """
213    if not locatedb:
214        locatedb = constants.LOCATE_CACHE
215    package_grep_re = r'^\s*package\s+[a-z][[:alnum:]]+[^{]'
216    if OSNAME == MACOSX:
217        find_cmd = r"locate -d%s '*.java' '*.kt'|grep -i test" % locatedb
218    else:
219        find_cmd = r"locate -d%s / | egrep -i '/*.test.*\.(java|kt)$'" % locatedb
220    find_java_cmd = find_cmd + '| xargs egrep -sH \'%s\' || true' % package_grep_re
221    logging.debug('Probing Java classes:\n %s', find_java_cmd)
222    return subprocess.check_output(find_java_cmd, shell=True)
223
224def _index_testable_modules(index):
225    """Dump testable modules read by tab completion.
226
227    Args:
228        index: A string path of the index file.
229    """
230    logging.debug('indexing testable modules.')
231    testable_modules = module_info.ModuleInfo().get_testable_modules()
232    with open(index, 'wb') as cache:
233        try:
234            pickle.dump(testable_modules, cache, protocol=2)
235            logging.debug('Done')
236        except IOError:
237            os.remove(cache)
238            logging.error('Failed in dumping %s', cache)
239
240def _index_cc_classes(output, index):
241    """Index CC classes.
242
243    The data structure is like:
244    {
245      'FooTestCase': {'/path1/to/the/FooTestCase.cpp',
246                      '/path2/to/the/FooTestCase.cc'}
247    }
248
249    Args:
250        output: A string object generated by _get_cc_result().
251        index: A string path of the index file.
252    """
253    logging.debug('indexing CC classes.')
254    _dump_index(dump_file=index, output=output,
255                output_re=constants.CC_OUTPUT_RE,
256                key='test_name', value='file_path')
257
258def _index_java_classes(output, index):
259    """Index Java classes.
260    The data structure is like:
261    {
262        'FooTestCase': {'/path1/to/the/FooTestCase.java',
263                        '/path2/to/the/FooTestCase.kt'}
264    }
265
266    Args:
267        output: A string object generated by _get_java_result().
268        index: A string path of the index file.
269    """
270    logging.debug('indexing Java classes.')
271    _dump_index(dump_file=index, output=output,
272                output_re=constants.CLASS_OUTPUT_RE,
273                key='class', value='java_path')
274
275def _index_packages(output, index):
276    """Index Java packages.
277    The data structure is like:
278    {
279        'a.b.c.d': {'/path1/to/a/b/c/d/',
280                    '/path2/to/a/b/c/d/'
281    }
282
283    Args:
284        output: A string object generated by _get_java_result().
285        index: A string path of the index file.
286    """
287    logging.debug('indexing packages.')
288    _dump_index(dump_file=index,
289                output=output, output_re=constants.PACKAGE_OUTPUT_RE,
290                key='package', value='java_dir')
291
292def _index_qualified_classes(output, index):
293    """Index Fully Qualified Java Classes(FQCN).
294    The data structure is like:
295    {
296        'a.b.c.d.FooTestCase': {'/path1/to/a/b/c/d/FooTestCase.java',
297                                '/path2/to/a/b/c/d/FooTestCase.kt'}
298    }
299
300    Args:
301        output: A string object generated by _get_java_result().
302        index: A string path of the index file.
303    """
304    logging.debug('indexing qualified classes.')
305    _dict = {}
306    with open(index, 'wb') as cache_file:
307        if isinstance(output, bytes):
308            output = output.decode()
309        for entry in output.split('\n'):
310            match = constants.QCLASS_OUTPUT_RE.match(entry)
311            if match:
312                fqcn = match.group('package') + '.' + match.group('class')
313                _dict.setdefault(fqcn, set()).add(match.group('java_path'))
314        try:
315            pickle.dump(_dict, cache_file, protocol=2)
316            logging.debug('Done')
317        except (KeyboardInterrupt, SystemExit):
318            logging.error('Process interrupted or failure.')
319            os.remove(index)
320        except IOError:
321            logging.error('Failed in dumping %s', index)
322
323def index_targets(output_cache=constants.LOCATE_CACHE, **kwargs):
324    """The entrypoint of indexing targets.
325
326    Utilise mlocate database to index reference types of CLASS, CC_CLASS,
327    PACKAGE and QUALIFIED_CLASS. Testable module for tab completion is also
328    generated in this method.
329
330    Args:
331        output_cache: A file path of the updatedb cache
332                      (e.g. /path/to/mlocate.db).
333        kwargs: (optional)
334            class_index: A path string of the Java class index.
335            qclass_index: A path string of the qualified class index.
336            package_index: A path string of the package index.
337            cc_class_index: A path string of the CC class index.
338            module_index: A path string of the testable module index.
339            integration_index: A path string of the integration index.
340    """
341    class_index = kwargs.pop('class_index', constants.CLASS_INDEX)
342    qclass_index = kwargs.pop('qclass_index', constants.QCLASS_INDEX)
343    package_index = kwargs.pop('package_index', constants.PACKAGE_INDEX)
344    cc_class_index = kwargs.pop('cc_class_index', constants.CC_CLASS_INDEX)
345    module_index = kwargs.pop('module_index', constants.MODULE_INDEX)
346    # Uncomment below if we decide to support INTEGRATION.
347    #integration_index = kwargs.pop('integration_index', constants.INT_INDEX)
348    if kwargs:
349        raise TypeError('Unexpected **kwargs: %r' % kwargs)
350
351    try:
352        # Step 0: generate mlocate database prior to indexing targets.
353        run_updatedb(SEARCH_TOP, constants.LOCATE_CACHE)
354        if not has_command(LOCATE):
355            return
356        # Step 1: generate output string for indexing targets.
357        logging.debug('Indexing targets... ')
358        cc_result = _get_cc_result(output_cache)
359        java_result = _get_java_result(output_cache)
360        # Step 2: index Java and CC classes.
361        _index_cc_classes(cc_result, cc_class_index)
362        _index_java_classes(java_result, class_index)
363        _index_qualified_classes(java_result, qclass_index)
364        _index_packages(java_result, package_index)
365        # Step 3: index testable mods and TEST_MAPPING files.
366        _index_testable_modules(module_index)
367
368    # Delete indexes when mlocate.db is locked() or other CalledProcessError.
369    # (b/141588997)
370    except subprocess.CalledProcessError as err:
371        logging.error('Executing %s error.', UPDATEDB)
372        metrics_utils.handle_exc_and_send_exit_event(
373            constants.MLOCATEDB_LOCKED)
374        if err.output:
375            logging.error(err.output)
376        _delete_indexes()
377
378def acloud_create(report_file, args="", no_metrics_notice=True):
379    """Method which runs acloud create with specified args in background.
380
381    Args:
382        report_file: A path string of acloud report file.
383        args: A string of arguments.
384        no_metrics_notice: Boolean whether sending data to metrics or not.
385    """
386    notice = constants.NO_METRICS_ARG if no_metrics_notice else ""
387    match = constants.ACLOUD_REPORT_FILE_RE.match(args)
388    report_file_arg = '--report-file={}'.format(report_file) if not match else ""
389    # (b/161759557) Assume yes for acloud create to streamline atest flow.
390    acloud_cmd = ('acloud create -y {ACLOUD_ARGS} '
391                  '{REPORT_FILE_ARG} '
392                  '{METRICS_NOTICE} '
393                  ).format(ACLOUD_ARGS=args,
394                           REPORT_FILE_ARG=report_file_arg,
395                           METRICS_NOTICE=notice)
396    au.colorful_print("\nCreating AVD via acloud...", constants.CYAN)
397    logging.debug('Executing: %s', acloud_cmd)
398    proc = subprocess.Popen(acloud_cmd, shell=True)
399    proc.communicate()
400
401def probe_acloud_status(report_file):
402    """Method which probes the 'acloud create' result status.
403
404    If the report file exists and the status is 'SUCCESS', then the creation is
405    successful.
406
407    Args:
408        report_file: A path string of acloud report file.
409
410    Returns:
411        0: success.
412        8: acloud creation failure.
413        9: invalid acloud create arguments.
414    """
415    # 1. Created but the status is not 'SUCCESS'
416    if os.path.exists(report_file):
417        try:
418            with open(report_file, 'r') as rfile:
419                result = json.load(rfile)
420        except json.JSONDecodeError:
421            logging.error('Failed loading %s', report_file)
422            return constants.EXIT_CODE_AVD_CREATE_FAILURE
423
424        if result.get('status') == 'SUCCESS':
425            logging.info('acloud create successfully!')
426            # Always fetch the adb of the first created AVD.
427            adb_port = result.get('data').get('devices')[0].get('adb_port')
428            os.environ[constants.ANDROID_SERIAL] = '127.0.0.1:{}'.format(adb_port)
429            return constants.EXIT_CODE_SUCCESS
430        au.colorful_print(
431            'acloud create failed. Please check\n{}\nfor detail'.format(
432                report_file), constants.RED)
433        return constants.EXIT_CODE_AVD_CREATE_FAILURE
434
435    # 2. Failed to create because of invalid acloud arguments.
436    logging.error('Invalid acloud arguments found!')
437    return constants.EXIT_CODE_AVD_INVALID_ARGS
438
439
440if __name__ == '__main__':
441    if not os.getenv(constants.ANDROID_HOST_OUT, ''):
442        sys.exit()
443    index_targets()
444