1#!/usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""ModuleData information.""" 18 19from __future__ import absolute_import 20 21import glob 22import logging 23import os 24import re 25 26from aidegen import constant 27from aidegen.lib import common_util 28from aidegen.lib import module_info 29from aidegen.lib import project_config 30 31# Parse package name from the package declaration line of a java. 32# Group matches "foo.bar" of line "package foo.bar;" or "package foo.bar" 33_PACKAGE_RE = re.compile(r'\s*package\s+(?P<package>[^(;|\s)]+)\s*', re.I) 34_ANDROID_SUPPORT_PATH_KEYWORD = 'prebuilts/sdk/current/' 35 36# File extensions 37_JAVA_EXT = '.java' 38_KOTLIN_EXT = '.kt' 39_SRCJAR_EXT = '.srcjar' 40_TARGET_FILES = [_JAVA_EXT, _KOTLIN_EXT] 41_JARJAR_RULES_FILE = 'jarjar-rules.txt' 42_KEY_JARJAR_RULES = 'jarjar_rules' 43_NAME_AAPT2 = 'aapt2' 44_TARGET_R_SRCJAR = 'R.srcjar' 45_TARGET_AAPT2_SRCJAR = _NAME_AAPT2 + _SRCJAR_EXT 46_TARGET_BUILD_FILES = [_TARGET_AAPT2_SRCJAR, _TARGET_R_SRCJAR] 47_IGNORE_DIRS = [ 48 # The java files under this directory have to be ignored because it will 49 # cause duplicated classes by libcore/ojluni/src/main/java. 50 'libcore/ojluni/src/lambda/java' 51] 52_ANDROID = 'android' 53_REPACKAGES = 'repackaged' 54_FRAMEWORK_SRCJARS_PATH = os.path.join(constant.FRAMEWORK_PATH, 55 constant.FRAMEWORK_SRCJARS) 56 57 58class ModuleData: 59 """ModuleData class. 60 61 Attributes: 62 All following relative paths stand for the path relative to the android 63 repo root. 64 65 module_path: A string of the relative path to the module. 66 src_dirs: A list to keep the unique source folder relative paths. 67 test_dirs: A list to keep the unique test folder relative paths. 68 jar_files: A list to keep the unique jar file relative paths. 69 r_java_paths: A list to keep the R folder paths to use in Eclipse. 70 srcjar_paths: A list to keep the srcjar source root paths to use in 71 IntelliJ. Some modules' srcjar_paths will be removed when 72 run with the MultiProjectInfo. 73 dep_paths: A list to keep the dependency modules' path. 74 referenced_by_jar: A boolean to check if the module is referenced by a 75 jar file. 76 build_targets: A set to keep the unique build target jar or srcjar file 77 relative paths which are ready to be rebuld. 78 missing_jars: A set to keep the jar file relative paths if it doesn't 79 exist. 80 specific_soong_path: A string of the relative path to the module's 81 intermediates folder under out/. 82 """ 83 84 def __init__(self, module_name, module_data, depth): 85 """Initialize ModuleData. 86 87 Args: 88 module_name: Name of the module. 89 module_data: A dictionary holding a module information. 90 depth: An integer shows the depth of module dependency referenced by 91 source. Zero means the max module depth. 92 For example: 93 { 94 'class': ['APPS'], 95 'path': ['path/to/the/module'], 96 'depth': 0, 97 'dependencies': ['bouncycastle', 'ims-common'], 98 'srcs': [ 99 'path/to/the/module/src/com/android/test.java', 100 'path/to/the/module/src/com/google/test.java', 101 'out/soong/.intermediates/path/to/the/module/test/src/ 102 com/android/test.srcjar' 103 ], 104 'installed': ['out/target/product/generic_x86_64/ 105 system/framework/framework.jar'], 106 'jars': ['settings.jar'], 107 'jarjar_rules': ['jarjar-rules.txt'] 108 } 109 """ 110 assert module_name, 'Module name can\'t be null.' 111 assert module_data, 'Module data of %s can\'t be null.' % module_name 112 self.module_name = module_name 113 self.module_data = module_data 114 self._init_module_path() 115 self._init_module_depth(depth) 116 self.src_dirs = [] 117 self.test_dirs = [] 118 self.jar_files = [] 119 self.r_java_paths = [] 120 self.srcjar_paths = [] 121 self.dep_paths = [] 122 self.referenced_by_jar = False 123 self.build_targets = set() 124 self.missing_jars = set() 125 self.specific_soong_path = os.path.join( 126 'out/soong/.intermediates', self.module_path, self.module_name) 127 128 def _is_app_module(self): 129 """Check if the current module's class is APPS""" 130 return self._check_key('class') and 'APPS' in self.module_data['class'] 131 132 def _is_target_module(self): 133 """Check if the current module is a target module. 134 135 A target module is the target project or a module under the 136 target project and it's module depth is 0. 137 For example: aidegen Settings framework 138 The target projects are Settings and framework so they are also 139 target modules. And the dependent module SettingsUnitTests's path 140 is packages/apps/Settings/tests/unit so it also a target module. 141 """ 142 return self.module_depth == 0 143 144 def _collect_r_srcs_paths(self): 145 """Collect the source folder of R.java. 146 147 Check if the path of aapt2.srcjar or R.srcjar exists, these are both the 148 values of key "srcjars" in module_data. If neither of the cases exists, 149 build it onto an intermediates directory. 150 151 For IntelliJ, we can set the srcjar file as a source root for 152 dependency. For Eclipse, we still use the R folder as dependencies until 153 we figure out how to set srcjar file as dependency. 154 # TODO(b/135594800): Set aapt2.srcjar or R.srcjar as a dependency in 155 Eclipse. 156 """ 157 if (self._is_app_module() and self._is_target_module() 158 and self._check_key(constant.KEY_SRCJARS)): 159 for srcjar in self.module_data[constant.KEY_SRCJARS]: 160 if not os.path.exists(common_util.get_abs_path(srcjar)): 161 self.build_targets.add(srcjar) 162 self._collect_srcjar_path(srcjar) 163 r_dir = self._get_r_dir(srcjar) 164 if r_dir and r_dir not in self.r_java_paths: 165 self.r_java_paths.append(r_dir) 166 167 def _collect_srcjar_path(self, srcjar): 168 """Collect the source folders from a srcjar path. 169 170 Set the aapt2.srcjar or R.srcjar as source root: 171 Case aapt2.srcjar: 172 The source path string is 173 out/.../Bluetooth_intermediates/aapt2.srcjar. 174 Case R.srcjar: 175 The source path string is out/soong/.../gen/android/R.srcjar. 176 177 Args: 178 srcjar: A file path string relative to ANDROID_BUILD_TOP, the build 179 target of the module to generate R.java. 180 """ 181 if (os.path.basename(srcjar) in _TARGET_BUILD_FILES 182 and srcjar not in self.srcjar_paths): 183 self.srcjar_paths.append(srcjar) 184 185 def _collect_all_srcjar_paths(self): 186 """Collect all srcjar files of target module as source folders. 187 188 Since the aidl files are built to *.java and collected in the 189 aidl.srcjar file by the build system. AIDEGen needs to collect these 190 aidl.srcjar files as the source root folders in IntelliJ. Furthermore, 191 AIDEGen collects all *.srcjar files for other cases to fulfil the same 192 purpose. 193 """ 194 if self._is_target_module() and self._check_key(constant.KEY_SRCJARS): 195 for srcjar in self.module_data[constant.KEY_SRCJARS]: 196 if not os.path.exists(common_util.get_abs_path(srcjar)): 197 self.build_targets.add(srcjar) 198 if srcjar not in self.srcjar_paths: 199 self.srcjar_paths.append(srcjar) 200 201 @staticmethod 202 def _get_r_dir(srcjar): 203 """Get the source folder of R.java for Eclipse. 204 205 Get the folder contains the R.java of aapt2.srcjar or R.srcjar: 206 Case aapt2.srcjar: 207 If the relative path of the aapt2.srcjar is a/b/aapt2.srcjar, the 208 source root of the R.java is a/b/aapt2 209 Case R.srcjar: 210 If the relative path of the R.srcjar is a/b/android/R.srcjar, the 211 source root of the R.java is a/b/aapt2/R 212 213 Args: 214 srcjar: A file path string, the build target of the module to 215 generate R.java. 216 217 Returns: 218 A relative source folder path string, and return None if the target 219 file name is not aapt2.srcjar or R.srcjar. 220 """ 221 target_folder, target_file = os.path.split(srcjar) 222 base_dirname = os.path.basename(target_folder) 223 if target_file == _TARGET_AAPT2_SRCJAR: 224 return os.path.join(target_folder, _NAME_AAPT2) 225 if target_file == _TARGET_R_SRCJAR and base_dirname == _ANDROID: 226 return os.path.join(os.path.dirname(target_folder), 227 _NAME_AAPT2, 'R') 228 return None 229 230 def _init_module_path(self): 231 """Inintialize self.module_path.""" 232 self.module_path = (self.module_data[constant.KEY_PATH][0] 233 if self._check_key(constant.KEY_PATH) else '') 234 235 def _init_module_depth(self, depth): 236 """Initialize module depth's settings. 237 238 Set the module's depth from module info when user have -d parameter. 239 Set the -d value from user input, default to 0. 240 241 Args: 242 depth: the depth to be set. 243 """ 244 self.module_depth = (int(self.module_data[constant.KEY_DEPTH]) 245 if depth else 0) 246 self.depth_by_source = depth 247 248 def _is_android_supported_module(self): 249 """Determine if this is an Android supported module.""" 250 return common_util.is_source_under_relative_path( 251 self.module_path, _ANDROID_SUPPORT_PATH_KEYWORD) 252 253 def _check_jarjar_rules_exist(self): 254 """Check if jarjar rules exist.""" 255 return (_KEY_JARJAR_RULES in self.module_data and 256 self.module_data[_KEY_JARJAR_RULES][0] == _JARJAR_RULES_FILE) 257 258 def _check_jars_exist(self): 259 """Check if jars exist.""" 260 return self._check_key(constant.KEY_JARS) 261 262 def _check_classes_jar_exist(self): 263 """Check if classes_jar exist.""" 264 return self._check_key(constant.KEY_CLASSES_JAR) 265 266 def _collect_srcs_paths(self): 267 """Collect source folder paths in src_dirs from module_data['srcs'].""" 268 if self._check_key(constant.KEY_SRCS): 269 scanned_dirs = set() 270 for src_item in self.module_data[constant.KEY_SRCS]: 271 src_dir = None 272 src_item = os.path.relpath(src_item) 273 if common_util.is_target(src_item, _TARGET_FILES): 274 # Only scan one java file in each source directories. 275 src_item_dir = os.path.dirname(src_item) 276 if src_item_dir not in scanned_dirs: 277 scanned_dirs.add(src_item_dir) 278 src_dir = self._get_source_folder(src_item) 279 else: 280 # To record what files except java and kt in the srcs. 281 logging.debug('%s is not in parsing scope.', src_item) 282 if src_dir: 283 self._add_to_source_or_test_dirs( 284 self._switch_repackaged(src_dir)) 285 286 def _check_key(self, key): 287 """Check if key is in self.module_data and not empty. 288 289 Args: 290 key: the key to be checked. 291 """ 292 return key in self.module_data and self.module_data[key] 293 294 def _add_to_source_or_test_dirs(self, src_dir): 295 """Add folder to source or test directories. 296 297 Args: 298 src_dir: the directory to be added. 299 """ 300 if (src_dir not in _IGNORE_DIRS and src_dir not in self.src_dirs 301 and src_dir not in self.test_dirs): 302 if self._is_test_module(src_dir): 303 self.test_dirs.append(src_dir) 304 else: 305 self.src_dirs.append(src_dir) 306 307 @staticmethod 308 def _is_test_module(src_dir): 309 """Check if the module path is a test module path. 310 311 Args: 312 src_dir: the directory to be checked. 313 314 Returns: 315 True if module path is a test module path, otherwise False. 316 """ 317 return constant.KEY_TESTS in src_dir.split(os.sep) 318 319 def _get_source_folder(self, java_file): 320 """Parsing a java to get the package name to filter out source path. 321 322 Args: 323 java_file: A string, the java file with relative path. 324 e.g. path/to/the/java/file.java 325 326 Returns: 327 source_folder: A string of path to source folder(e.g. src/main/java) 328 or none when it failed to get package name. 329 """ 330 abs_java_path = common_util.get_abs_path(java_file) 331 if os.path.exists(abs_java_path): 332 package_name = self._get_package_name(abs_java_path) 333 if package_name: 334 return self._parse_source_path(java_file, package_name) 335 return None 336 337 @staticmethod 338 def _parse_source_path(java_file, package_name): 339 """Parse the source path by filter out the package name. 340 341 Case 1: 342 java file: a/b/c/d/e.java 343 package name: c.d 344 The source folder is a/b. 345 346 Case 2: 347 java file: a/b/c.d/e.java 348 package name: c.d 349 The source folder is a/b. 350 351 Case 3: 352 java file: a/b/c/d/e.java 353 package name: x.y 354 The source folder is a/b/c/d. 355 356 Case 4: 357 java file: a/b/c.d/e/c/d/f.java 358 package name: c.d 359 The source folder is a/b/c.d/e. 360 361 Case 5: 362 java file: a/b/c.d/e/c.d/e/f.java 363 package name: c.d.e 364 The source folder is a/b/c.d/e. 365 366 Args: 367 java_file: A string of the java file relative path. 368 package_name: A string of the java file's package name. 369 370 Returns: 371 A string, the source folder path. 372 """ 373 java_file_name = os.path.basename(java_file) 374 pattern = r'%s/%s$' % (package_name, java_file_name) 375 search_result = re.search(pattern, java_file) 376 if search_result: 377 return java_file[:search_result.start()].strip(os.sep) 378 return os.path.dirname(java_file) 379 380 @staticmethod 381 def _switch_repackaged(src_dir): 382 """Changes the directory to repackaged if it does exist. 383 384 Args: 385 src_dir: a string of relative path. 386 387 Returns: 388 The source folder under repackaged if it exists, otherwise the 389 original one. 390 """ 391 root_path = common_util.get_android_root_dir() 392 dir_list = src_dir.split(os.sep) 393 for i in range(1, len(dir_list)): 394 tmp_dir = dir_list.copy() 395 tmp_dir.insert(i, _REPACKAGES) 396 real_path = os.path.join(root_path, os.path.join(*tmp_dir)) 397 if os.path.exists(real_path): 398 return os.path.relpath(real_path, root_path) 399 return src_dir 400 401 @staticmethod 402 def _get_package_name(abs_java_path): 403 """Get the package name by parsing a java file. 404 405 Args: 406 abs_java_path: A string of the java file with absolute path. 407 e.g. /root/path/to/the/java/file.java 408 409 Returns: 410 package_name: A string of package name. 411 """ 412 package_name = None 413 with open(abs_java_path, encoding='utf8') as data: 414 for line in data.read().splitlines(): 415 match = _PACKAGE_RE.match(line) 416 if match: 417 package_name = match.group('package') 418 break 419 return package_name 420 421 def _append_jar_file(self, jar_path): 422 """Append a path to the jar file into self.jar_files if it's exists. 423 424 Args: 425 jar_path: A path supposed to be a jar file. 426 427 Returns: 428 Boolean: True if jar_path is an existing jar file. 429 """ 430 if common_util.is_target(jar_path, constant.TARGET_LIBS): 431 self.referenced_by_jar = True 432 if os.path.isfile(common_util.get_abs_path(jar_path)): 433 if jar_path not in self.jar_files: 434 self.jar_files.append(jar_path) 435 else: 436 self.missing_jars.add(jar_path) 437 return True 438 return False 439 440 def _append_classes_jar(self): 441 """Append the jar file as dependency for prebuilt modules.""" 442 for jar in self.module_data[constant.KEY_CLASSES_JAR]: 443 if self._append_jar_file(jar): 444 break 445 446 def _append_jar_from_installed(self, specific_dir=None): 447 """Append a jar file's path to the list of jar_files with matching 448 path_prefix. 449 450 There might be more than one jar in "installed" parameter and only the 451 first jar file is returned. If specific_dir is set, the jar file must be 452 under the specific directory or its sub-directory. 453 454 Args: 455 specific_dir: A string of path. 456 """ 457 if self._check_key(constant.KEY_INSTALLED): 458 for jar in self.module_data[constant.KEY_INSTALLED]: 459 if specific_dir and not jar.startswith(specific_dir): 460 continue 461 if self._append_jar_file(jar): 462 break 463 464 def _set_jars_jarfile(self): 465 """Append prebuilt jars of module into self.jar_files. 466 467 Some modules' sources are prebuilt jar files instead of source java 468 files. The jar files can be imported into IntelliJ as a dependency 469 directly. There is only jar file name in self.module_data['jars'], it 470 has to be combined with self.module_data['path'] to append into 471 self.jar_files. 472 Once the file doesn't exist, it's not assumed to be a prebuilt jar so 473 that we can ignore it. 474 # TODO(b/141959125): Collect the correct prebuilt jar files by jdeps.go. 475 476 For example: 477 'asm-6.0': { 478 'jars': [ 479 'asm-6.0.jar' 480 ], 481 'path': [ 482 'prebuilts/misc/common/asm' 483 ], 484 }, 485 Path to the jar file is prebuilts/misc/common/asm/asm-6.0.jar. 486 """ 487 if self._check_key(constant.KEY_JARS): 488 for jar_name in self.module_data[constant.KEY_JARS]: 489 if self._check_key(constant.KEY_INSTALLED): 490 self._append_jar_from_installed() 491 else: 492 jar_path = os.path.join(self.module_path, jar_name) 493 jar_abs = common_util.get_abs_path(jar_path) 494 if not os.path.isfile(jar_abs) and jar_name.endswith( 495 'prebuilt.jar'): 496 rel_path = self._get_jar_path_from_prebuilts(jar_name) 497 if rel_path: 498 jar_path = rel_path 499 if os.path.exists(common_util.get_abs_path(jar_path)): 500 self._append_jar_file(jar_path) 501 502 @staticmethod 503 def _get_jar_path_from_prebuilts(jar_name): 504 """Get prebuilt jar file from prebuilts folder. 505 506 If the prebuilt jar file we get from method _set_jars_jarfile() does not 507 exist, we should search the prebuilt jar file in prebuilts folder. 508 For example: 509 'platformprotos': { 510 'jars': [ 511 'platformprotos-prebuilt.jar' 512 ], 513 'path': [ 514 'frameworks/base' 515 ], 516 }, 517 We get an incorrect path: 'frameworks/base/platformprotos-prebuilt.jar' 518 If the file does not exist, we should search the file name from 519 prebuilts folder. If we can get the correct path from 'prebuilts', we 520 can replace it with the incorrect path. 521 522 Args: 523 jar_name: The prebuilt jar file name. 524 525 Return: 526 A relative prebuilt jar file path if found, otherwise None. 527 """ 528 rel_path = '' 529 search = os.sep.join( 530 [common_util.get_android_root_dir(), 'prebuilts/**', jar_name]) 531 results = glob.glob(search, recursive=True) 532 if results: 533 jar_abs = results[0] 534 rel_path = os.path.relpath( 535 jar_abs, common_util.get_android_root_dir()) 536 return rel_path 537 538 def _collect_specific_jars(self): 539 """Collect specific types of jar files.""" 540 if self._is_android_supported_module(): 541 self._append_jar_from_installed() 542 elif self._check_jarjar_rules_exist(): 543 self._append_jar_from_installed(self.specific_soong_path) 544 elif self._check_jars_exist(): 545 self._set_jars_jarfile() 546 547 def _collect_classes_jars(self): 548 """Collect classes jar files.""" 549 # If there is no source/tests folder of the module, reference the 550 # module by jar. 551 if not self.src_dirs and not self.test_dirs: 552 # Add the classes.jar from the classes_jar attribute as 553 # dependency if it exists. If the classes.jar doesn't exist, 554 # find the jar file from the installed attribute and add the jar 555 # as dependency. 556 if self._check_classes_jar_exist(): 557 self._append_classes_jar() 558 else: 559 self._append_jar_from_installed() 560 561 def _collect_srcs_and_r_srcs_paths(self): 562 """Collect source and R source folder paths for the module.""" 563 self._collect_specific_jars() 564 self._collect_srcs_paths() 565 self._collect_classes_jars() 566 self._collect_r_srcs_paths() 567 self._collect_all_srcjar_paths() 568 569 def _collect_missing_jars(self): 570 """Collect missing jar files to rebuild them.""" 571 if self.referenced_by_jar and self.missing_jars: 572 self.build_targets |= self.missing_jars 573 574 def _collect_dep_paths(self): 575 """Collects the path of dependency modules.""" 576 config = project_config.ProjectConfig.get_instance() 577 modules_info = config.atest_module_info 578 self.dep_paths = [] 579 if self.module_path != constant.FRAMEWORK_PATH: 580 self.dep_paths.append(constant.FRAMEWORK_PATH) 581 self.dep_paths.append(_FRAMEWORK_SRCJARS_PATH) 582 if self.module_path != constant.LIBCORE_PATH: 583 self.dep_paths.append(constant.LIBCORE_PATH) 584 for module in self.module_data.get(constant.KEY_DEPENDENCIES, []): 585 for path in modules_info.get_paths(module): 586 if path not in self.dep_paths and path != self.module_path: 587 self.dep_paths.append(path) 588 589 def locate_sources_path(self): 590 """Locate source folders' paths or jar files.""" 591 # Check if users need to reference source according to source depth. 592 if not self.module_depth <= self.depth_by_source: 593 self._append_jar_from_installed(self.specific_soong_path) 594 else: 595 self._collect_srcs_and_r_srcs_paths() 596 self._collect_missing_jars() 597 598 599class EclipseModuleData(ModuleData): 600 """Deal with modules data for Eclipse 601 602 Only project target modules use source folder type and the other ones use 603 jar as their source. We'll combine both to establish the whole project's 604 dependencies. If the source folder used to build dependency jar file exists 605 in Android, we should provide the jar file path as <linkedResource> item in 606 source data. 607 """ 608 609 def __init__(self, module_name, module_data, project_relpath): 610 """Initialize EclipseModuleData. 611 612 Only project target modules apply source folder type, so set the depth 613 of module referenced by source to 0. 614 615 Args: 616 module_name: String type, name of the module. 617 module_data: A dictionary contains a module information. 618 project_relpath: A string stands for the project's relative path. 619 """ 620 super().__init__(module_name, module_data, depth=0) 621 related = module_info.AidegenModuleInfo.is_project_path_relative_module( 622 module_data, project_relpath) 623 self.is_project = related 624 625 def locate_sources_path(self): 626 """Locate source folders' paths or jar files. 627 628 Only collect source folders for the project modules and collect jar 629 files for the other dependent modules. 630 """ 631 if self.is_project: 632 self._locate_project_source_path() 633 else: 634 self._locate_jar_path() 635 self._collect_classes_jars() 636 self._collect_missing_jars() 637 638 def _add_to_source_or_test_dirs(self, src_dir): 639 """Add a folder to source list if it is not in ignored directories. 640 641 Override the parent method since the tests folder has no difference 642 with source folder in Eclipse. 643 644 Args: 645 src_dir: a string of relative path to the Android root. 646 """ 647 if src_dir not in _IGNORE_DIRS and src_dir not in self.src_dirs: 648 self.src_dirs.append(src_dir) 649 650 def _locate_project_source_path(self): 651 """Locate the source folder paths of the project module. 652 653 A project module is the target modules or paths that users key in 654 aidegen command. Collecting the source folders is necessary for 655 developers to edit code. And also collect the central R folder for the 656 dependency of resources. 657 """ 658 self._collect_srcs_paths() 659 self._collect_r_srcs_paths() 660 661 def _locate_jar_path(self): 662 """Locate the jar path of the module. 663 664 Use jar files for dependency modules for Eclipse. Collect the jar file 665 path with different cases. 666 """ 667 if self._check_jarjar_rules_exist(): 668 self._append_jar_from_installed(self.specific_soong_path) 669 elif self._check_jars_exist(): 670 self._set_jars_jarfile() 671 elif self._check_classes_jar_exist(): 672 self._append_classes_jar() 673 else: 674 self._append_jar_from_installed() 675