1 /*
<lambda>null2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 @file:JvmName("Driver")
17
18 package com.android.tools.metalava
19
20 import com.android.SdkConstants.DOT_JAR
21 import com.android.SdkConstants.DOT_JAVA
22 import com.android.SdkConstants.DOT_KT
23 import com.android.SdkConstants.DOT_TXT
24 import com.android.ide.common.process.CachedProcessOutputHandler
25 import com.android.ide.common.process.DefaultProcessExecutor
26 import com.android.ide.common.process.ProcessInfoBuilder
27 import com.android.ide.common.process.ProcessOutput
28 import com.android.ide.common.process.ProcessOutputHandler
29 import com.android.tools.lint.KotlinLintAnalyzerFacade
30 import com.android.tools.lint.UastEnvironment
31 import com.android.tools.lint.annotations.Extractor
32 import com.android.tools.lint.checks.infrastructure.ClassName
33 import com.android.tools.lint.detector.api.assertionsEnabled
34 import com.android.tools.metalava.CompatibilityCheck.CheckRequest
35 import com.android.tools.metalava.apilevels.ApiGenerator
36 import com.android.tools.metalava.doclava1.ApiPredicate
37 import com.android.tools.metalava.doclava1.FilterPredicate
38 import com.android.tools.metalava.doclava1.Issues
39 import com.android.tools.metalava.model.ClassItem
40 import com.android.tools.metalava.model.Codebase
41 import com.android.tools.metalava.model.Item
42 import com.android.tools.metalava.model.PackageDocs
43 import com.android.tools.metalava.model.psi.PsiBasedCodebase
44 import com.android.tools.metalava.model.psi.packageHtmlToJavadoc
45 import com.android.tools.metalava.model.text.TextCodebase
46 import com.android.tools.metalava.model.visitors.ApiVisitor
47 import com.android.tools.metalava.stub.StubWriter
48 import com.android.utils.StdLogger
49 import com.android.utils.StdLogger.Level.ERROR
50 import com.google.common.base.Stopwatch
51 import com.google.common.collect.Lists
52 import com.google.common.io.Files
53 import com.intellij.core.CoreApplicationEnvironment
54 import com.intellij.openapi.Disposable
55 import com.intellij.openapi.diagnostic.DefaultLogger
56 import com.intellij.openapi.extensions.Extensions
57 import com.intellij.openapi.roots.LanguageLevelProjectExtension
58 import com.intellij.openapi.util.Disposer
59 import com.intellij.pom.java.LanguageLevel
60 import com.intellij.psi.javadoc.CustomJavadocTagProvider
61 import com.intellij.psi.javadoc.JavadocTagInfo
62 import org.jetbrains.kotlin.config.LanguageVersionSettings
63 import java.io.File
64 import java.io.IOException
65 import java.io.OutputStream
66 import java.io.OutputStreamWriter
67 import java.io.PrintWriter
68 import java.util.concurrent.TimeUnit.SECONDS
69 import java.util.function.Predicate
70 import kotlin.system.exitProcess
71 import kotlin.text.Charsets.UTF_8
72
73 const val PROGRAM_NAME = "metalava"
74 const val HELP_PROLOGUE = "$PROGRAM_NAME extracts metadata from source code to generate artifacts such as the " +
75 "signature files, the SDK stub files, external annotations etc."
76
77 @Suppress("PropertyName") // Can't mark const because trimIndent() :-(
78 val BANNER: String = """
79 _ _
80 _ __ ___ ___| |_ __ _| | __ ___ ____ _
81 | '_ ` _ \ / _ \ __/ _` | |/ _` \ \ / / _` |
82 | | | | | | __/ || (_| | | (_| |\ V / (_| |
83 |_| |_| |_|\___|\__\__,_|_|\__,_| \_/ \__,_|
84 """.trimIndent()
85
86 fun main(args: Array<String>) {
87 run(args, setExitCode = true)
88 }
89
90 internal var hasFileReadViolations = false
91
92 /**
93 * The metadata driver is a command line interface to extracting various metadata
94 * from a source tree (or existing signature files etc). Run with --help to see
95 * more details.
96 */
runnull97 fun run(
98 originalArgs: Array<String>,
99 stdout: PrintWriter = PrintWriter(OutputStreamWriter(System.out)),
100 stderr: PrintWriter = PrintWriter(OutputStreamWriter(System.err)),
101 setExitCode: Boolean = false
102 ): Boolean {
103 var exitCode = 0
104
105 try {
106 val modifiedArgs = preprocessArgv(originalArgs)
107
108 progress("$PROGRAM_NAME started\n")
109
110 // Dump the arguments, and maybe generate a rerun-script.
111 maybeDumpArgv(stdout, originalArgs, modifiedArgs)
112
113 // Actual work begins here.
114 compatibility = Compatibility(compat = Options.useCompatMode(modifiedArgs))
115 options = Options(modifiedArgs, stdout, stderr)
116
117 maybeActivateSandbox()
118
119 processFlags()
120
121 if (options.allReporters.any { it.hasErrors() } && !options.passBaselineUpdates) {
122 // Repeat the errors at the end to make it easy to find the actual problems.
123 if (options.repeatErrorsMax > 0) {
124 repeatErrors(stderr, options.allReporters, options.repeatErrorsMax)
125 }
126 exitCode = -1
127 }
128 if (hasFileReadViolations) {
129 if (options.strictInputFiles.shouldFail) {
130 stderr.print("Error: ")
131 exitCode = -1
132 } else {
133 stderr.print("Warning: ")
134 }
135 stderr.println("$PROGRAM_NAME detected access to files that are not explicitly specified. See ${options.strictInputViolationsFile} for details.")
136 }
137 } catch (e: DriverException) {
138 stdout.flush()
139 stderr.flush()
140
141 val prefix = if (e.exitCode != 0) { "Aborting: " } else { "" }
142
143 if (e.stderr.isNotBlank()) {
144 stderr.println("\n${prefix}${e.stderr}")
145 }
146 if (e.stdout.isNotBlank()) {
147 stdout.println("\n${prefix}${e.stdout}")
148 }
149 exitCode = e.exitCode
150 } finally {
151 disposeUastEnvironment()
152 }
153
154 // Update and close all baseline files.
155 options.allBaselines.forEach { baseline ->
156 if (options.verbose) {
157 baseline.dumpStats(options.stdout)
158 }
159 if (baseline.close()) {
160 if (!options.quiet) {
161 stdout.println("$PROGRAM_NAME wrote updated baseline to ${baseline.updateFile}")
162 }
163 }
164 }
165
166 options.reportEvenIfSuppressedWriter?.close()
167 options.strictInputViolationsPrintWriter?.close()
168
169 // Show failure messages, if any.
170 options.allReporters.forEach {
171 it.writeErrorMessage(stderr)
172 }
173
174 stdout.flush()
175 stderr.flush()
176
177 if (setExitCode) {
178 exit(exitCode)
179 }
180
181 return exitCode == 0
182 }
183
exitnull184 private fun exit(exitCode: Int = 0) {
185 if (options.verbose) {
186 progress("$PROGRAM_NAME exiting with exit code $exitCode\n")
187 }
188 options.stdout.flush()
189 options.stderr.flush()
190 exitProcess(exitCode)
191 }
192
maybeActivateSandboxnull193 private fun maybeActivateSandbox() {
194 // Set up a sandbox to detect access to files that are not explicitly specified.
195 if (options.strictInputFiles == Options.StrictInputFileMode.PERMISSIVE) {
196 return
197 }
198
199 val writer = options.strictInputViolationsPrintWriter!!
200
201 // Writes all violations to [Options.strictInputFiles].
202 // If Options.StrictInputFile.Mode is STRICT, then all violations on reads are logged, and the
203 // tool exits with a negative error code if there are any file read violations. Directory read
204 // violations are logged, but are considered to be a "warning" and doesn't affect the exit code.
205 // If STRICT_WARN, all violations on reads are logged similar to STRICT, but the exit code is
206 // unaffected.
207 // If STRICT_WITH_STACK, similar to STRICT, but also logs the stack trace to
208 // Options.strictInputFiles.
209 // See [FileReadSandbox] for the details.
210 FileReadSandbox.activate(object : FileReadSandbox.Listener {
211 var seen = mutableSetOf<String>()
212 override fun onViolation(absolutePath: String, isDirectory: Boolean) {
213 if (!seen.contains(absolutePath)) {
214 val suffix = if (isDirectory) "/" else ""
215 writer.println("$absolutePath$suffix")
216 if (options.strictInputFiles == Options.StrictInputFileMode.STRICT_WITH_STACK) {
217 Throwable().printStackTrace(writer)
218 }
219 seen.add(absolutePath)
220 if (!isDirectory) {
221 hasFileReadViolations = true
222 }
223 }
224 }
225 })
226 }
227
repeatErrorsnull228 private fun repeatErrors(writer: PrintWriter, reporters: List<Reporter>, max: Int) {
229 writer.println("Error: $PROGRAM_NAME detected the following problems:")
230 val totalErrors = reporters.sumBy { it.errorCount }
231 var remainingCap = max
232 var totalShown = 0
233 reporters.forEach {
234 var numShown = it.printErrors(writer, remainingCap)
235 remainingCap -= numShown
236 totalShown += numShown
237 }
238 if (totalShown < totalErrors) {
239 writer.println("${totalErrors - totalShown} more error(s) omitted. Search the log for 'error:' to find all of them.")
240 }
241 }
242
processFlagsnull243 private fun processFlags() {
244 val stopwatch = Stopwatch.createStarted()
245
246 processNonCodebaseFlags()
247
248 val sources = options.sources
249 val codebase =
250 if (sources.isNotEmpty() && sources[0].path.endsWith(DOT_TXT)) {
251 // Make sure all the source files have .txt extensions.
252 sources.firstOrNull { !it.path.endsWith(DOT_TXT) }?. let {
253 throw DriverException("Inconsistent input file types: The first file is of $DOT_TXT, but detected different extension in ${it.path}")
254 }
255 SignatureFileLoader.loadFiles(sources, options.inputKotlinStyleNulls)
256 } else if (options.apiJar != null) {
257 loadFromJarFile(options.apiJar!!)
258 } else if (sources.size == 1 && sources[0].path.endsWith(DOT_JAR)) {
259 loadFromJarFile(sources[0])
260 } else if (sources.isNotEmpty() || options.sourcePath.isNotEmpty()) {
261 loadFromSources()
262 } else {
263 return
264 }
265 options.manifest?.let { codebase.manifest = it }
266
267 if (options.verbose) {
268 progress("$PROGRAM_NAME analyzed API in ${stopwatch.elapsed(SECONDS)} seconds\n")
269 }
270
271 options.subtractApi?.let {
272 progress("Subtracting API: ")
273 subtractApi(codebase, it)
274 }
275
276 val androidApiLevelXml = options.generateApiLevelXml
277 val apiLevelJars = options.apiLevelJars
278 if (androidApiLevelXml != null && apiLevelJars != null) {
279 progress("Generating API levels XML descriptor file, ${androidApiLevelXml.name}: ")
280 ApiGenerator.generate(apiLevelJars, androidApiLevelXml, codebase)
281 }
282
283 if (options.docStubsDir != null && codebase.supportsDocumentation()) {
284 progress("Enhancing docs: ")
285 val docAnalyzer = DocAnalyzer(codebase)
286 docAnalyzer.enhance()
287
288 val applyApiLevelsXml = options.applyApiLevelsXml
289 if (applyApiLevelsXml != null) {
290 progress("Applying API levels")
291 docAnalyzer.applyApiLevels(applyApiLevelsXml)
292 }
293 }
294
295 // Generate the documentation stubs *before* we migrate nullness information.
296 options.docStubsDir?.let {
297 createStubFiles(
298 it, codebase, docStubs = true,
299 writeStubList = options.docStubsSourceList != null
300 )
301 }
302
303 // Based on the input flags, generates various output files such
304 // as signature files and/or stubs files
305 options.apiFile?.let { apiFile ->
306 val apiType = ApiType.PUBLIC_API
307 val apiEmit = apiType.getEmitFilter()
308 val apiReference = apiType.getReferenceFilter()
309
310 createReportFile(codebase, apiFile, "API") { printWriter ->
311 SignatureWriter(printWriter, apiEmit, apiReference, codebase.preFiltered)
312 }
313 }
314
315 options.apiXmlFile?.let { apiFile ->
316 val apiType = ApiType.PUBLIC_API
317 val apiEmit = apiType.getEmitFilter()
318 val apiReference = apiType.getReferenceFilter()
319
320 createReportFile(codebase, apiFile, "XML API") { printWriter ->
321 JDiffXmlWriter(printWriter, apiEmit, apiReference, codebase.preFiltered)
322 }
323 }
324
325 options.removedApiFile?.let { apiFile ->
326 val unfiltered = codebase.original ?: codebase
327
328 val apiType = ApiType.REMOVED
329 val removedEmit = apiType.getEmitFilter()
330 val removedReference = apiType.getReferenceFilter()
331
332 createReportFile(unfiltered, apiFile, "removed API") { printWriter ->
333 SignatureWriter(printWriter, removedEmit, removedReference, codebase.original != null)
334 }
335 }
336
337 options.removedDexApiFile?.let { apiFile ->
338 val unfiltered = codebase.original ?: codebase
339
340 val removedFilter = FilterPredicate(ApiPredicate(matchRemoved = true))
341 val removedReference = ApiPredicate(ignoreShown = true, ignoreRemoved = true)
342 val memberIsNotCloned: Predicate<Item> = Predicate { !it.isCloned() }
343 val removedDexEmit = memberIsNotCloned.and(removedFilter)
344
345 createReportFile(
346 unfiltered, apiFile, "removed DEX API"
347 ) { printWriter -> DexApiWriter(printWriter, removedDexEmit, removedReference) }
348 }
349
350 options.sdkValueDir?.let { dir ->
351 dir.mkdirs()
352 SdkFileWriter(codebase, dir).generate()
353 }
354
355 for (check in options.compatibilityChecks) {
356 checkCompatibility(codebase, check)
357 }
358
359 val previousApiFile = options.migrateNullsFrom
360 if (previousApiFile != null) {
361 val previous =
362 if (previousApiFile.path.endsWith(DOT_JAR)) {
363 loadFromJarFile(previousApiFile)
364 } else {
365 SignatureFileLoader.load(
366 file = previousApiFile,
367 kotlinStyleNulls = options.inputKotlinStyleNulls
368 )
369 }
370
371 // If configured, checks for newly added nullness information compared
372 // to the previous stable API and marks the newly annotated elements
373 // as migrated (which will cause the Kotlin compiler to treat problems
374 // as warnings instead of errors
375
376 migrateNulls(codebase, previous)
377
378 previous.dispose()
379 }
380
381 convertToWarningNullabilityAnnotations(codebase, options.forceConvertToWarningNullabilityAnnotations)
382
383 // Now that we've migrated nullness information we can proceed to write non-doc stubs, if any.
384
385 options.stubsDir?.let {
386 createStubFiles(
387 it, codebase, docStubs = false,
388 writeStubList = options.stubsSourceList != null
389 )
390
391 val stubAnnotations = options.copyStubAnnotationsFrom
392 if (stubAnnotations != null) {
393 // Support pointing to both stub-annotations and stub-annotations/src/main/java
394 val src = File(stubAnnotations, "src${File.separator}main${File.separator}java")
395 val source = if (src.isDirectory) src else stubAnnotations
396 source.listFiles()?.forEach { file ->
397 RewriteAnnotations().copyAnnotations(codebase, file, File(it, file.name))
398 }
399 }
400 }
401
402 if (options.docStubsDir == null && options.stubsDir == null) {
403 val writeStubsFile: (File) -> Unit = { file ->
404 val root = File("").absoluteFile
405 val rootPath = root.path
406 val contents = sources.joinToString(" ") {
407 val path = it.path
408 if (path.startsWith(rootPath)) {
409 path.substring(rootPath.length)
410 } else {
411 path
412 }
413 }
414 file.writeText(contents)
415 }
416 options.stubsSourceList?.let(writeStubsFile)
417 options.docStubsSourceList?.let(writeStubsFile)
418 }
419 options.externalAnnotations?.let { extractAnnotations(codebase, it) }
420
421 // Coverage stats?
422 if (options.dumpAnnotationStatistics) {
423 progress("Measuring annotation statistics: ")
424 AnnotationStatistics(codebase).count()
425 }
426 if (options.annotationCoverageOf.isNotEmpty()) {
427 progress("Measuring annotation coverage: ")
428 AnnotationStatistics(codebase).measureCoverageOf(options.annotationCoverageOf)
429 }
430
431 if (options.verbose) {
432 val packageCount = codebase.size()
433 progress("$PROGRAM_NAME finished handling $packageCount packages in ${stopwatch.elapsed(SECONDS)} seconds\n")
434 }
435
436 invokeDocumentationTool()
437 }
438
subtractApinull439 fun subtractApi(codebase: Codebase, subtractApiFile: File) {
440 val path = subtractApiFile.path
441 val oldCodebase =
442 when {
443 path.endsWith(DOT_TXT) -> SignatureFileLoader.load(subtractApiFile)
444 path.endsWith(DOT_JAR) -> loadFromJarFile(subtractApiFile)
445 else -> throw DriverException("Unsupported $ARG_SUBTRACT_API format, expected .txt or .jar: ${subtractApiFile.name}")
446 }
447
448 CodebaseComparator().compare(object : ComparisonVisitor() {
449 override fun compare(old: ClassItem, new: ClassItem) {
450 new.emit = false
451 }
452 }, oldCodebase, codebase, ApiType.ALL.getReferenceFilter())
453 }
454
processNonCodebaseFlagsnull455 fun processNonCodebaseFlags() {
456 // --copy-annotations?
457 val privateAnnotationsSource = options.privateAnnotationsSource
458 val privateAnnotationsTarget = options.privateAnnotationsTarget
459 if (privateAnnotationsSource != null && privateAnnotationsTarget != null) {
460 val rewrite = RewriteAnnotations()
461 // Support pointing to both stub-annotations and stub-annotations/src/main/java
462 val src = File(privateAnnotationsSource, "src${File.separator}main${File.separator}java")
463 val source = if (src.isDirectory) src else privateAnnotationsSource
464 source.listFiles()?.forEach { file ->
465 rewrite.modifyAnnotationSources(null, file, File(privateAnnotationsTarget, file.name))
466 }
467 }
468
469 // --rewrite-annotations?
470 options.rewriteAnnotations?.let { RewriteAnnotations().rewriteAnnotations(it) }
471
472 // Convert android.jar files?
473 options.androidJarSignatureFiles?.let { root ->
474 // Generate API signature files for all the historical JAR files
475 ConvertJarsToSignatureFiles().convertJars(root)
476 }
477
478 for (convert in options.convertToXmlFiles) {
479 val signatureApi = SignatureFileLoader.load(
480 file = convert.fromApiFile,
481 kotlinStyleNulls = options.inputKotlinStyleNulls
482 )
483
484 val apiType = ApiType.ALL
485 val apiEmit = apiType.getEmitFilter()
486 val strip = convert.strip
487 val apiReference = if (strip) apiType.getEmitFilter() else apiType.getReferenceFilter()
488 val baseFile = convert.baseApiFile
489
490 val outputApi =
491 if (baseFile != null) {
492 // Convert base on a diff
493 val baseApi = SignatureFileLoader.load(
494 file = baseFile,
495 kotlinStyleNulls = options.inputKotlinStyleNulls
496 )
497
498 val includeFields =
499 if (convert.outputFormat == FileFormat.V2) true else compatibility.includeFieldsInApiDiff
500 TextCodebase.computeDelta(baseFile, baseApi, signatureApi, includeFields)
501 } else {
502 signatureApi
503 }
504
505 if (outputApi.isEmpty() && baseFile != null && compatibility.compat) {
506 // doclava compatibility: emits error warning instead of emitting empty <api/> element
507 options.stdout.println("No API change detected, not generating diff")
508 } else {
509 val output = convert.outputFile
510 if (convert.outputFormat == FileFormat.JDIFF) {
511 // See JDiff's XMLToAPI#nameAPI
512 val apiName = convert.outputFile.nameWithoutExtension.replace(' ', '_')
513 createReportFile(outputApi, output, "JDiff File") { printWriter ->
514 JDiffXmlWriter(printWriter, apiEmit, apiReference, signatureApi.preFiltered && !strip, apiName)
515 }
516 } else {
517 val prevOptions = options
518 val prevCompatibility = compatibility
519 try {
520 when (convert.outputFormat) {
521 FileFormat.V1 -> {
522 compatibility = Compatibility(true)
523 options = Options(emptyArray(), options.stdout, options.stderr)
524 FileFormat.V1.configureOptions(options, compatibility)
525 }
526 FileFormat.V2 -> {
527 compatibility = Compatibility(false)
528 options = Options(emptyArray(), options.stdout, options.stderr)
529 FileFormat.V2.configureOptions(options, compatibility)
530 }
531 else -> error("Unsupported format ${convert.outputFormat}")
532 }
533
534 createReportFile(outputApi, output, "Diff API File") { printWriter ->
535 SignatureWriter(
536 printWriter, apiEmit, apiReference, signatureApi.preFiltered && !strip
537 )
538 }
539 } finally {
540 options = prevOptions
541 compatibility = prevCompatibility
542 }
543 }
544 }
545 }
546 }
547
548 /**
549 * Checks compatibility of the given codebase with the codebase described in the
550 * signature file.
551 */
checkCompatibilitynull552 fun checkCompatibility(
553 codebase: Codebase,
554 check: CheckRequest
555 ) {
556 progress("Checking API compatibility ($check): ")
557 val signatureFile = check.file
558
559 val current =
560 if (signatureFile.path.endsWith(DOT_JAR)) {
561 loadFromJarFile(signatureFile)
562 } else {
563 SignatureFileLoader.load(
564 file = signatureFile,
565 kotlinStyleNulls = options.inputKotlinStyleNulls
566 )
567 }
568
569 if (current is TextCodebase && current.format > FileFormat.V1 && options.outputFormat == FileFormat.V1) {
570 throw DriverException("Cannot perform compatibility check of signature file $signatureFile in format ${current.format} without analyzing current codebase with $ARG_FORMAT=${current.format}")
571 }
572
573 var base: Codebase? = null
574 val releaseType = check.releaseType
575 val apiType = check.apiType
576
577 // If diffing with a system-api or test-api (or other signature-based codebase
578 // generated from --show-annotations), the API is partial: it's only listing
579 // the API that is *different* from the base API. This really confuses the
580 // codebase comparison when diffing with a complete codebase, since it looks like
581 // many classes and members have been added and removed. Therefore, the comparison
582 // is simpler if we just make the comparison with the same generated signature
583 // file. If we've only emitted one for the new API, use it directly, if not, generate
584 // it first
585 val new =
586 if (check.codebase != null) {
587 SignatureFileLoader.load(
588 file = check.codebase,
589 kotlinStyleNulls = options.inputKotlinStyleNulls
590 )
591 } else if (!options.showUnannotated || apiType != ApiType.PUBLIC_API) {
592 val apiFile = apiType.getSignatureFile(codebase, "compat-check-signatures-$apiType")
593
594 // Fast path: if the signature files are identical, we're already good!
595 if (apiFile.readText(UTF_8) == signatureFile.readText(UTF_8)) {
596 return
597 }
598
599 base = codebase
600
601 SignatureFileLoader.load(
602 file = apiFile,
603 kotlinStyleNulls = options.inputKotlinStyleNulls
604 )
605 } else {
606 // Fast path: if we've already generated a signature file and it's identical, we're good!
607 val apiFile = options.apiFile
608 if (apiFile != null && apiFile.readText(UTF_8) == signatureFile.readText(UTF_8)) {
609 return
610 }
611
612 codebase
613 }
614
615 // If configured, compares the new API with the previous API and reports
616 // any incompatibilities.
617 CompatibilityCheck.checkCompatibility(new, current, releaseType, apiType, base)
618
619 // Make sure the text files are identical too? (only applies for *current.txt;
620 // last-released is expected to differ)
621 if (releaseType == ReleaseType.DEV && !options.allowCompatibleDifferences) {
622 val apiFile = if (new.location.isFile)
623 new.location
624 else
625 apiType.getSignatureFile(codebase, "compat-diff-signatures-$apiType")
626
627 fun getCanonicalSignatures(file: File): String {
628 // Get rid of trailing newlines and Windows line endings
629 val text = file.readText(UTF_8)
630 return text.replace("\r\n", "\n").trim()
631 }
632 val currentTxt = getCanonicalSignatures(signatureFile)
633 val newTxt = getCanonicalSignatures(apiFile)
634 if (newTxt != currentTxt) {
635 val diff = getNativeDiff(signatureFile, apiFile) ?: getDiff(currentTxt, newTxt, 1)
636 val updateApi = if (isBuildingAndroid())
637 "Run make update-api to update.\n"
638 else
639 ""
640 val message =
641 """
642 Your changes have resulted in differences in the signature file
643 for the ${apiType.displayName} API.
644
645 The changes may be compatible, but the signature file needs to be updated.
646 $updateApi
647 Diffs:
648 """.trimIndent() + "\n" + diff
649
650 throw DriverException(exitCode = -1, stderr = message)
651 }
652 }
653 }
654
createTempFilenull655 fun createTempFile(namePrefix: String, nameSuffix: String): File {
656 val tempFolder = options.tempFolder
657 return if (tempFolder != null) {
658 val preferred = File(tempFolder, namePrefix + nameSuffix)
659 if (!preferred.exists()) {
660 return preferred
661 }
662 File.createTempFile(namePrefix, nameSuffix, tempFolder)
663 } else {
664 File.createTempFile(namePrefix, nameSuffix)
665 }
666 }
667
invokeDocumentationToolnull668 fun invokeDocumentationTool() {
669 if (options.noDocs) {
670 return
671 }
672
673 val args = options.invokeDocumentationToolArguments
674 if (args.isNotEmpty()) {
675 if (!options.quiet) {
676 options.stdout.println(
677 "Invoking external documentation tool ${args[0]} with arguments\n\"${
678 args.slice(1 until args.size).joinToString(separator = "\",\n\"") { it }}\""
679 )
680 options.stdout.flush()
681 }
682
683 val builder = ProcessInfoBuilder()
684
685 builder.setExecutable(File(args[0]))
686 builder.addArgs(args.slice(1 until args.size))
687
688 val processOutputHandler =
689 if (options.quiet) {
690 CachedProcessOutputHandler()
691 } else {
692 object : ProcessOutputHandler {
693 override fun handleOutput(processOutput: ProcessOutput?) {
694 }
695
696 override fun createOutput(): ProcessOutput {
697 val out = PrintWriterOutputStream(options.stdout)
698 val err = PrintWriterOutputStream(options.stderr)
699 return object : ProcessOutput {
700 override fun getStandardOutput(): OutputStream {
701 return out
702 }
703
704 override fun getErrorOutput(): OutputStream {
705 return err
706 }
707
708 override fun close() {
709 out.flush()
710 err.flush()
711 }
712 }
713 }
714 }
715 }
716
717 val result = DefaultProcessExecutor(StdLogger(ERROR))
718 .execute(builder.createProcess(), processOutputHandler)
719
720 val exitCode = result.exitValue
721 if (!options.quiet) {
722 options.stdout.println("${args[0]} finished with exitCode $exitCode")
723 options.stdout.flush()
724 }
725 if (exitCode != 0) {
726 val stdout = if (processOutputHandler is CachedProcessOutputHandler)
727 processOutputHandler.processOutput.standardOutputAsString
728 else ""
729 val stderr = if (processOutputHandler is CachedProcessOutputHandler)
730 processOutputHandler.processOutput.errorOutputAsString
731 else ""
732 throw DriverException(
733 stdout = "Invoking documentation tool ${args[0]} failed with exit code $exitCode\n$stdout",
734 stderr = stderr,
735 exitCode = exitCode
736 )
737 }
738 }
739 }
740
741 class PrintWriterOutputStream(private val writer: PrintWriter) : OutputStream() {
742
writenull743 override fun write(b: ByteArray) {
744 writer.write(String(b, UTF_8))
745 }
746
writenull747 override fun write(b: Int) {
748 write(byteArrayOf(b.toByte()), 0, 1)
749 }
750
writenull751 override fun write(b: ByteArray, off: Int, len: Int) {
752 writer.write(String(b, off, len, UTF_8))
753 }
754
flushnull755 override fun flush() {
756 writer.flush()
757 }
758
closenull759 override fun close() {
760 writer.close()
761 }
762 }
763
migrateNullsnull764 private fun migrateNulls(codebase: Codebase, previous: Codebase) {
765 previous.compareWith(NullnessMigration(), codebase)
766 }
767
convertToWarningNullabilityAnnotationsnull768 private fun convertToWarningNullabilityAnnotations(codebase: Codebase, filter: PackageFilter?) {
769 if (filter != null) {
770 // Our caller has asked for these APIs to not trigger nullness errors (only warnings) if
771 // their callers make incorrect nullness assumptions (for example, calling a function on a
772 // reference of nullable type). The way to communicate this to kotlinc is to mark these
773 // APIs as RecentlyNullable/RecentlyNonNull
774 codebase.accept(MarkPackagesAsRecent(filter))
775 }
776 }
777
loadFromSourcesnull778 private fun loadFromSources(): Codebase {
779 progress("Processing sources: ")
780
781 val sources = if (options.sources.isEmpty()) {
782 if (options.verbose) {
783 options.stdout.println("No source files specified: recursively including all sources found in the source path (${options.sourcePath.joinToString()}})")
784 }
785 gatherSources(options.sourcePath)
786 } else {
787 options.sources
788 }
789
790 progress("Reading Codebase: ")
791 val codebase = parseSources(sources, "Codebase loaded from source folders")
792
793 progress("Analyzing API: ")
794
795 val analyzer = ApiAnalyzer(codebase)
796 analyzer.mergeExternalInclusionAnnotations()
797 analyzer.computeApi()
798
799 val filterEmit = ApiPredicate(ignoreShown = true, ignoreRemoved = false)
800 val apiEmit = ApiPredicate(ignoreShown = true)
801 val apiReference = ApiPredicate(ignoreShown = true)
802
803 // Copy methods from soon-to-be-hidden parents into descendant classes, when necessary. Do
804 // this before merging annotations or performing checks on the API to ensure that these methods
805 // can have annotations added and are checked properly.
806 progress("Insert missing stubs methods: ")
807 analyzer.generateInheritedStubs(apiEmit, apiReference)
808
809 analyzer.mergeExternalQualifierAnnotations()
810 options.nullabilityAnnotationsValidator?.validateAllFrom(codebase, options.validateNullabilityFromList)
811 options.nullabilityAnnotationsValidator?.report()
812 analyzer.handleStripping()
813
814 // General API checks for Android APIs
815 AndroidApiChecks().check(codebase)
816
817 if (options.checkApi) {
818 progress("API Lint: ")
819 val localTimer = Stopwatch.createStarted()
820 // See if we should provide a previous codebase to provide a delta from?
821 val previousApiFile = options.checkApiBaselineApiFile
822 val previous =
823 when {
824 previousApiFile == null -> null
825 previousApiFile.path.endsWith(DOT_JAR) -> loadFromJarFile(previousApiFile)
826 else -> SignatureFileLoader.load(
827 file = previousApiFile,
828 kotlinStyleNulls = options.inputKotlinStyleNulls
829 )
830 }
831 val apiLintReporter = options.reporterApiLint
832 ApiLint.check(codebase, previous, apiLintReporter)
833 progress("$PROGRAM_NAME ran api-lint in ${localTimer.elapsed(SECONDS)} seconds with ${apiLintReporter.getBaselineDescription()}")
834 }
835
836 // Compute default constructors (and add missing package private constructors
837 // to make stubs compilable if necessary). Do this after all the checks as
838 // these are not part of the API.
839 if (options.stubsDir != null || options.docStubsDir != null) {
840 progress("Insert missing constructors: ")
841 analyzer.addConstructors(filterEmit)
842 }
843
844 progress("Performing misc API checks: ")
845 analyzer.performChecks()
846
847 return codebase
848 }
849
850 /**
851 * Returns a codebase initialized from the given Java or Kotlin source files, with the given
852 * description. The codebase will use a project environment initialized according to the current
853 * [options].
854 */
parseSourcesnull855 internal fun parseSources(
856 sources: List<File>,
857 description: String,
858 sourcePath: List<File> = options.sourcePath,
859 classpath: List<File> = options.classpath,
860 javaLanguageLevel: LanguageLevel = options.javaLanguageLevel,
861 kotlinLanguageLevel: LanguageVersionSettings = options.kotlinLanguageLevel,
862 manifest: File? = options.manifest,
863 currentApiLevel: Int = options.currentApiLevel + if (options.currentCodeName != null) 1 else 0
864 ): PsiBasedCodebase {
865 val environment = createProjectEnvironment()
866 val projectEnvironment = environment.projectEnvironment
867 val project = projectEnvironment.project
868
869 // Push language level to PSI handler
870 project.getComponent(LanguageLevelProjectExtension::class.java)?.languageLevel = javaLanguageLevel
871
872 val joined = mutableListOf<File>()
873 joined.addAll(sourcePath.mapNotNull { if (it.path.isNotBlank()) it.absoluteFile else null })
874 joined.addAll(classpath.map { it.absoluteFile })
875
876 // Add in source roots implied by the source files
877 val sourceRoots = mutableListOf<File>()
878 if (options.allowImplicitRoot) {
879 extractRoots(sources, sourceRoots)
880 joined.addAll(sourceRoots)
881 }
882
883 // Create project environment with those paths
884 projectEnvironment.registerPaths(joined)
885
886 val kotlinFiles = sources.filter { it.path.endsWith(DOT_KT) }
887 val trace = KotlinLintAnalyzerFacade().analyze(
888 files = kotlinFiles,
889 contentRoots = joined,
890 project = project,
891 environment = environment,
892 javaLanguageLevel = javaLanguageLevel,
893 kotlinLanguageLevel = kotlinLanguageLevel
894 )
895
896 val rootDir = sourceRoots.firstOrNull() ?: sourcePath.firstOrNull() ?: File("").canonicalFile
897
898 val units = Extractor.createUnitsForFiles(project, sources)
899 val packageDocs = gatherHiddenPackagesFromJavaDocs(sourcePath)
900
901 val codebase = PsiBasedCodebase(rootDir, description)
902 codebase.initialize(project, units, packageDocs)
903 codebase.manifest = manifest
904 codebase.apiLevel = currentApiLevel
905 codebase.bindingContext = trace.bindingContext
906 return codebase
907 }
908
loadFromJarFilenull909 fun loadFromJarFile(apiJar: File, manifest: File? = null, preFiltered: Boolean = false): Codebase {
910 val environment = createProjectEnvironment()
911 val projectEnvironment = environment.projectEnvironment
912
913 progress("Processing jar file: ")
914
915 // Create project environment with those paths
916 val project = projectEnvironment.project
917 projectEnvironment.registerPaths(listOf(apiJar))
918
919 val kotlinFiles = emptyList<File>()
920 val trace = KotlinLintAnalyzerFacade().analyze(kotlinFiles, listOf(apiJar), project, environment)
921
922 val codebase = PsiBasedCodebase(apiJar, "Codebase loaded from $apiJar")
923 codebase.initialize(project, apiJar, preFiltered)
924 if (manifest != null) {
925 codebase.manifest = options.manifest
926 }
927 val apiEmit = ApiPredicate(ignoreShown = true)
928 val apiReference = ApiPredicate(ignoreShown = true)
929 val analyzer = ApiAnalyzer(codebase)
930 analyzer.mergeExternalInclusionAnnotations()
931 analyzer.computeApi()
932 analyzer.mergeExternalQualifierAnnotations()
933 options.nullabilityAnnotationsValidator?.validateAllFrom(codebase, options.validateNullabilityFromList)
934 options.nullabilityAnnotationsValidator?.report()
935 analyzer.generateInheritedStubs(apiEmit, apiReference)
936 codebase.bindingContext = trace.bindingContext
937 return codebase
938 }
939
createProjectEnvironmentnull940 private fun createProjectEnvironment(): UastEnvironment {
941 ensurePsiFileCapacity()
942 val disposable = Disposer.newDisposable()
943 disposables.add(disposable)
944 val environment = UastEnvironment.create(disposable)
945
946 if (!assertionsEnabled() &&
947 System.getenv(ENV_VAR_METALAVA_DUMP_ARGV) == null &&
948 !isUnderTest()
949 ) {
950 DefaultLogger.disableStderrDumping(disposable)
951 }
952
953 val projectEnvironment = environment.projectEnvironment
954
955 // Missing service needed in metalava but not in lint: javadoc handling
956 projectEnvironment.project.registerService(
957 com.intellij.psi.javadoc.JavadocManager::class.java,
958 com.intellij.psi.impl.source.javadoc.JavadocManagerImpl::class.java
959 )
960 projectEnvironment.registerProjectExtensionPoint(JavadocTagInfo.EP_NAME,
961 com.intellij.psi.javadoc.JavadocTagInfo::class.java)
962 CoreApplicationEnvironment.registerExtensionPoint(
963 Extensions.getRootArea(), CustomJavadocTagProvider.EP_NAME, CustomJavadocTagProvider::class.java
964 )
965
966 return environment
967 }
968
969 private val disposables = mutableListOf<Disposable>()
970
disposeUastEnvironmentnull971 private fun disposeUastEnvironment() {
972 for (project in disposables) {
973 Disposer.dispose(project)
974 }
975 UastEnvironment.disposeApplicationEnvironment()
976 }
977
ensurePsiFileCapacitynull978 private fun ensurePsiFileCapacity() {
979 val fileSize = System.getProperty("idea.max.intellisense.filesize")
980 if (fileSize == null) {
981 // Ensure we can handle large compilation units like android.R
982 System.setProperty("idea.max.intellisense.filesize", "100000")
983 }
984 }
985
extractAnnotationsnull986 private fun extractAnnotations(codebase: Codebase, file: File) {
987 val localTimer = Stopwatch.createStarted()
988
989 options.externalAnnotations?.let { outputFile ->
990 @Suppress("UNCHECKED_CAST")
991 ExtractAnnotations(
992 codebase,
993 outputFile
994 ).extractAnnotations()
995 if (options.verbose) {
996 progress("$PROGRAM_NAME extracted annotations into $file in ${localTimer.elapsed(SECONDS)} seconds\n")
997 }
998 }
999 }
1000
createStubFilesnull1001 private fun createStubFiles(stubDir: File, codebase: Codebase, docStubs: Boolean, writeStubList: Boolean) {
1002 // Generating stubs from a sig-file-based codebase is problematic
1003 assert(codebase.supportsDocumentation())
1004
1005 // Temporary bug workaround for org.chromium.arc
1006 if (options.sourcePath.firstOrNull()?.path?.endsWith("org.chromium.arc") == true) {
1007 codebase.findClass("org.chromium.mojo.bindings.Callbacks")?.hidden = true
1008 }
1009
1010 if (docStubs) {
1011 progress("Generating documentation stub files: ")
1012 } else {
1013 progress("Generating stub files: ")
1014 }
1015
1016 val localTimer = Stopwatch.createStarted()
1017 val prevCompatibility = compatibility
1018 if (compatibility.compat) {
1019 compatibility = Compatibility(false)
1020 // But preserve the setting for whether we want to erase throws signatures (to ensure the API
1021 // stays compatible)
1022 compatibility.useErasureInThrows = prevCompatibility.useErasureInThrows
1023 }
1024
1025 val stubWriter =
1026 StubWriter(
1027 codebase = codebase,
1028 stubsDir = stubDir,
1029 generateAnnotations = options.generateAnnotations,
1030 preFiltered = codebase.preFiltered,
1031 docStubs = docStubs
1032 )
1033 codebase.accept(stubWriter)
1034
1035 if (docStubs) {
1036 // Overview docs? These are generally in the empty package.
1037 codebase.findPackage("")?.let { empty ->
1038 val overview = codebase.getPackageDocs()?.getOverviewDocumentation(empty)
1039 if (overview != null && overview.isNotBlank()) {
1040 stubWriter.writeDocOverview(empty, overview)
1041 }
1042 }
1043 }
1044
1045 if (writeStubList) {
1046 // Optionally also write out a list of source files that were generated; used
1047 // for example to point javadoc to the stubs output to generate documentation
1048 val file = if (docStubs) {
1049 options.docStubsSourceList ?: options.stubsSourceList
1050 } else {
1051 options.stubsSourceList
1052 }
1053 file?.let {
1054 val root = File("").absoluteFile
1055 stubWriter.writeSourceList(it, root)
1056 }
1057 }
1058
1059 compatibility = prevCompatibility
1060
1061 progress(
1062 "$PROGRAM_NAME wrote ${if (docStubs) "documentation" else ""} stubs directory $stubDir in ${
1063 localTimer.elapsed(SECONDS)} seconds\n"
1064 )
1065 }
1066
createReportFilenull1067 fun createReportFile(
1068 codebase: Codebase,
1069 apiFile: File,
1070 description: String?,
1071 createVisitor: (PrintWriter) -> ApiVisitor
1072 ) {
1073 if (description != null) {
1074 progress("Writing $description file: ")
1075 }
1076 val localTimer = Stopwatch.createStarted()
1077 try {
1078 val writer = PrintWriter(Files.asCharSink(apiFile, UTF_8).openBufferedStream())
1079 writer.use { printWriter ->
1080 val apiWriter = createVisitor(printWriter)
1081 codebase.accept(apiWriter)
1082 }
1083 } catch (e: IOException) {
1084 reporter.report(Issues.IO_ERROR, apiFile, "Cannot open file for write.")
1085 }
1086 if (description != null && options.verbose) {
1087 progress("$PROGRAM_NAME wrote $description file $apiFile in ${localTimer.elapsed(SECONDS)} seconds\n")
1088 }
1089 }
1090
skippableDirectorynull1091 private fun skippableDirectory(file: File): Boolean = file.path.endsWith(".git") && file.name == ".git"
1092
1093 private fun addSourceFiles(list: MutableList<File>, file: File) {
1094 if (file.isDirectory) {
1095 if (skippableDirectory(file)) {
1096 return
1097 }
1098 if (java.nio.file.Files.isSymbolicLink(file.toPath())) {
1099 reporter.report(
1100 Issues.IGNORING_SYMLINK, file,
1101 "Ignoring symlink during source file discovery directory traversal"
1102 )
1103 return
1104 }
1105 val files = file.listFiles()
1106 if (files != null) {
1107 for (child in files) {
1108 addSourceFiles(list, child)
1109 }
1110 }
1111 } else {
1112 if (file.isFile && (file.path.endsWith(DOT_JAVA) || file.path.endsWith(DOT_KT))) {
1113 list.add(file)
1114 }
1115 }
1116 }
1117
gatherSourcesnull1118 fun gatherSources(sourcePath: List<File>): List<File> {
1119 val sources = Lists.newArrayList<File>()
1120 for (file in sourcePath) {
1121 if (file.path.isBlank()) {
1122 // --source-path "" means don't search source path; use "." for pwd
1123 continue
1124 }
1125 addSourceFiles(sources, file.absoluteFile)
1126 }
1127 return sources.sortedWith(compareBy { it.name })
1128 }
1129
addHiddenPackagesnull1130 private fun addHiddenPackages(
1131 packageToDoc: MutableMap<String, String>,
1132 packageToOverview: MutableMap<String, String>,
1133 hiddenPackages: MutableSet<String>,
1134 file: File,
1135 pkg: String
1136 ) {
1137 if (FileReadSandbox.isDirectory(file)) {
1138 if (skippableDirectory(file)) {
1139 return
1140 }
1141 // Ignore symbolic links during traversal
1142 if (java.nio.file.Files.isSymbolicLink(file.toPath())) {
1143 reporter.report(
1144 Issues.IGNORING_SYMLINK, file,
1145 "Ignoring symlink during package.html discovery directory traversal"
1146 )
1147 return
1148 }
1149 val files = file.listFiles()
1150 if (files != null) {
1151 for (child in files) {
1152 var subPkg =
1153 if (FileReadSandbox.isDirectory(child))
1154 if (pkg.isEmpty())
1155 child.name
1156 else pkg + "." + child.name
1157 else pkg
1158
1159 if (subPkg.endsWith("src.main.java")) {
1160 // It looks like the source path was incorrectly configured; make corrections here
1161 // to ensure that we map the package.html files to the real packages.
1162 subPkg = ""
1163 }
1164
1165 addHiddenPackages(packageToDoc, packageToOverview, hiddenPackages, child, subPkg)
1166 }
1167 }
1168 } else if (FileReadSandbox.isFile(file)) {
1169 var javadoc = false
1170 val map = when (file.name) {
1171 "package.html" -> {
1172 javadoc = true; packageToDoc
1173 }
1174 "overview.html" -> {
1175 packageToOverview
1176 }
1177 else -> return
1178 }
1179 var contents = Files.asCharSource(file, UTF_8).read()
1180 if (javadoc) {
1181 contents = packageHtmlToJavadoc(contents)
1182 }
1183
1184 var realPkg = pkg
1185 // Sanity check the package; it's computed from the directory name
1186 // relative to the source path, but if the real source path isn't
1187 // passed in (and is instead some directory containing the source path)
1188 // then we compute the wrong package here. Instead, look for an adjacent
1189 // java class and pick the package from it
1190 for (sibling in file.parentFile?.listFiles() ?: emptyArray()) {
1191 if (sibling.path.endsWith(DOT_JAVA)) {
1192 val javaPkg = ClassName(sibling.readText()).packageName
1193 if (javaPkg != null) {
1194 realPkg = javaPkg
1195 break
1196 }
1197 }
1198 }
1199
1200 map[realPkg] = contents
1201 if (contents.contains("@hide")) {
1202 hiddenPackages.add(realPkg)
1203 }
1204 }
1205 }
1206
gatherHiddenPackagesFromJavaDocsnull1207 private fun gatherHiddenPackagesFromJavaDocs(sourcePath: List<File>): PackageDocs {
1208 val packageComments = HashMap<String, String>(100)
1209 val overviewHtml = HashMap<String, String>(10)
1210 val hiddenPackages = HashSet<String>(100)
1211 for (file in sourcePath) {
1212 if (file.path.isBlank()) {
1213 // Ignoring empty paths, which means "no source path search". Use "." for current directory.
1214 continue
1215 }
1216 addHiddenPackages(packageComments, overviewHtml, hiddenPackages, file, "")
1217 }
1218
1219 return PackageDocs(packageComments, overviewHtml, hiddenPackages)
1220 }
1221
extractRootsnull1222 fun extractRoots(sources: List<File>, sourceRoots: MutableList<File> = mutableListOf()): List<File> {
1223 // Cache for each directory since computing root for a source file is
1224 // expensive
1225 val dirToRootCache = mutableMapOf<String, File>()
1226 for (file in sources) {
1227 val parent = file.parentFile ?: continue
1228 val found = dirToRootCache[parent.path]
1229 if (found != null) {
1230 continue
1231 }
1232
1233 val root = findRoot(file) ?: continue
1234 dirToRootCache[parent.path] = root
1235
1236 if (!sourceRoots.contains(root)) {
1237 sourceRoots.add(root)
1238 }
1239 }
1240
1241 return sourceRoots
1242 }
1243
1244 /**
1245 * If given a full path to a Java or Kotlin source file, produces the path to
1246 * the source root if possible.
1247 */
findRootnull1248 private fun findRoot(file: File): File? {
1249 val path = file.path
1250 if (path.endsWith(DOT_JAVA) || path.endsWith(DOT_KT)) {
1251 val pkg = findPackage(file) ?: return null
1252 val parent = file.parentFile ?: return null
1253 val endIndex = parent.path.length - pkg.length
1254 val before = path[endIndex - 1]
1255 if (before == '/' || before == '\\') {
1256 return File(path.substring(0, endIndex))
1257 } else {
1258 reporter.report(
1259 Issues.IO_ERROR, file, "$PROGRAM_NAME was unable to determine the package name. " +
1260 "This usually means that a source file was where the directory does not seem to match the package " +
1261 "declaration; we expected the path $path to end with /${pkg.replace('.', '/') + '/' + file.name}"
1262 )
1263 }
1264 }
1265
1266 return null
1267 }
1268
1269 /** Finds the package of the given Java/Kotlin source file, if possible */
findPackagenull1270 fun findPackage(file: File): String? {
1271 val source = Files.asCharSource(file, UTF_8).read()
1272 return findPackage(source)
1273 }
1274
1275 /** Finds the package of the given Java/Kotlin source code, if possible */
findPackagenull1276 fun findPackage(source: String): String? {
1277 return ClassName(source).packageName
1278 }
1279
1280 /** Whether metalava is running unit tests */
isUnderTestnull1281 fun isUnderTest() = java.lang.Boolean.getBoolean(ENV_VAR_METALAVA_TESTS_RUNNING)
1282
1283 /** Whether metalava is being invoked as part of an Android platform build */
1284 fun isBuildingAndroid() = System.getenv("ANDROID_BUILD_TOP") != null && !isUnderTest()
1285