1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "Optimize.h"
18 
19 #include <memory>
20 #include <vector>
21 
22 #include "android-base/file.h"
23 #include "android-base/stringprintf.h"
24 
25 #include "androidfw/ConfigDescription.h"
26 #include "androidfw/ResourceTypes.h"
27 #include "androidfw/StringPiece.h"
28 
29 #include "Diagnostics.h"
30 #include "LoadedApk.h"
31 #include "ResourceUtils.h"
32 #include "SdkConstants.h"
33 #include "ValueVisitor.h"
34 #include "cmd/Util.h"
35 #include "configuration/ConfigurationParser.h"
36 #include "filter/AbiFilter.h"
37 #include "format/binary/TableFlattener.h"
38 #include "format/binary/XmlFlattener.h"
39 #include "io/BigBufferStream.h"
40 #include "io/Util.h"
41 #include "optimize/MultiApkGenerator.h"
42 #include "optimize/ResourceDeduper.h"
43 #include "optimize/ResourceFilter.h"
44 #include "optimize/ResourcePathShortener.h"
45 #include "optimize/VersionCollapser.h"
46 #include "split/TableSplitter.h"
47 #include "util/Files.h"
48 #include "util/Util.h"
49 
50 using ::aapt::configuration::Abi;
51 using ::aapt::configuration::OutputArtifact;
52 using ::android::ConfigDescription;
53 using ::android::ResTable_config;
54 using ::android::StringPiece;
55 using ::android::base::ReadFileToString;
56 using ::android::base::WriteStringToFile;
57 using ::android::base::StringAppendF;
58 using ::android::base::StringPrintf;
59 
60 namespace aapt {
61 
62 class OptimizeContext : public IAaptContext {
63  public:
64   OptimizeContext() = default;
65 
GetPackageType()66   PackageType GetPackageType() override {
67     // Not important here. Using anything other than kApp adds EXTRA validation, which we want to
68     // avoid.
69     return PackageType::kApp;
70   }
71 
GetDiagnostics()72   IDiagnostics* GetDiagnostics() override {
73     return &diagnostics_;
74   }
75 
GetNameMangler()76   NameMangler* GetNameMangler() override {
77     UNIMPLEMENTED(FATAL);
78     return nullptr;
79   }
80 
GetCompilationPackage()81   const std::string& GetCompilationPackage() override {
82     static std::string empty;
83     return empty;
84   }
85 
GetPackageId()86   uint8_t GetPackageId() override {
87     return 0;
88   }
89 
GetExternalSymbols()90   SymbolTable* GetExternalSymbols() override {
91     UNIMPLEMENTED(FATAL);
92     return nullptr;
93   }
94 
IsVerbose()95   bool IsVerbose() override {
96     return verbose_;
97   }
98 
SetVerbose(bool val)99   void SetVerbose(bool val) {
100     verbose_ = val;
101   }
102 
SetMinSdkVersion(int sdk_version)103   void SetMinSdkVersion(int sdk_version) {
104     sdk_version_ = sdk_version;
105   }
106 
GetMinSdkVersion()107   int GetMinSdkVersion() override {
108     return sdk_version_;
109   }
110 
111  private:
112   DISALLOW_COPY_AND_ASSIGN(OptimizeContext);
113 
114   StdErrDiagnostics diagnostics_;
115   bool verbose_ = false;
116   int sdk_version_ = 0;
117 };
118 
119 class Optimizer {
120  public:
Optimizer(OptimizeContext * context,const OptimizeOptions & options)121   Optimizer(OptimizeContext* context, const OptimizeOptions& options)
122       : options_(options), context_(context) {
123   }
124 
Run(std::unique_ptr<LoadedApk> apk)125   int Run(std::unique_ptr<LoadedApk> apk) {
126     if (context_->IsVerbose()) {
127       context_->GetDiagnostics()->Note(DiagMessage() << "Optimizing APK...");
128     }
129     if (!options_.resources_blacklist.empty()) {
130       ResourceFilter filter(options_.resources_blacklist);
131       if (!filter.Consume(context_, apk->GetResourceTable())) {
132         context_->GetDiagnostics()->Error(DiagMessage() << "failed filtering resources");
133         return 1;
134       }
135     }
136 
137     VersionCollapser collapser;
138     if (!collapser.Consume(context_, apk->GetResourceTable())) {
139       return 1;
140     }
141 
142     ResourceDeduper deduper;
143     if (!deduper.Consume(context_, apk->GetResourceTable())) {
144       context_->GetDiagnostics()->Error(DiagMessage() << "failed deduping resources");
145       return 1;
146     }
147 
148     if (options_.shorten_resource_paths) {
149       ResourcePathShortener shortener(options_.table_flattener_options.shortened_path_map);
150       if (!shortener.Consume(context_, apk->GetResourceTable())) {
151         context_->GetDiagnostics()->Error(DiagMessage() << "failed shortening resource paths");
152         return 1;
153       }
154       if (options_.shortened_paths_map_path
155           && !WriteShortenedPathsMap(options_.table_flattener_options.shortened_path_map,
156                                       options_.shortened_paths_map_path.value())) {
157         context_->GetDiagnostics()->Error(DiagMessage()
158                                           << "failed to write shortened resource paths to file");
159         return 1;
160       }
161     }
162 
163     // Adjust the SplitConstraints so that their SDK version is stripped if it is less than or
164     // equal to the minSdk.
165     options_.split_constraints =
166         AdjustSplitConstraintsForMinSdk(context_->GetMinSdkVersion(), options_.split_constraints);
167 
168     // Stripping the APK using the TableSplitter. The resource table is modified in place in the
169     // LoadedApk.
170     TableSplitter splitter(options_.split_constraints, options_.table_splitter_options);
171     if (!splitter.VerifySplitConstraints(context_)) {
172       return 1;
173     }
174     splitter.SplitTable(apk->GetResourceTable());
175 
176     auto path_iter = options_.split_paths.begin();
177     auto split_constraints_iter = options_.split_constraints.begin();
178     for (std::unique_ptr<ResourceTable>& split_table : splitter.splits()) {
179       if (context_->IsVerbose()) {
180         context_->GetDiagnostics()->Note(
181             DiagMessage(*path_iter) << "generating split with configurations '"
182                                     << util::Joiner(split_constraints_iter->configs, ", ") << "'");
183       }
184 
185       // Generate an AndroidManifest.xml for each split.
186       std::unique_ptr<xml::XmlResource> split_manifest =
187           GenerateSplitManifest(options_.app_info, *split_constraints_iter);
188       std::unique_ptr<IArchiveWriter> split_writer =
189           CreateZipFileArchiveWriter(context_->GetDiagnostics(), *path_iter);
190       if (!split_writer) {
191         return 1;
192       }
193 
194       if (!WriteSplitApk(split_table.get(), split_manifest.get(), split_writer.get())) {
195         return 1;
196       }
197 
198       ++path_iter;
199       ++split_constraints_iter;
200     }
201 
202     if (options_.apk_artifacts && options_.output_dir) {
203       MultiApkGenerator generator{apk.get(), context_};
204       MultiApkGeneratorOptions generator_options = {
205           options_.output_dir.value(), options_.apk_artifacts.value(),
206           options_.table_flattener_options, options_.kept_artifacts};
207       if (!generator.FromBaseApk(generator_options)) {
208         return 1;
209       }
210     }
211 
212     if (options_.output_path) {
213       std::unique_ptr<IArchiveWriter> writer =
214           CreateZipFileArchiveWriter(context_->GetDiagnostics(), options_.output_path.value());
215       if (!apk->WriteToArchive(context_, options_.table_flattener_options, writer.get())) {
216         return 1;
217       }
218     }
219 
220     return 0;
221   }
222 
223  private:
WriteSplitApk(ResourceTable * table,xml::XmlResource * manifest,IArchiveWriter * writer)224   bool WriteSplitApk(ResourceTable* table, xml::XmlResource* manifest, IArchiveWriter* writer) {
225     BigBuffer manifest_buffer(4096);
226     XmlFlattener xml_flattener(&manifest_buffer, {});
227     if (!xml_flattener.Consume(context_, manifest)) {
228       return false;
229     }
230 
231     io::BigBufferInputStream manifest_buffer_in(&manifest_buffer);
232     if (!io::CopyInputStreamToArchive(context_, &manifest_buffer_in, "AndroidManifest.xml",
233                                       ArchiveEntry::kCompress, writer)) {
234       return false;
235     }
236 
237     std::map<std::pair<ConfigDescription, StringPiece>, FileReference*> config_sorted_files;
238     for (auto& pkg : table->packages) {
239       for (auto& type : pkg->types) {
240         // Sort by config and name, so that we get better locality in the zip file.
241         config_sorted_files.clear();
242 
243         for (auto& entry : type->entries) {
244           for (auto& config_value : entry->values) {
245             auto* file_ref = ValueCast<FileReference>(config_value->value.get());
246             if (file_ref == nullptr) {
247               continue;
248             }
249 
250             if (file_ref->file == nullptr) {
251               ResourceNameRef name(pkg->name, type->type, entry->name);
252               context_->GetDiagnostics()->Warn(DiagMessage(file_ref->GetSource())
253                                                << "file for resource " << name << " with config '"
254                                                << config_value->config << "' not found");
255               continue;
256             }
257 
258             const StringPiece entry_name = entry->name;
259             config_sorted_files[std::make_pair(config_value->config, entry_name)] = file_ref;
260           }
261         }
262 
263         for (auto& entry : config_sorted_files) {
264           FileReference* file_ref = entry.second;
265           if (!io::CopyFileToArchivePreserveCompression(context_, file_ref->file, *file_ref->path,
266                                                         writer)) {
267             return false;
268           }
269         }
270       }
271     }
272 
273     BigBuffer table_buffer(4096);
274     TableFlattener table_flattener(options_.table_flattener_options, &table_buffer);
275     if (!table_flattener.Consume(context_, table)) {
276       return false;
277     }
278 
279     io::BigBufferInputStream table_buffer_in(&table_buffer);
280     return io::CopyInputStreamToArchive(context_, &table_buffer_in, "resources.arsc",
281                                         ArchiveEntry::kAlign, writer);
282   }
283 
WriteShortenedPathsMap(const std::map<std::string,std::string> & path_map,const std::string & file_path)284   bool WriteShortenedPathsMap(const std::map<std::string, std::string> &path_map,
285                                const std::string &file_path) {
286     std::stringstream ss;
287     for (auto it = path_map.cbegin(); it != path_map.cend(); ++it) {
288       ss << it->first << " -> " << it->second << "\n";
289     }
290     return WriteStringToFile(ss.str(), file_path);
291   }
292 
293   OptimizeOptions options_;
294   OptimizeContext* context_;
295 };
296 
ExtractObfuscationWhitelistFromConfig(const std::string & path,OptimizeContext * context,OptimizeOptions * options)297 bool ExtractObfuscationWhitelistFromConfig(const std::string& path, OptimizeContext* context,
298                                            OptimizeOptions* options) {
299   std::string contents;
300   if (!ReadFileToString(path, &contents, true)) {
301     context->GetDiagnostics()->Error(DiagMessage()
302                                      << "failed to parse whitelist from config file: " << path);
303     return false;
304   }
305   for (StringPiece resource_name : util::Tokenize(contents, ',')) {
306     options->table_flattener_options.whitelisted_resources.insert(
307         resource_name.to_string());
308   }
309   return true;
310 }
311 
ExtractConfig(const std::string & path,OptimizeContext * context,OptimizeOptions * options)312 bool ExtractConfig(const std::string& path, OptimizeContext* context,
313                                     OptimizeOptions* options) {
314   std::string content;
315   if (!android::base::ReadFileToString(path, &content, true /*follow_symlinks*/)) {
316     context->GetDiagnostics()->Error(DiagMessage(path) << "failed reading whitelist");
317     return false;
318   }
319 
320   size_t line_no = 0;
321   for (StringPiece line : util::Tokenize(content, '\n')) {
322     line_no++;
323     line = util::TrimWhitespace(line);
324     if (line.empty()) {
325       continue;
326     }
327 
328     auto split_line = util::Split(line, '#');
329     if (split_line.size() < 2) {
330       context->GetDiagnostics()->Error(DiagMessage(line) << "No # found in line");
331       return false;
332     }
333     StringPiece resource_string = split_line[0];
334     StringPiece directives = split_line[1];
335     ResourceNameRef resource_name;
336     if (!ResourceUtils::ParseResourceName(resource_string, &resource_name)) {
337       context->GetDiagnostics()->Error(DiagMessage(line) << "Malformed resource name");
338       return false;
339     }
340     if (!resource_name.package.empty()) {
341       context->GetDiagnostics()->Error(DiagMessage(line)
342                                        << "Package set for resource. Only use type/name");
343       return false;
344     }
345     for (StringPiece directive : util::Tokenize(directives, ',')) {
346       if (directive == "remove") {
347         options->resources_blacklist.insert(resource_name.ToResourceName());
348       } else if (directive == "no_obfuscate") {
349         options->table_flattener_options.whitelisted_resources.insert(
350             resource_name.entry.to_string());
351       }
352     }
353   }
354   return true;
355 }
356 
ExtractAppDataFromManifest(OptimizeContext * context,const LoadedApk * apk,OptimizeOptions * out_options)357 bool ExtractAppDataFromManifest(OptimizeContext* context, const LoadedApk* apk,
358                                 OptimizeOptions* out_options) {
359   const xml::XmlResource* manifest = apk->GetManifest();
360   if (manifest == nullptr) {
361     return false;
362   }
363 
364   Maybe<AppInfo> app_info = ExtractAppInfoFromBinaryManifest(*manifest, context->GetDiagnostics());
365   if (!app_info) {
366     context->GetDiagnostics()->Error(DiagMessage()
367                                      << "failed to extract data from AndroidManifest.xml");
368     return false;
369   }
370 
371   out_options->app_info = std::move(app_info.value());
372   context->SetMinSdkVersion(out_options->app_info.min_sdk_version.value_or_default(0));
373   return true;
374 }
375 
Action(const std::vector<std::string> & args)376 int OptimizeCommand::Action(const std::vector<std::string>& args) {
377   if (args.size() != 1u) {
378     std::cerr << "must have one APK as argument.\n\n";
379     Usage(&std::cerr);
380     return 1;
381   }
382 
383   const std::string& apk_path = args[0];
384   OptimizeContext context;
385   context.SetVerbose(verbose_);
386   IDiagnostics* diag = context.GetDiagnostics();
387 
388   if (config_path_) {
389     std::string& path = config_path_.value();
390     Maybe<ConfigurationParser> for_path = ConfigurationParser::ForPath(path);
391     if (for_path) {
392       options_.apk_artifacts = for_path.value().WithDiagnostics(diag).Parse(apk_path);
393       if (!options_.apk_artifacts) {
394         diag->Error(DiagMessage() << "Failed to parse the output artifact list");
395         return 1;
396       }
397 
398     } else {
399       diag->Error(DiagMessage() << "Could not parse config file " << path);
400       return 1;
401     }
402 
403     if (print_only_) {
404       for (const OutputArtifact& artifact : options_.apk_artifacts.value()) {
405         std::cout << artifact.name << std::endl;
406       }
407       return 0;
408     }
409 
410     if (!kept_artifacts_.empty()) {
411       for (const std::string& artifact_str : kept_artifacts_) {
412         for (const StringPiece& artifact : util::Tokenize(artifact_str, ',')) {
413           options_.kept_artifacts.insert(artifact.to_string());
414         }
415       }
416     }
417 
418     // Since we know that we are going to process the APK (not just print targets), make sure we
419     // have somewhere to write them to.
420     if (!options_.output_dir) {
421       diag->Error(DiagMessage() << "Output directory is required when using a configuration file");
422       return 1;
423     }
424   } else if (print_only_) {
425     diag->Error(DiagMessage() << "Asked to print artifacts without providing a configurations");
426     return 1;
427   }
428 
429   std::unique_ptr<LoadedApk> apk = LoadedApk::LoadApkFromPath(apk_path, context.GetDiagnostics());
430   if (!apk) {
431     return 1;
432   }
433 
434   if (target_densities_) {
435     // Parse the target screen densities.
436     for (const StringPiece& config_str : util::Tokenize(target_densities_.value(), ',')) {
437       Maybe<uint16_t> target_density = ParseTargetDensityParameter(config_str, diag);
438       if (!target_density) {
439         return 1;
440       }
441       options_.table_splitter_options.preferred_densities.push_back(target_density.value());
442     }
443   }
444 
445   std::unique_ptr<IConfigFilter> filter;
446   if (!configs_.empty()) {
447     filter = ParseConfigFilterParameters(configs_, diag);
448     if (filter == nullptr) {
449       return 1;
450     }
451     options_.table_splitter_options.config_filter = filter.get();
452   }
453 
454   // Parse the split parameters.
455   for (const std::string& split_arg : split_args_) {
456     options_.split_paths.emplace_back();
457     options_.split_constraints.emplace_back();
458     if (!ParseSplitParameter(split_arg, diag, &options_.split_paths.back(),
459         &options_.split_constraints.back())) {
460       return 1;
461     }
462   }
463 
464   if (options_.table_flattener_options.collapse_key_stringpool) {
465     if (whitelist_path_) {
466       std::string& path = whitelist_path_.value();
467       if (!ExtractObfuscationWhitelistFromConfig(path, &context, &options_)) {
468         return 1;
469       }
470     }
471   }
472 
473   if (resources_config_path_) {
474     std::string& path = resources_config_path_.value();
475     if (!ExtractConfig(path, &context, &options_)) {
476       return 1;
477     }
478   }
479 
480   if (!ExtractAppDataFromManifest(&context, apk.get(), &options_)) {
481     return 1;
482   }
483 
484   Optimizer cmd(&context, options_);
485   return cmd.Run(std::move(apk));
486 }
487 
488 }  // namespace aapt
489