1#!/bin/bash
2#
3# Copyright (C) 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# Note: Requires $ANDROID_BUILD_TOP/build/envsetup.sh to have been run.
18#
19# This script takes in a logcat containing Sanitizer traces and outputs several
20# files, prints information regarding the traces, and plots information as well.
21ALL_PIDS=false
22USE_TEMP=true
23DO_REDO=false
24PACKAGE_NAME=""
25BAKSMALI_NUM=0
26# EXACT_ARG and MIN_ARG are passed to prune_sanitizer_output.py
27EXACT_ARG=""
28MIN_ARG=()
29OFFSET_ARGS=()
30TIME_ARGS=()
31usage() {
32  echo "Usage: $0 [options] [LOGCAT_FILE] [CATEGORIES...]"
33  echo "    -a"
34  echo "        Forces all pids associated with registered dex"
35  echo "        files in the logcat to be processed."
36  echo "        default: only the last pid is processed"
37  echo
38  echo "    -b  [DEX_FILE_NUMBER]"
39  echo "        Outputs data for the specified baksmali"
40  echo "        dump if -p is provided."
41  echo "        default: first baksmali dump in order of dex"
42  echo "          file registration"
43  echo
44  echo "    -d  OUT_DIRECTORY"
45  echo "        Puts all output in specified directory."
46  echo "        If not given, output will be put in a local"
47  echo "        temp folder which will be deleted after"
48  echo "        execution."
49  echo
50  echo "    -e"
51  echo "        All traces will have exactly the same number"
52  echo "        of categories which is specified by either"
53  echo "        the -m argument or by prune_sanitizer_output.py"
54  echo
55  echo "    -f"
56  echo "        Forces redo of all commands even if output"
57  echo "        files exist. Steps are skipped if their output"
58  echo "        exist already and this is not enabled."
59  echo
60  echo "    -m  [MINIMUM_CALLS_PER_TRACE]"
61  echo "        Filters out all traces that do not have"
62  echo "        at least MINIMUM_CALLS_PER_TRACE lines."
63  echo "        default: specified by prune_sanitizer_output.py"
64  echo
65  echo "    -o  [OFFSET],[OFFSET]"
66  echo "        Filters out all Dex File offsets outside the"
67  echo "        range between provided offsets. 'inf' can be"
68  echo "        provided for infinity."
69  echo "        default: 0,inf"
70  echo
71  echo "    -p  [PACKAGE_NAME]"
72  echo "        Using the package name, uses baksmali to get"
73  echo "        a dump of the Dex File format for the package."
74  echo
75  echo "    -t  [TIME_OFFSET],[TIME_OFFSET]"
76  echo "        Filters out all time offsets outside the"
77  echo "        range between provided offsets. 'inf' can be"
78  echo "        provided for infinity."
79  echo "        default: 0,inf"
80  echo
81  echo "    CATEGORIES are words that are expected to show in"
82  echo "       a large subset of symbolized traces. Splits"
83  echo "       output based on each word."
84  echo
85  echo "    LOGCAT_FILE is the piped output from adb logcat."
86  echo
87}
88
89
90while getopts ":ab:d:efm:o:p:t:" opt ; do
91case ${opt} in
92  a)
93    ALL_PIDS=true
94    ;;
95  b)
96    if ! [[ "$OPTARG" -eq "$OPTARG" ]]; then
97      usage
98      exit
99    fi
100    BAKSMALI_NUM=$OPTARG
101    ;;
102  d)
103    USE_TEMP=false
104    OUT_DIR=$OPTARG
105    ;;
106  e)
107    EXACT_ARG='-e'
108    ;;
109  f)
110    DO_REDO=true
111    ;;
112  m)
113    if ! [[ "$OPTARG" -eq "$OPTARG" ]]; then
114      usage
115      exit
116    fi
117    MIN_ARG=( "-m" "$OPTARG" )
118    ;;
119  o)
120    set -f
121    old_ifs=$IFS
122    IFS=","
123    OFFSET_ARGS=( $OPTARG )
124    if [[ "${#OFFSET_ARGS[@]}" -ne 2 ]]; then
125      usage
126      exit
127    fi
128    OFFSET_ARGS=( "--offsets" "${OFFSET_ARGS[@]}" )
129    IFS=$old_ifs
130    set +f
131    ;;
132  t)
133    set -f
134    old_ifs=$IFS
135    IFS=","
136    TIME_ARGS=( $OPTARG )
137    if [[ "${#TIME_ARGS[@]}" -ne 2 ]]; then
138      usage
139      exit
140    fi
141    TIME_ARGS=( "--times" "${TIME_ARGS[@]}" )
142    IFS=$old_ifs
143    set +f
144    ;;
145  p)
146    PACKAGE_NAME=$OPTARG
147    ;;
148  \?)
149    usage
150    exit
151esac
152done
153shift $((OPTIND -1))
154
155if [[ $# -lt 1 ]]; then
156  usage
157  exit
158fi
159
160LOGCAT_FILE=$1
161NUM_CAT=$(($# - 1))
162
163# Use a temp directory that will be deleted
164if [[ $USE_TEMP = true ]]; then
165  OUT_DIR=$(mktemp -d --tmpdir="$PWD")
166  DO_REDO=true
167fi
168
169if [[ ! -d "$OUT_DIR" ]]; then
170  mkdir "$OUT_DIR"
171  DO_REDO=true
172fi
173
174# Note: Steps are skipped if their output exists until -f flag is enabled
175echo "Output folder: $OUT_DIR"
176# Finds the lines matching pattern criteria and prints out unique instances of
177# the 3rd word (PID)
178unique_pids=( $(awk '/RegisterDexFile:/ && !/zygote/ {if(!a[$3]++) print $3}' \
179  "$LOGCAT_FILE") )
180echo "List of pids: ${unique_pids[@]}"
181if [[ $ALL_PIDS = false ]]; then
182  unique_pids=( ${unique_pids[-1]} )
183fi
184
185for pid in "${unique_pids[@]}"
186do
187  echo
188  echo "Current pid: $pid"
189  echo
190  pid_dir=$OUT_DIR/$pid
191  if [[ ! -d "$pid_dir" ]]; then
192    mkdir "$pid_dir"
193    DO_REDO[$pid]=true
194  fi
195
196  intermediates_dir=$pid_dir/intermediates
197  results_dir=$pid_dir/results
198  logcat_pid_file=$pid_dir/logcat
199
200  if [[ ! -f "$logcat_pid_file" ]] || \
201     [[ "${DO_REDO[$pid]}" = true ]] || \
202     [[ $DO_REDO = true ]]; then
203    DO_REDO[$pid]=true
204    awk "{if(\$3 == $pid) print \$0}" "$LOGCAT_FILE" > "$logcat_pid_file"
205  fi
206
207  if [[ ! -d "$intermediates_dir" ]]; then
208    mkdir "$intermediates_dir"
209    DO_REDO[$pid]=true
210  fi
211
212  # Step 1 - Only output lines related to Sanitizer
213  # Folder that holds all file output
214  asan_out=$intermediates_dir/asan_output
215  if [[ ! -f "$asan_out" ]] || \
216     [[ "${DO_REDO[$pid]}" = true ]] || \
217     [[ $DO_REDO = true ]]; then
218    DO_REDO[$pid]=true
219    echo "Extracting ASAN output"
220    grep "app_process64" "$logcat_pid_file" > "$asan_out"
221  else
222    echo "Skipped: Extracting ASAN output"
223  fi
224
225  # Step 2 - Only output lines containing Dex File Start Addresses
226  dex_start=$intermediates_dir/dex_start
227  if [[ ! -f "$dex_start" ]] || \
228     [[ "${DO_REDO[$pid]}" = true ]] || \
229     [[ $DO_REDO = true ]]; then
230    DO_REDO[$pid]=true
231    echo "Extracting Start of Dex File(s)"
232    if [[ ! -z "$PACKAGE_NAME" ]]; then
233      awk '/RegisterDexFile:/ && /'"$PACKAGE_NAME"'/ && /\/data\/app/' \
234        "$logcat_pid_file" > "$dex_start"
235    else
236      grep "RegisterDexFile:" "$logcat_pid_file" > "$dex_start"
237    fi
238  else
239    echo "Skipped: Extracting Start of Dex File(s)"
240  fi
241
242  # Step 3 - Clean Sanitizer output from Step 2 since logcat cannot
243  # handle large amounts of output.
244  asan_out_filtered=$intermediates_dir/asan_output_filtered
245  if [[ ! -f "$asan_out_filtered" ]] || \
246     [[ "${DO_REDO[$pid]}" = true ]] || \
247     [[ $DO_REDO = true ]]; then
248    DO_REDO[$pid]=true
249    echo "Filtering/Cleaning ASAN output"
250    python "$ANDROID_BUILD_TOP"/art/tools/runtime_memusage/prune_sanitizer_output.py \
251      "$EXACT_ARG" "${MIN_ARG[@]}" -d "$intermediates_dir" "$asan_out"
252  else
253    echo "Skipped: Filtering/Cleaning ASAN output"
254  fi
255
256  # Step 4 - Retrieve symbolized stack traces from Step 3 output
257  sym_filtered=$intermediates_dir/sym_filtered
258  if [[ ! -f "$sym_filtered" ]] || \
259     [[ "${DO_REDO[$pid]}" = true ]] || \
260     [[ $DO_REDO = true ]]; then
261    DO_REDO[$pid]=true
262    echo "Retrieving symbolized traces"
263    "$ANDROID_BUILD_TOP"/development/scripts/stack "$asan_out_filtered" \
264      > "$sym_filtered"
265  else
266    echo "Skipped: Retrieving symbolized traces"
267  fi
268
269  # Step 4.5 - Obtain Dex File Format of dex file related to package
270  filtered_dex_start=$intermediates_dir/filtered_dex_start
271  baksmali_dmp_ctr=0
272  baksmali_dmp_prefix=$intermediates_dir"/baksmali_dex_file_"
273  baksmali_dmp_files=( $baksmali_dmp_prefix* )
274  baksmali_dmp_arg="--dex-file "${baksmali_dmp_files[$BAKSMALI_NUM]}
275  apk_dex_files=( )
276  if [[ ! -f "$baksmali_dmp_prefix""$BAKSMALI_NUM" ]] || \
277     [[ ! -f "$filtered_dex_start" ]] || \
278     [[ "${DO_REDO[$pid]}" = true ]] || \
279     [[ $DO_REDO = true ]]; then
280    if [[ ! -z "$PACKAGE_NAME" ]]; then
281      DO_REDO[$pid]=true
282      # Extracting Dex File path on device from Dex File related to package
283      apk_directory=$(dirname "$(tail -n1 "$dex_start" | awk "{print \$8}")")
284      for dex_file in $(awk "{print \$8}" "$dex_start"); do
285        apk_dex_files+=( $(basename "$dex_file") )
286      done
287      apk_oat_files=$(adb shell find "$apk_directory" -name "*.?dex" -type f \
288        2> /dev/null)
289      # Pulls the .odex and .vdex files associated with the package
290      for apk_file in $apk_oat_files; do
291        base_name=$(basename "$apk_file")
292        adb pull "$apk_file" "$intermediates_dir/base.${base_name#*.}"
293      done
294      oatdump --oat-file="$intermediates_dir"/base.odex \
295        --export-dex-to="$intermediates_dir" --output=/dev/null
296      for dex_file in "${apk_dex_files[@]}"; do
297        exported_dex_file=$intermediates_dir/$dex_file"_export.dex"
298        baksmali_dmp_out="$baksmali_dmp_prefix""$((baksmali_dmp_ctr++))"
299        baksmali -JXmx1024M dump "$exported_dex_file" \
300          > "$baksmali_dmp_out" 2> "$intermediates_dir"/error
301        if ! [[ -s "$baksmali_dmp_out" ]]; then
302          rm "$baksmali_dmp_prefix"*
303          baksmali_dmp_arg=""
304          echo "Failed to retrieve Dex File format"
305          break
306        fi
307      done
308      baksmali_dmp_files=( "$baksmali_dmp_prefix"* )
309      baksmali_dmp_arg="--dex-file "${baksmali_dmp_files[$BAKSMALI_NUM]}
310      # Gets the baksmali dump associated with BAKSMALI_NUM
311      awk "NR == $((BAKSMALI_NUM + 1))" "$dex_start" > "$filtered_dex_start"
312      results_dir=$results_dir"_"$BAKSMALI_NUM
313      echo "Skipped: Retrieving Dex File format from baksmali; no package given"
314    else
315      cp "$dex_start" "$filtered_dex_start"
316      baksmali_dmp_arg=""
317    fi
318  else
319    awk "NR == $((BAKSMALI_NUM + 1))" "$dex_start" > "$filtered_dex_start"
320    results_dir=$results_dir"_"$BAKSMALI_NUM
321    echo "Skipped: Retrieving Dex File format from baksmali"
322  fi
323
324  if [[ ! -d "$results_dir" ]]; then
325    mkdir "$results_dir"
326    DO_REDO[$pid]=true
327  fi
328
329  # Step 5 - Using Steps 2, 3, 4 outputs in order to output graph data
330  # and trace data
331  # Only the category names are needed for the commands giving final output
332  shift
333  time_output=($results_dir/time_output_*.dat)
334  if [[ ! -e ${time_output[0]} ]] || \
335     [[ "${DO_REDO[$pid]}" = true ]] || \
336     [[ $DO_REDO = true ]]; then
337    DO_REDO[$pid]=true
338    echo "Creating Categorized Time Table"
339    baksmali_dmp_args=( $baksmali_dmp_arg )
340    python "$ANDROID_BUILD_TOP"/art/tools/runtime_memusage/symbol_trace_info.py \
341      -d "$results_dir" "${OFFSET_ARGS[@]}" "${baksmali_dmp_args[@]}" \
342      "${TIME_ARGS[@]}" "$asan_out_filtered" "$sym_filtered" \
343      "$filtered_dex_start" "$@"
344  else
345    echo "Skipped: Creating Categorized Time Table"
346  fi
347
348  # Step 6 - Use graph data from Step 5 to plot graph
349  # Contains the category names used for legend of gnuplot
350  plot_cats="\"Uncategorized $*\""
351  package_string=""
352  dex_name=""
353  if [[ ! -z "$PACKAGE_NAME" ]]; then
354    package_string="Package name: $PACKAGE_NAME "
355  fi
356  if [[ ! -z "$baksmali_dmp_arg" ]]; then
357    dex_file_path="$(awk "{print \$8}" "$filtered_dex_start" | tail -n1)"
358    dex_name="Dex File name: $(basename "$dex_file_path") "
359  fi
360  echo "Plotting Categorized Time Table"
361  # Plots the information from logcat
362  gnuplot --persist -e \
363    'filename(n) = sprintf("'"$results_dir"'/time_output_%d.dat", n);
364     catnames = '"$plot_cats"';
365     set title "'"$package_string""$dex_name"'PID: '"$pid"'";
366     set xlabel "Time (milliseconds)";
367     set ylabel "Dex File Offset (bytes)";
368     plot for [i=0:'"$NUM_CAT"'] filename(i) using 1:2 title word(catnames, i + 1);'
369
370  if [[ $USE_TEMP = true ]]; then
371    echo "Removing temp directory and files"
372    rm -rf "$OUT_DIR"
373  fi
374done
375