1#!/bin/bash
2#
3# Build benchmark app and run it, mimicking a user-initiated run
4#
5# Output is logged to a temporary folder and summarized in txt and JSON formats.
6# parallel-inference-stress tests produce no output except for the success or failure notification,
7# which is not logged.
8
9
10OPTS="$(getopt -o f:rb -l filter-driver:,include-nnapi-reference,nnapi-reference-only,skip-build -- "$@")"
11
12if [ $? -ne 0 ]; then
13    echo "Invalid arguments, accepted options are"
14    echo " -f <regex> | --filter-driver <regex> : to run crash tests only on the drivers (ignoring nnapi-reference) matching the specified regular expression"
15    echo " -r | --include-nnapi-reference : to include nnapi-reference in target drivers"
16    echo " --nnapi-reference-only : to run tests only vs nnapi-reference"
17    echo " -b | --skip-build : skip build and installation of tests"
18    exit
19fi
20
21eval set -- "$OPTS"
22
23DRIVER_FILTER_OPT=""
24INCLUDE_NNAPI_REF_OPT=""
25BUILD_AND_INSTALL=true
26while [ $# -gt 0 ] ; do
27  case "$1" in
28    -f|--filter-driver)
29      DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter $2"
30      shift 2
31      ;;
32    -r|--include-nnapi-reference)
33      INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true"
34      shift
35      ;;
36    --nnapi-reference-only)
37      DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter no-device"
38      INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true"
39      shift
40      ;;
41    -b|--skip-build)
42      BUILD_AND_INSTALL=false
43      shift
44      ;;
45    --)
46      shift
47      break
48      ;;
49    *)
50      echo "Unsupported arg $1"
51      exit 1
52  esac
53done
54
55MODE="${1:-scoring}"
56INSTALL_NATIVE_TESTS=false
57CRASH_TEST_APP="NeuralNetworksApiCrashTest"
58APP="NeuralNetworksApiBenchmark"
59case "$MODE" in
60  scoring)
61    CLASS=com.android.nn.benchmark.app.NNScoringTest
62    ;;
63  inference-stress)
64    CLASS=com.android.nn.benchmark.app.NNInferenceStressTest
65    ;;
66  model-loading-stress)
67    CLASS=com.android.nn.benchmark.app.NNModelLoadingStressTest
68    ;;
69  parallel-inference-stress)
70    CLASS=com.android.nn.crashtest.app.NNParallelCrashResistantInferenceTest
71    APP="$CRASH_TEST_APP"
72    ;;
73  parallel-inference-stress-in-process)
74    CLASS=com.android.nn.crashtest.app.NNParallelInProcessInferenceTest
75    APP="$CRASH_TEST_APP"
76    ;;
77  client-early-termination-stress)
78    CLASS=com.android.nn.crashtest.app.NNClientEarlyTerminationTest
79    APP="$CRASH_TEST_APP"
80    ;;
81  multi-process-inference-stress)
82    CLASS=com.android.nn.crashtest.app.NNMultipleProcessInferenceTest
83    APP="$CRASH_TEST_APP"
84    INSTALL_NATIVE_TESTS=true
85    ;;
86  multi-process-model-load-stress)
87    CLASS=com.android.nn.crashtest.app.NNMultipleProcessModelLoadTest
88    APP="$CRASH_TEST_APP"
89    INSTALL_NATIVE_TESTS=true
90    ;;
91  memory-mapped-model-load-stress)
92    CLASS=com.android.nn.crashtest.app.NNMemoryMappedModelCompilationTest
93    APP="$CRASH_TEST_APP"
94    ;;
95  model-load-random-stress)
96    APP="$CRASH_TEST_APP"
97    CLASS=com.android.nn.crashtest.app.NNRandomGraphLoadTest
98    ;;
99  inference-random-stress)
100    APP="$CRASH_TEST_APP"
101    CLASS=com.android.nn.crashtest.app.NNRandomGraphExecutionTest
102    ;;
103  performance-degradation-stress)
104    APP="$CRASH_TEST_APP"
105    CLASS=com.android.nn.crashtest.app.NNPerformanceDegradationTest
106    ;;
107  *)
108    echo "Unknown execution mode: $1"
109    echo "Known modes: scoring (default), inference-stress, model-loading-stress, " \
110      "parallel-inference-stress, parallel-inference-stress-in-process, " \
111      "client-early-termination-stress, multi-process-inference-stress, " \
112      "multi-process-model-load-stress memory-mapped-model-load-stress, " \
113      "model-load-random-stress, inference-random-stress, performance-degradation-stress"
114    exit 1
115    ;;
116esac
117
118if [[ -z "$ANDROID_BUILD_TOP" ]]; then
119  echo ANDROID_BUILD_TOP not set, bailing out
120  echo you must run lunch before running this script
121  exit 1
122fi
123
124set -e
125cd $ANDROID_BUILD_TOP
126
127if [ "$BUILD_AND_INSTALL" = true ]; then
128  # Build and install benchmark app
129  TMPFILE=$(mktemp)
130  build/soong/soong_ui.bash --make-mode ${APP} 2>&1 | tee ${TMPFILE}
131  TARGET_ARCH=$(cat ${TMPFILE} | grep TARGET_ARCH= | sed -e 's/TARGET_ARCH=//')
132  if [ "${TARGET_ARCH}" = "aarch64" ]; then
133      APK_DIR=arm64
134  else
135      APK_DIR=${TARGET_ARCH}
136  fi
137  if ! adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk; then
138    adb uninstall com.android.nn.benchmark.app
139    adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk
140  fi
141
142  if [ "$INSTALL_NATIVE_TESTS" = true ]; then
143    build/soong/soong_ui.bash --make-mode nn_stress_test
144    adb push $OUT/system/bin/nn_stress_test /bin/
145  fi
146fi
147
148# Should we figure out if we run on release device
149if [ -z "$MLTS_RELEASE_DEVICE" ]; then
150  BUILD_DESCRIPTION=`adb shell getprop ro.build.description`
151  if [[ $BUILD_DESCRIPTION =~ .*release.* ]]
152  then
153    MLTS_RELEASE_DEVICE=True
154  else
155    MLTS_RELEASE_DEVICE=False
156  fi
157fi
158
159# Pass --no-isolated-storage to am instrument?
160BUILD_VERSION_RELEASE=`adb shell getprop ro.build.version.release`
161AM_INSTRUMENT_FLAGS="$DRIVER_FILTER_OPT $INCLUDE_NNAPI_REF_OPT"
162if [[ $BUILD_VERSION_RELEASE == "Q" ]]; then
163  AM_INSTRUMENT_FLAGS+=" --no-isolated-storage"
164fi
165
166if [[ "$MODE" == "scoring" ]]; then
167  if [[ "$MLTS_RELEASE_DEVICE" == "True" ]]; then
168    TEST_EXTENRAL_STORAGE="com.android.nn.benchmark.app/com.android.nn.benchmark.util.TestExternalStorageActivity"
169    while ! adb shell "am start -W $TEST_EXTENRAL_STORAGE && rm /sdcard/mlts_write_external_storage" > /dev/null 2>&1; do
170       echo "************************************************************"
171       echo "Grant External storage write permissions to MLTS to proceed!"
172       echo "************************************************************"
173       read -n 1 -r -p "Continue? (press any key)"
174       echo
175    done
176  else
177    adb root
178    adb shell "pm grant com.android.nn.benchmark.app android.permission.WRITE_EXTERNAL_STORAGE"
179    # Skip setup wizard and remount (read-write)
180    if ! adb shell test -f /data/local.prop; then
181      adb shell 'echo ro.setupwizard.mode=DISABLED > /data/local.prop'
182      adb shell 'chmod 644 /data/local.prop'
183      adb shell 'settings put global device_provisioned 1*'
184      adb shell 'settings put secure user_setup_complete 1'
185      adb disable-verity
186      adb reboot
187      sleep 5
188      adb wait-for-usb-device root
189      adb wait-for-usb-device remount
190      sleep 5
191    fi
192    set +e
193    # Enable menu key press through adb
194    adb shell 'echo testing > /data/local/enable_menu_key'
195    # Leave screen on (affects scheduling)
196    adb shell settings put system screen_off_timeout 86400000
197    # Stop background apps, seem to take ~10% CPU otherwise
198    adb shell 'pm disable com.google.android.googlequicksearchbox'
199    adb shell 'pm list packages -f' | sed -e 's/.*=//' | sed 's/\r//g' | grep "com.breel.wallpapers" | while read pkg; do adb shell "pm disable $pkg"; done;
200    set -e
201  fi
202fi
203
204adb shell setprop debug.nn.cpuonly 0
205adb shell setprop debug.nn.vlog "''"
206
207# Menukey - make sure screen is on
208adb shell "input keyevent 82"
209# Show homescreen
210adb shell wm dismiss-keyguard
211
212if [[ "$MODE" == "scoring" ]]; then
213  LOGDIR=$(mktemp -d)/mlts-logs
214  HOST_CSV=$LOGDIR/benchmark.csv
215  RESULT_HTML=$LOGDIR/result.html
216  DEVICE_CSV=/sdcard/mlts_benchmark.csv
217
218  mkdir -p $LOGDIR
219  echo Creating logs in $LOGDIR
220
221  # Remove old benchmark csv data
222  adb shell rm -f ${DEVICE_CSV}
223fi
224
225# Set the shell pid as a top-app and run tests
226time adb shell "echo $$ > /dev/stune/top-app/tasks; am instrument ${AM_INSTRUMENT_FLAGS} -w -e class $CLASS com.android.nn.benchmark.app/androidx.test.runner.AndroidJUnitRunner"
227
228if [[ "$MODE" == "scoring" ]]; then
229  adb pull $DEVICE_CSV $HOST_CSV
230  echo Benchmark data saved in $HOST_CSV
231
232  $ANDROID_BUILD_TOP/test/mlts/benchmark/results/generate_result.py $HOST_CSV $RESULT_HTML
233  echo Results stored  in $RESULT_HTML
234  xdg-open $RESULT_HTML
235fi
236