1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.dogfood;
18 
19 import android.app.NotificationChannel;
20 import android.app.NotificationManager;
21 import android.app.job.JobParameters;
22 import android.app.job.JobScheduler;
23 import android.app.job.JobService;
24 import android.content.SharedPreferences;
25 import android.util.Log;
26 
27 import androidx.core.app.NotificationCompat;
28 import androidx.core.app.NotificationManagerCompat;
29 
30 import com.android.nn.benchmark.core.BenchmarkResult;
31 import com.android.nn.benchmark.core.Processor;
32 import com.android.nn.benchmark.core.TestModels;
33 
34 import java.util.List;
35 import java.util.Random;
36 import java.util.concurrent.ExecutorService;
37 import java.util.concurrent.Executors;
38 
39 /** Regularly runs a random selection of the NN API benchmark models */
40 public class BenchmarkJobService extends JobService implements Processor.Callback {
41 
42     private static final String TAG = "NN_BENCHMARK";
43     private static final String CHANNEL_ID = "default";
44     private static final int NOTIFICATION_ID = 999;
45     public static final int JOB_ID = 1;
46     private NotificationManagerCompat mNotificationManager;
47     private NotificationCompat.Builder mNotification;
48     private boolean mJobStopped = false;
49     private static final int NUM_RUNS = 10;
50     private Processor mProcessor;
51     private JobParameters mJobParameters;
52     private static final String NN_API_DOGFOOD_PREF = "nn_api_dogfood";
53 
54     private static int DOGFOOD_MODELS_PER_RUN = 20;
55     private BenchmarkResult mTestResults[];
56     private final ExecutorService processorRunner = Executors.newSingleThreadExecutor();
57 
58 
59     @Override
onStartJob(JobParameters jobParameters)60     public boolean onStartJob(JobParameters jobParameters) {
61         mJobParameters = jobParameters;
62         incrementNumRuns();
63         Log.d(TAG, String.format("NN API Benchmarking job %d/%d started", getNumRuns(), NUM_RUNS));
64         showNotification();
65         doBenchmark();
66 
67         return true;
68     }
69 
70     @Override
onStopJob(JobParameters jobParameters)71     public boolean onStopJob(JobParameters jobParameters) {
72         Log.d(TAG, String.format("NN API Benchmarking job %d/%d stopped", getNumRuns(), NUM_RUNS));
73         mJobStopped = true;
74 
75         return false;
76     }
77 
doBenchmark()78     public void doBenchmark() {
79         mProcessor = new Processor(this, this, randomModelList());
80         mProcessor.setUseNNApi(true);
81         mProcessor.setToggleLong(true);
82         mProcessor.setMaxRunIterations(1);
83         processorRunner.submit(mProcessor);
84     }
85 
onBenchmarkFinish(boolean ok)86     public void onBenchmarkFinish(boolean ok) {
87         mProcessor.exit();
88         if (getNumRuns() >= NUM_RUNS) {
89             mNotification
90                     .setProgress(0, 0, false)
91                     .setContentText(
92                             "Benchmarking done! please upload a bug report via BetterBug under"
93                                 + " Android > Android OS & > Apps Runtime > Machine Learning")
94                     .setOngoing(false);
95             mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
96             JobScheduler jobScheduler = getSystemService(JobScheduler.class);
97             jobScheduler.cancel(JOB_ID);
98             resetNumRuns();
99         } else {
100             mNotification
101                     .setProgress(0, 0, false)
102                     .setContentText(
103                             String.format(
104                                     "Background test %d of %d is complete", getNumRuns(), NUM_RUNS))
105                     .setOngoing(false);
106             mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
107         }
108 
109         Log.d(TAG, "NN API Benchmarking job finished");
110         jobFinished(mJobParameters, false);
111     }
112 
onStatusUpdate(int testNumber, int numTests, String modelName)113     public void onStatusUpdate(int testNumber, int numTests, String modelName) {
114         Log.d(
115                 TAG,
116                 String.format("Benchmark progress %d of %d - %s", testNumber, numTests, modelName));
117         mNotification.setProgress(numTests, testNumber, false);
118         mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
119     }
120 
showNotification()121     private void showNotification() {
122         mNotificationManager = NotificationManagerCompat.from(this);
123         NotificationChannel channel =
124                 new NotificationChannel(CHANNEL_ID, "Default", NotificationManager.IMPORTANCE_LOW);
125         // mNotificationManager.createNotificationChannel(channel);
126         mNotificationManager = NotificationManagerCompat.from(this);
127         String title = "NN API Dogfood";
128         String msg = String.format("Background test %d of %d is running", getNumRuns(), NUM_RUNS);
129 
130         mNotification =
131                 new NotificationCompat.Builder(this, CHANNEL_ID)
132                         .setSmallIcon(R.mipmap.ic_launcher)
133                         .setContentTitle(title)
134                         .setContentText("NN API Benchmarking Job")
135                         .setPriority(NotificationCompat.PRIORITY_MAX)
136                         .setOngoing(true);
137         mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
138     }
139 
randomModelList()140     private int[] randomModelList() {
141         long seed = System.currentTimeMillis();
142         List<TestModels.TestModelEntry> testList = TestModels.modelsList();
143 
144         Log.v(TAG, "Dogfood run seed " + seed);
145         Random random = new Random(seed);
146         int numModelsToSelect = Math.min(DOGFOOD_MODELS_PER_RUN, testList.size());
147         int[] randomModelIndices = new int[numModelsToSelect];
148 
149         for (int i = 0; i < numModelsToSelect; i++) {
150             randomModelIndices[i] = random.nextInt(testList.size());
151         }
152 
153         return randomModelIndices;
154     }
155 
incrementNumRuns()156     private void incrementNumRuns() {
157         SharedPreferences.Editor editor =
158                 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit();
159         editor.putInt("num_runs", getNumRuns() + 1);
160         editor.apply();
161     }
162 
resetNumRuns()163     private void resetNumRuns() {
164         SharedPreferences.Editor editor =
165                 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit();
166         editor.putInt("num_runs", 0);
167         editor.apply();
168     }
169 
getNumRuns()170     private int getNumRuns() {
171         SharedPreferences prefs = getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE);
172         return prefs.getInt("num_runs", 0);
173     }
174 }
175