1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.car.assist.client;
18 
19 
20 import android.annotation.Nullable;
21 import android.app.ActivityManager;
22 import android.app.Notification;
23 import android.app.Notification.MessagingStyle.Message;
24 import android.app.PendingIntent;
25 import android.app.Person;
26 import android.content.Context;
27 import android.content.Intent;
28 import android.os.Parcelable;
29 import android.service.notification.StatusBarNotification;
30 import android.util.Log;
31 import android.widget.Toast;
32 
33 import androidx.core.app.NotificationCompat;
34 
35 import com.android.car.assist.client.tts.TextToSpeechHelper;
36 
37 import java.util.ArrayList;
38 import java.util.Collections;
39 import java.util.HashMap;
40 import java.util.List;
41 import java.util.Map;
42 
43 /**
44  * Handles Assistant request fallbacks in the case that Assistant cannot fulfill the request for
45  * any given reason.
46  * <p/>
47  * Simply reads out the notification messages for read requests, and speaks out
48  * an error message for other requests.
49  */
50 public class FallbackAssistant {
51 
52     private static final String TAG = FallbackAssistant.class.getSimpleName();
53 
54     private final Context mContext;
55     private final TextToSpeechHelper mTextToSpeechHelper;
56     private final RequestIdGenerator mRequestIdGenerator;
57     private Map<Long, ActionRequestInfo> mRequestIdToActionRequestInfo = new HashMap<>();
58     // String that means "says", to be used when reading out a message (i.e. <Sender> says
59     // <Message).
60     private final String mVerbForSays;
61 
62     private final TextToSpeechHelper.Listener mListener = new TextToSpeechHelper.Listener() {
63         @Override
64         public void onTextToSpeechStarted(long requestId) {
65             if (Log.isLoggable(TAG, Log.DEBUG)) {
66                 Log.d(TAG, "onTextToSpeechStarted");
67             }
68         }
69 
70         @Override
71         public void onTextToSpeechStopped(long requestId, boolean error) {
72             if (Log.isLoggable(TAG, Log.DEBUG)) {
73                 Log.d(TAG, "onTextToSpeechStopped");
74             }
75 
76             if (error) {
77                 Toast.makeText(mContext, mContext.getString(R.string.assist_action_failed_toast),
78                         Toast.LENGTH_LONG).show();
79             }
80             finishAction(requestId, error);
81         }
82     };
83 
84     /** Listener to allow clients to be alerted when their requested message has been read. **/
85     public interface Listener {
86         /**
87          * Called after the TTS engine has finished reading aloud the message.
88          */
onMessageRead(boolean hasError)89         void onMessageRead(boolean hasError);
90     }
91 
FallbackAssistant(Context context)92     public FallbackAssistant(Context context) {
93         mContext = context;
94         mTextToSpeechHelper = new TextToSpeechHelper(context, mListener);
95         mRequestIdGenerator = new RequestIdGenerator();
96         mVerbForSays = mContext.getString(R.string.says);
97     }
98 
99     /**
100      * Handles a fallback read action by reading all messages in the notification.
101      *
102      * @param sbn the payload notification from which to extract messages from
103      */
handleReadAction(StatusBarNotification sbn, Listener listener)104     public void handleReadAction(StatusBarNotification sbn, Listener listener) {
105         if (mTextToSpeechHelper.isSpeaking()) {
106             mTextToSpeechHelper.requestStop();
107         }
108 
109         Parcelable[] messagesBundle = sbn.getNotification().extras
110                 .getParcelableArray(Notification.EXTRA_MESSAGES);
111 
112         if (messagesBundle == null || messagesBundle.length == 0) {
113             listener.onMessageRead(/* hasError= */ true);
114             return;
115         }
116 
117         List<CharSequence> messages = new ArrayList<>();
118 
119         List<Message> messageList = Message.getMessagesFromBundleArray(messagesBundle);
120         if (messageList == null || messageList.isEmpty()) {
121             Log.w(TAG, "No messages could be extracted from the bundle");
122             listener.onMessageRead(/* hasError= */ true);
123             return;
124         }
125 
126         Person previousSender = messageList.get(0).getSenderPerson();
127         if (previousSender != null) {
128             messages.add(previousSender.getName());
129             messages.add(mVerbForSays);
130         }
131         for (Message message : messageList) {
132             if (!message.getSenderPerson().equals(previousSender)) {
133                 messages.add(message.getSenderPerson().getName());
134                 messages.add(mVerbForSays);
135                 previousSender = message.getSenderPerson();
136             }
137             messages.add(message.getText());
138         }
139 
140         long requestId = mRequestIdGenerator.generateRequestId();
141 
142         if (mTextToSpeechHelper.requestPlay(messages, requestId)) {
143             if (Log.isLoggable(TAG, Log.DEBUG)) {
144                 Log.d(TAG, "Requesting TTS to read message with requestId: " + requestId);
145             }
146             mRequestIdToActionRequestInfo.put(requestId, new ActionRequestInfo(sbn, listener));
147         } else {
148             listener.onMessageRead(/* hasError= */ true);
149         }
150     }
151 
152     /**
153      * Handles generic (non-read) actions by reading out an error message.
154      *
155      * @param errorMessage the error message to read out
156      */
handleErrorMessage(CharSequence errorMessage, Listener listener)157     public void handleErrorMessage(CharSequence errorMessage, Listener listener) {
158         if (mTextToSpeechHelper.isSpeaking()) {
159             mTextToSpeechHelper.requestStop();
160         }
161 
162         long requestId = mRequestIdGenerator.generateRequestId();
163         if (mTextToSpeechHelper.requestPlay(Collections.singletonList(errorMessage),
164                 requestId)) {
165             if (Log.isLoggable(TAG, Log.DEBUG)) {
166                 Log.d(TAG, "Requesting TTS to read error with requestId: " + requestId);
167             }
168             mRequestIdToActionRequestInfo.put(requestId, new ActionRequestInfo(
169                     /* statusBarNotification= */ null,
170                     listener));
171         } else {
172             listener.onMessageRead(/* hasError= */ true);
173         }
174     }
175 
finishAction(long requestId, boolean hasError)176     private void finishAction(long requestId, boolean hasError) {
177         if (!mRequestIdToActionRequestInfo.containsKey(requestId)) {
178             Log.w(TAG, "No actionRequestInfo found for requestId: " + requestId);
179             return;
180         }
181 
182         ActionRequestInfo info = mRequestIdToActionRequestInfo.remove(requestId);
183 
184         if (info.getStatusBarNotification() != null && !hasError) {
185             sendMarkAsReadIntent(info.getStatusBarNotification());
186         }
187 
188         info.getListener().onMessageRead(hasError);
189     }
190 
sendMarkAsReadIntent(StatusBarNotification sbn)191     private void sendMarkAsReadIntent(StatusBarNotification sbn) {
192         NotificationCompat.Action markAsReadAction = CarAssistUtils.getMarkAsReadAction(
193                 sbn.getNotification());
194         boolean isDebugLoggable = Log.isLoggable(TAG, Log.DEBUG);
195 
196         if (markAsReadAction != null) {
197             if (sendPendingIntent(markAsReadAction.getActionIntent(),
198                     null /* resultIntent */) != ActivityManager.START_SUCCESS
199                     && isDebugLoggable) {
200                 Log.d(TAG, "Could not relay mark as read event to the messaging app.");
201             }
202         } else if (isDebugLoggable) {
203             Log.d(TAG, "Car compat message notification has no mark as read action: "
204                     + sbn.getKey());
205         }
206     }
207 
sendPendingIntent(PendingIntent pendingIntent, Intent resultIntent)208     private int sendPendingIntent(PendingIntent pendingIntent, Intent resultIntent) {
209         try {
210             return pendingIntent.sendAndReturnResult(/* context= */ mContext, /* code= */ 0,
211                     /* intent= */ resultIntent, /* onFinished= */null,
212                     /* handler= */ null, /* requiredPermissions= */ null,
213                     /* options= */ null);
214         } catch (PendingIntent.CanceledException e) {
215             // Do not take down the app over this
216             Log.w(TAG, "Sending contentIntent failed: " + e);
217             return ActivityManager.START_ABORTED;
218         }
219     }
220 
221     /** Helper class that generates unique IDs per TTS request. **/
222     private class RequestIdGenerator {
223         private long mCounter;
224 
RequestIdGenerator()225         RequestIdGenerator() {
226             mCounter = 0;
227         }
228 
generateRequestId()229         public long generateRequestId() {
230             return ++mCounter;
231         }
232     }
233 
234     /**
235      * Contains all of the information needed to start and finish actions supported by the
236      * FallbackAssistant.
237      **/
238     private class ActionRequestInfo {
239         private final StatusBarNotification mStatusBarNotification;
240         private final Listener mListener;
241 
ActionRequestInfo(@ullable StatusBarNotification statusBarNotification, Listener listener)242         ActionRequestInfo(@Nullable StatusBarNotification statusBarNotification,
243                 Listener listener) {
244             mStatusBarNotification = statusBarNotification;
245             mListener = listener;
246         }
247 
248         @Nullable
getStatusBarNotification()249         StatusBarNotification getStatusBarNotification() {
250             return mStatusBarNotification;
251         }
252 
getListener()253         Listener getListener() {
254             return mListener;
255         }
256     }
257 }
258