1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.car.assist.client; 18 19 20 import android.annotation.Nullable; 21 import android.app.ActivityManager; 22 import android.app.Notification; 23 import android.app.Notification.MessagingStyle.Message; 24 import android.app.PendingIntent; 25 import android.app.Person; 26 import android.content.Context; 27 import android.content.Intent; 28 import android.os.Parcelable; 29 import android.service.notification.StatusBarNotification; 30 import android.util.Log; 31 import android.widget.Toast; 32 33 import androidx.core.app.NotificationCompat; 34 35 import com.android.car.assist.client.tts.TextToSpeechHelper; 36 37 import java.util.ArrayList; 38 import java.util.Collections; 39 import java.util.HashMap; 40 import java.util.List; 41 import java.util.Map; 42 43 /** 44 * Handles Assistant request fallbacks in the case that Assistant cannot fulfill the request for 45 * any given reason. 46 * <p/> 47 * Simply reads out the notification messages for read requests, and speaks out 48 * an error message for other requests. 49 */ 50 public class FallbackAssistant { 51 52 private static final String TAG = FallbackAssistant.class.getSimpleName(); 53 54 private final Context mContext; 55 private final TextToSpeechHelper mTextToSpeechHelper; 56 private final RequestIdGenerator mRequestIdGenerator; 57 private Map<Long, ActionRequestInfo> mRequestIdToActionRequestInfo = new HashMap<>(); 58 // String that means "says", to be used when reading out a message (i.e. <Sender> says 59 // <Message). 60 private final String mVerbForSays; 61 62 private final TextToSpeechHelper.Listener mListener = new TextToSpeechHelper.Listener() { 63 @Override 64 public void onTextToSpeechStarted(long requestId) { 65 if (Log.isLoggable(TAG, Log.DEBUG)) { 66 Log.d(TAG, "onTextToSpeechStarted"); 67 } 68 } 69 70 @Override 71 public void onTextToSpeechStopped(long requestId, boolean error) { 72 if (Log.isLoggable(TAG, Log.DEBUG)) { 73 Log.d(TAG, "onTextToSpeechStopped"); 74 } 75 76 if (error) { 77 Toast.makeText(mContext, mContext.getString(R.string.assist_action_failed_toast), 78 Toast.LENGTH_LONG).show(); 79 } 80 finishAction(requestId, error); 81 } 82 }; 83 84 /** Listener to allow clients to be alerted when their requested message has been read. **/ 85 public interface Listener { 86 /** 87 * Called after the TTS engine has finished reading aloud the message. 88 */ onMessageRead(boolean hasError)89 void onMessageRead(boolean hasError); 90 } 91 FallbackAssistant(Context context)92 public FallbackAssistant(Context context) { 93 mContext = context; 94 mTextToSpeechHelper = new TextToSpeechHelper(context, mListener); 95 mRequestIdGenerator = new RequestIdGenerator(); 96 mVerbForSays = mContext.getString(R.string.says); 97 } 98 99 /** 100 * Handles a fallback read action by reading all messages in the notification. 101 * 102 * @param sbn the payload notification from which to extract messages from 103 */ handleReadAction(StatusBarNotification sbn, Listener listener)104 public void handleReadAction(StatusBarNotification sbn, Listener listener) { 105 if (mTextToSpeechHelper.isSpeaking()) { 106 mTextToSpeechHelper.requestStop(); 107 } 108 109 Parcelable[] messagesBundle = sbn.getNotification().extras 110 .getParcelableArray(Notification.EXTRA_MESSAGES); 111 112 if (messagesBundle == null || messagesBundle.length == 0) { 113 listener.onMessageRead(/* hasError= */ true); 114 return; 115 } 116 117 List<CharSequence> messages = new ArrayList<>(); 118 119 List<Message> messageList = Message.getMessagesFromBundleArray(messagesBundle); 120 if (messageList == null || messageList.isEmpty()) { 121 Log.w(TAG, "No messages could be extracted from the bundle"); 122 listener.onMessageRead(/* hasError= */ true); 123 return; 124 } 125 126 Person previousSender = messageList.get(0).getSenderPerson(); 127 if (previousSender != null) { 128 messages.add(previousSender.getName()); 129 messages.add(mVerbForSays); 130 } 131 for (Message message : messageList) { 132 if (!message.getSenderPerson().equals(previousSender)) { 133 messages.add(message.getSenderPerson().getName()); 134 messages.add(mVerbForSays); 135 previousSender = message.getSenderPerson(); 136 } 137 messages.add(message.getText()); 138 } 139 140 long requestId = mRequestIdGenerator.generateRequestId(); 141 142 if (mTextToSpeechHelper.requestPlay(messages, requestId)) { 143 if (Log.isLoggable(TAG, Log.DEBUG)) { 144 Log.d(TAG, "Requesting TTS to read message with requestId: " + requestId); 145 } 146 mRequestIdToActionRequestInfo.put(requestId, new ActionRequestInfo(sbn, listener)); 147 } else { 148 listener.onMessageRead(/* hasError= */ true); 149 } 150 } 151 152 /** 153 * Handles generic (non-read) actions by reading out an error message. 154 * 155 * @param errorMessage the error message to read out 156 */ handleErrorMessage(CharSequence errorMessage, Listener listener)157 public void handleErrorMessage(CharSequence errorMessage, Listener listener) { 158 if (mTextToSpeechHelper.isSpeaking()) { 159 mTextToSpeechHelper.requestStop(); 160 } 161 162 long requestId = mRequestIdGenerator.generateRequestId(); 163 if (mTextToSpeechHelper.requestPlay(Collections.singletonList(errorMessage), 164 requestId)) { 165 if (Log.isLoggable(TAG, Log.DEBUG)) { 166 Log.d(TAG, "Requesting TTS to read error with requestId: " + requestId); 167 } 168 mRequestIdToActionRequestInfo.put(requestId, new ActionRequestInfo( 169 /* statusBarNotification= */ null, 170 listener)); 171 } else { 172 listener.onMessageRead(/* hasError= */ true); 173 } 174 } 175 finishAction(long requestId, boolean hasError)176 private void finishAction(long requestId, boolean hasError) { 177 if (!mRequestIdToActionRequestInfo.containsKey(requestId)) { 178 Log.w(TAG, "No actionRequestInfo found for requestId: " + requestId); 179 return; 180 } 181 182 ActionRequestInfo info = mRequestIdToActionRequestInfo.remove(requestId); 183 184 if (info.getStatusBarNotification() != null && !hasError) { 185 sendMarkAsReadIntent(info.getStatusBarNotification()); 186 } 187 188 info.getListener().onMessageRead(hasError); 189 } 190 sendMarkAsReadIntent(StatusBarNotification sbn)191 private void sendMarkAsReadIntent(StatusBarNotification sbn) { 192 NotificationCompat.Action markAsReadAction = CarAssistUtils.getMarkAsReadAction( 193 sbn.getNotification()); 194 boolean isDebugLoggable = Log.isLoggable(TAG, Log.DEBUG); 195 196 if (markAsReadAction != null) { 197 if (sendPendingIntent(markAsReadAction.getActionIntent(), 198 null /* resultIntent */) != ActivityManager.START_SUCCESS 199 && isDebugLoggable) { 200 Log.d(TAG, "Could not relay mark as read event to the messaging app."); 201 } 202 } else if (isDebugLoggable) { 203 Log.d(TAG, "Car compat message notification has no mark as read action: " 204 + sbn.getKey()); 205 } 206 } 207 sendPendingIntent(PendingIntent pendingIntent, Intent resultIntent)208 private int sendPendingIntent(PendingIntent pendingIntent, Intent resultIntent) { 209 try { 210 return pendingIntent.sendAndReturnResult(/* context= */ mContext, /* code= */ 0, 211 /* intent= */ resultIntent, /* onFinished= */null, 212 /* handler= */ null, /* requiredPermissions= */ null, 213 /* options= */ null); 214 } catch (PendingIntent.CanceledException e) { 215 // Do not take down the app over this 216 Log.w(TAG, "Sending contentIntent failed: " + e); 217 return ActivityManager.START_ABORTED; 218 } 219 } 220 221 /** Helper class that generates unique IDs per TTS request. **/ 222 private class RequestIdGenerator { 223 private long mCounter; 224 RequestIdGenerator()225 RequestIdGenerator() { 226 mCounter = 0; 227 } 228 generateRequestId()229 public long generateRequestId() { 230 return ++mCounter; 231 } 232 } 233 234 /** 235 * Contains all of the information needed to start and finish actions supported by the 236 * FallbackAssistant. 237 **/ 238 private class ActionRequestInfo { 239 private final StatusBarNotification mStatusBarNotification; 240 private final Listener mListener; 241 ActionRequestInfo(@ullable StatusBarNotification statusBarNotification, Listener listener)242 ActionRequestInfo(@Nullable StatusBarNotification statusBarNotification, 243 Listener listener) { 244 mStatusBarNotification = statusBarNotification; 245 mListener = listener; 246 } 247 248 @Nullable getStatusBarNotification()249 StatusBarNotification getStatusBarNotification() { 250 return mStatusBarNotification; 251 } 252 getListener()253 Listener getListener() { 254 return mListener; 255 } 256 } 257 } 258