1 /**
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package android.ext.services.notification;
17 
18 import android.annotation.Nullable;
19 import android.app.Notification;
20 import android.app.PendingIntent;
21 import android.app.Person;
22 import android.app.RemoteAction;
23 import android.app.RemoteInput;
24 import android.content.Context;
25 import android.content.Intent;
26 import android.ext.services.R;
27 import android.graphics.drawable.Icon;
28 import android.os.Bundle;
29 import android.os.Parcelable;
30 import android.os.Process;
31 import android.service.notification.NotificationAssistantService;
32 import android.text.TextUtils;
33 import android.util.ArrayMap;
34 import android.util.LruCache;
35 import android.util.Pair;
36 import android.view.textclassifier.ConversationAction;
37 import android.view.textclassifier.ConversationActions;
38 import android.view.textclassifier.TextClassificationContext;
39 import android.view.textclassifier.TextClassificationManager;
40 import android.view.textclassifier.TextClassifier;
41 import android.view.textclassifier.TextClassifierEvent;
42 
43 import com.android.internal.util.ArrayUtils;
44 
45 import java.time.Instant;
46 import java.time.ZoneOffset;
47 import java.time.ZonedDateTime;
48 import java.util.ArrayDeque;
49 import java.util.ArrayList;
50 import java.util.Collections;
51 import java.util.Deque;
52 import java.util.List;
53 import java.util.Map;
54 import java.util.Objects;
55 
56 /**
57  * Generates suggestions from incoming notifications.
58  *
59  * Methods in this class should be called in a single worker thread.
60  */
61 public class SmartActionsHelper {
62     static final String ENTITIES_EXTRAS = "entities-extras";
63     static final String KEY_ACTION_TYPE = "action_type";
64     static final String KEY_ACTION_SCORE = "action_score";
65     static final String KEY_TEXT = "text";
66     // If a notification has any of these flags set, it's inelgibile for actions being added.
67     private static final int FLAG_MASK_INELGIBILE_FOR_ACTIONS =
68             Notification.FLAG_ONGOING_EVENT
69                     | Notification.FLAG_FOREGROUND_SERVICE
70                     | Notification.FLAG_GROUP_SUMMARY
71                     | Notification.FLAG_NO_CLEAR;
72     private static final int MAX_RESULT_ID_TO_CACHE = 20;
73 
74     private static final List<String> HINTS =
75             Collections.singletonList(ConversationActions.Request.HINT_FOR_NOTIFICATION);
76     private static final ConversationActions EMPTY_CONVERSATION_ACTIONS =
77             new ConversationActions(Collections.emptyList(), null);
78 
79     private Context mContext;
80     private TextClassificationManager mTextClassificationManager;
81     private AssistantSettings mSettings;
82     private LruCache<String, Session> mSessionCache = new LruCache<>(MAX_RESULT_ID_TO_CACHE);
83 
SmartActionsHelper(Context context, AssistantSettings settings)84     SmartActionsHelper(Context context, AssistantSettings settings) {
85         mContext = context;
86         mTextClassificationManager = mContext.getSystemService(TextClassificationManager.class);
87         mSettings = settings;
88     }
89 
suggest(NotificationEntry entry)90     SmartSuggestions suggest(NotificationEntry entry) {
91         // Whenever suggest() is called on a notification, its previous session is ended.
92         mSessionCache.remove(entry.getSbn().getKey());
93 
94         boolean eligibleForReplyAdjustment =
95                 mSettings.mGenerateReplies && isEligibleForReplyAdjustment(entry);
96         boolean eligibleForActionAdjustment =
97                 mSettings.mGenerateActions && isEligibleForActionAdjustment(entry);
98 
99         ConversationActions conversationActionsResult =
100                 suggestConversationActions(
101                         entry,
102                         eligibleForReplyAdjustment,
103                         eligibleForActionAdjustment);
104 
105         String resultId = conversationActionsResult.getId();
106         List<ConversationAction> conversationActions =
107                 conversationActionsResult.getConversationActions();
108 
109         ArrayList<CharSequence> replies = new ArrayList<>();
110         Map<CharSequence, Float> repliesScore = new ArrayMap<>();
111         for (ConversationAction conversationAction : conversationActions) {
112             CharSequence textReply = conversationAction.getTextReply();
113             if (TextUtils.isEmpty(textReply)) {
114                 continue;
115             }
116             replies.add(textReply);
117             repliesScore.put(textReply, conversationAction.getConfidenceScore());
118         }
119 
120         ArrayList<Notification.Action> actions = new ArrayList<>();
121         for (ConversationAction conversationAction : conversationActions) {
122             if (!TextUtils.isEmpty(conversationAction.getTextReply())) {
123                 continue;
124             }
125             Notification.Action notificationAction;
126             if (conversationAction.getAction() == null) {
127                 notificationAction =
128                         createNotificationActionWithoutRemoteAction(conversationAction);
129             } else {
130                 notificationAction = createNotificationActionFromRemoteAction(
131                         conversationAction.getAction(),
132                         conversationAction.getType(),
133                         conversationAction.getConfidenceScore());
134             }
135             if (notificationAction != null) {
136                 actions.add(notificationAction);
137             }
138         }
139 
140         // Start a new session for logging if necessary.
141         if (!TextUtils.isEmpty(resultId)
142                 && !conversationActions.isEmpty()
143                 && suggestionsMightBeUsedInNotification(
144                 entry, !actions.isEmpty(), !replies.isEmpty())) {
145             mSessionCache.put(entry.getSbn().getKey(), new Session(resultId, repliesScore));
146         }
147 
148         return new SmartSuggestions(replies, actions);
149     }
150 
151     /**
152      * Creates notification action from ConversationAction that does not come up a RemoteAction.
153      * It could happen because we don't have common intents for some actions, like copying text.
154      */
155     @Nullable
createNotificationActionWithoutRemoteAction( ConversationAction conversationAction)156     private Notification.Action createNotificationActionWithoutRemoteAction(
157             ConversationAction conversationAction) {
158         if (ConversationAction.TYPE_COPY.equals(conversationAction.getType())) {
159             return createCopyCodeAction(conversationAction);
160         }
161         return null;
162     }
163 
164     @Nullable
createCopyCodeAction(ConversationAction conversationAction)165     private Notification.Action createCopyCodeAction(ConversationAction conversationAction) {
166         Bundle extras = conversationAction.getExtras();
167         if (extras == null) {
168             return null;
169         }
170         Bundle entitiesExtas = extras.getParcelable(ENTITIES_EXTRAS);
171         if (entitiesExtas == null) {
172             return null;
173         }
174         String code = entitiesExtas.getString(KEY_TEXT);
175         if (TextUtils.isEmpty(code)) {
176             return null;
177         }
178         String contentDescription = mContext.getString(R.string.copy_code_desc, code);
179         Intent intent = new Intent(mContext, CopyCodeActivity.class);
180         intent.putExtra(Intent.EXTRA_TEXT, code);
181 
182         RemoteAction remoteAction = new RemoteAction(
183                 Icon.createWithResource(mContext, R.drawable.ic_menu_copy_material),
184                 code,
185                 contentDescription,
186                 PendingIntent.getActivity(
187                         mContext,
188                         code.hashCode(),
189                         intent,
190                         PendingIntent.FLAG_UPDATE_CURRENT
191                 ));
192 
193         return createNotificationActionFromRemoteAction(
194                 remoteAction,
195                 ConversationAction.TYPE_COPY,
196                 conversationAction.getConfidenceScore());
197     }
198 
199     /**
200      * Returns whether the suggestion might be used in the notifications in SysUI.
201      * <p>
202      * Currently, NAS has no idea if suggestions will actually be used in the notification, and thus
203      * this function tries to make a heuristic. This function tries to optimize the precision,
204      * that means when it is unsure, it will return false. The objective is to avoid false positive,
205      * which could pollute the log and CTR as we are logging click rate of suggestions that could
206      * be never visible to users. On the other hand, it is fine to have false negative because
207      * it would be just like sampling.
208      */
suggestionsMightBeUsedInNotification( NotificationEntry notificationEntry, boolean hasSmartAction, boolean hasSmartReply)209     private boolean suggestionsMightBeUsedInNotification(
210             NotificationEntry notificationEntry, boolean hasSmartAction, boolean hasSmartReply) {
211         Notification notification = notificationEntry.getNotification();
212         boolean hasAppGeneratedContextualActions = !notification.getContextualActions().isEmpty();
213 
214         Pair<RemoteInput, Notification.Action> freeformRemoteInputAndAction =
215                 notification.findRemoteInputActionPair(/* requiresFreeform */ true);
216         boolean hasAppGeneratedReplies = false;
217         boolean allowGeneratedReplies = false;
218         if (freeformRemoteInputAndAction != null) {
219             RemoteInput freeformRemoteInput = freeformRemoteInputAndAction.first;
220             Notification.Action actionWithFreeformRemoteInput = freeformRemoteInputAndAction.second;
221             hasAppGeneratedReplies = !ArrayUtils.isEmpty(freeformRemoteInput.getChoices());
222             allowGeneratedReplies = actionWithFreeformRemoteInput.getAllowGeneratedReplies();
223         }
224 
225         if (hasAppGeneratedReplies || hasAppGeneratedContextualActions) {
226             return false;
227         }
228         return hasSmartAction && notification.getAllowSystemGeneratedContextualActions()
229                 || hasSmartReply && allowGeneratedReplies;
230     }
231 
reportActionsGenerated( String resultId, List<ConversationAction> conversationActions)232     private void reportActionsGenerated(
233             String resultId, List<ConversationAction> conversationActions) {
234         if (TextUtils.isEmpty(resultId)) {
235             return;
236         }
237         TextClassifierEvent textClassifierEvent =
238                 createTextClassifierEventBuilder(
239                         TextClassifierEvent.TYPE_ACTIONS_GENERATED, resultId)
240                         .setEntityTypes(conversationActions.stream()
241                                 .map(ConversationAction::getType)
242                                 .toArray(String[]::new))
243                         .build();
244         getTextClassifier().onTextClassifierEvent(textClassifierEvent);
245     }
246 
247     /**
248      * Adds action adjustments based on the notification contents.
249      */
suggestConversationActions( NotificationEntry entry, boolean includeReplies, boolean includeActions)250     private ConversationActions suggestConversationActions(
251             NotificationEntry entry,
252             boolean includeReplies,
253             boolean includeActions) {
254         if (!includeReplies && !includeActions) {
255             return EMPTY_CONVERSATION_ACTIONS;
256         }
257         List<ConversationActions.Message> messages = extractMessages(entry.getNotification());
258         if (messages.isEmpty()) {
259             return EMPTY_CONVERSATION_ACTIONS;
260         }
261         // Do not generate smart actions if the last message is from the local user.
262         ConversationActions.Message lastMessage = messages.get(messages.size() - 1);
263         if (arePersonsEqual(
264                 ConversationActions.Message.PERSON_USER_SELF, lastMessage.getAuthor())) {
265             return EMPTY_CONVERSATION_ACTIONS;
266         }
267 
268         TextClassifier.EntityConfig.Builder typeConfigBuilder =
269                 new TextClassifier.EntityConfig.Builder();
270         if (!includeReplies) {
271             typeConfigBuilder.setExcludedTypes(
272                     Collections.singletonList(ConversationAction.TYPE_TEXT_REPLY));
273         } else if (!includeActions) {
274             typeConfigBuilder
275                     .setIncludedTypes(
276                             Collections.singletonList(ConversationAction.TYPE_TEXT_REPLY))
277                     .includeTypesFromTextClassifier(false);
278         }
279         ConversationActions.Request request =
280                 new ConversationActions.Request.Builder(messages)
281                         .setMaxSuggestions(mSettings.mMaxSuggestions)
282                         .setHints(HINTS)
283                         .setTypeConfig(typeConfigBuilder.build())
284                         .build();
285         ConversationActions conversationActions =
286                 getTextClassifier().suggestConversationActions(request);
287         reportActionsGenerated(
288                 conversationActions.getId(), conversationActions.getConversationActions());
289         return conversationActions;
290     }
291 
onNotificationExpansionChanged(NotificationEntry entry, boolean isExpanded)292     void onNotificationExpansionChanged(NotificationEntry entry, boolean isExpanded) {
293         if (!isExpanded) {
294             return;
295         }
296         Session session = mSessionCache.get(entry.getSbn().getKey());
297         if (session == null) {
298             return;
299         }
300         // Only report if this is the first time the user sees these suggestions.
301         if (entry.isShowActionEventLogged()) {
302             return;
303         }
304         entry.setShowActionEventLogged();
305         TextClassifierEvent textClassifierEvent =
306                 createTextClassifierEventBuilder(
307                         TextClassifierEvent.TYPE_ACTIONS_SHOWN, session.resultId)
308                         .build();
309         // TODO: If possible, report which replies / actions are actually seen by user.
310         getTextClassifier().onTextClassifierEvent(textClassifierEvent);
311     }
312 
onNotificationDirectReplied(String key)313     void onNotificationDirectReplied(String key) {
314         Session session = mSessionCache.get(key);
315         if (session == null) {
316             return;
317         }
318         TextClassifierEvent textClassifierEvent =
319                 createTextClassifierEventBuilder(
320                         TextClassifierEvent.TYPE_MANUAL_REPLY, session.resultId)
321                         .build();
322         getTextClassifier().onTextClassifierEvent(textClassifierEvent);
323     }
324 
onSuggestedReplySent(String key, CharSequence reply, @NotificationAssistantService.Source int source)325     void onSuggestedReplySent(String key, CharSequence reply,
326             @NotificationAssistantService.Source int source) {
327         if (source != NotificationAssistantService.SOURCE_FROM_ASSISTANT) {
328             return;
329         }
330         Session session = mSessionCache.get(key);
331         if (session == null) {
332             return;
333         }
334         TextClassifierEvent textClassifierEvent =
335                 createTextClassifierEventBuilder(
336                         TextClassifierEvent.TYPE_SMART_ACTION, session.resultId)
337                         .setEntityTypes(ConversationAction.TYPE_TEXT_REPLY)
338                         .setScores(session.repliesScores.getOrDefault(reply, 0f))
339                         .build();
340         getTextClassifier().onTextClassifierEvent(textClassifierEvent);
341     }
342 
onActionClicked(String key, Notification.Action action, @NotificationAssistantService.Source int source)343     void onActionClicked(String key, Notification.Action action,
344             @NotificationAssistantService.Source int source) {
345         if (source != NotificationAssistantService.SOURCE_FROM_ASSISTANT) {
346             return;
347         }
348         Session session = mSessionCache.get(key);
349         if (session == null) {
350             return;
351         }
352         String actionType = action.getExtras().getString(KEY_ACTION_TYPE);
353         if (actionType == null) {
354             return;
355         }
356         TextClassifierEvent textClassifierEvent =
357                 createTextClassifierEventBuilder(
358                         TextClassifierEvent.TYPE_SMART_ACTION, session.resultId)
359                         .setEntityTypes(actionType)
360                         .build();
361         getTextClassifier().onTextClassifierEvent(textClassifierEvent);
362     }
363 
createNotificationActionFromRemoteAction( RemoteAction remoteAction, String actionType, float score)364     private Notification.Action createNotificationActionFromRemoteAction(
365             RemoteAction remoteAction, String actionType, float score) {
366         Icon icon = remoteAction.shouldShowIcon()
367                 ? remoteAction.getIcon()
368                 : Icon.createWithResource(mContext, R.drawable.ic_action_open);
369         Bundle extras = new Bundle();
370         extras.putString(KEY_ACTION_TYPE, actionType);
371         extras.putFloat(KEY_ACTION_SCORE, score);
372         return new Notification.Action.Builder(
373                 icon,
374                 remoteAction.getTitle(),
375                 remoteAction.getActionIntent())
376                 .setContextual(true)
377                 .addExtras(extras)
378                 .build();
379     }
380 
createTextClassifierEventBuilder( int eventType, String resultId)381     private TextClassifierEvent.ConversationActionsEvent.Builder createTextClassifierEventBuilder(
382             int eventType, String resultId) {
383         return new TextClassifierEvent.ConversationActionsEvent.Builder(eventType)
384                 .setEventContext(
385                         new TextClassificationContext.Builder(
386                                 mContext.getPackageName(), TextClassifier.WIDGET_TYPE_NOTIFICATION)
387                         .build())
388                 .setResultId(resultId);
389     }
390 
391     /**
392      * Returns whether a notification is eligible for action adjustments.
393      *
394      * <p>We exclude system notifications, those that get refreshed frequently, or ones that relate
395      * to fundamental phone functionality where any error would result in a very negative user
396      * experience.
397      */
isEligibleForActionAdjustment(NotificationEntry entry)398     private boolean isEligibleForActionAdjustment(NotificationEntry entry) {
399         Notification notification = entry.getNotification();
400         String pkg = entry.getSbn().getPackageName();
401         if (!Process.myUserHandle().equals(entry.getSbn().getUser())) {
402             return false;
403         }
404         if ((notification.flags & FLAG_MASK_INELGIBILE_FOR_ACTIONS) != 0) {
405             return false;
406         }
407         if (TextUtils.isEmpty(pkg) || pkg.equals("android")) {
408             return false;
409         }
410         // For now, we are only interested in messages.
411         return entry.isMessaging();
412     }
413 
isEligibleForReplyAdjustment(NotificationEntry entry)414     private boolean isEligibleForReplyAdjustment(NotificationEntry entry) {
415         if (!Process.myUserHandle().equals(entry.getSbn().getUser())) {
416             return false;
417         }
418         String pkg = entry.getSbn().getPackageName();
419         if (TextUtils.isEmpty(pkg) || pkg.equals("android")) {
420             return false;
421         }
422         // For now, we are only interested in messages.
423         if (!entry.isMessaging()) {
424             return false;
425         }
426         // Does not make sense to provide suggested replies if it is not something that can be
427         // replied.
428         if (!entry.hasInlineReply()) {
429             return false;
430         }
431         return true;
432     }
433 
434     /** Returns the text most salient for action extraction in a notification. */
extractMessages(Notification notification)435     private List<ConversationActions.Message> extractMessages(Notification notification) {
436         Parcelable[] messages = notification.extras.getParcelableArray(Notification.EXTRA_MESSAGES);
437         if (messages == null || messages.length == 0) {
438             return Collections.singletonList(new ConversationActions.Message.Builder(
439                     ConversationActions.Message.PERSON_USER_OTHERS)
440                     .setText(notification.extras.getCharSequence(Notification.EXTRA_TEXT))
441                     .build());
442         }
443         Person localUser = notification.extras.getParcelable(Notification.EXTRA_MESSAGING_PERSON);
444         Deque<ConversationActions.Message> extractMessages = new ArrayDeque<>();
445         for (int i = messages.length - 1; i >= 0; i--) {
446             Notification.MessagingStyle.Message message =
447                     Notification.MessagingStyle.Message.getMessageFromBundle((Bundle) messages[i]);
448             if (message == null) {
449                 continue;
450             }
451             // As per the javadoc of Notification.addMessage, null means local user.
452             Person senderPerson = message.getSenderPerson();
453             if (senderPerson == null) {
454                 senderPerson = localUser;
455             }
456             Person author = localUser != null && arePersonsEqual(localUser, senderPerson)
457                     ? ConversationActions.Message.PERSON_USER_SELF : senderPerson;
458             extractMessages.push(new ConversationActions.Message.Builder(author)
459                     .setText(message.getText())
460                     .setReferenceTime(
461                             ZonedDateTime.ofInstant(Instant.ofEpochMilli(message.getTimestamp()),
462                                     ZoneOffset.systemDefault()))
463                     .build());
464             if (extractMessages.size() >= mSettings.mMaxMessagesToExtract) {
465                 break;
466             }
467         }
468         return new ArrayList<>(extractMessages);
469     }
470 
getTextClassifier()471     private TextClassifier getTextClassifier() {
472         return mTextClassificationManager.getTextClassifier();
473     }
474 
arePersonsEqual(Person left, Person right)475     private static boolean arePersonsEqual(Person left, Person right) {
476         return Objects.equals(left.getKey(), right.getKey())
477                 && Objects.equals(left.getName(), right.getName())
478                 && Objects.equals(left.getUri(), right.getUri());
479     }
480 
481     static class SmartSuggestions {
482         public final ArrayList<CharSequence> replies;
483         public final ArrayList<Notification.Action> actions;
484 
SmartSuggestions( ArrayList<CharSequence> replies, ArrayList<Notification.Action> actions)485         SmartSuggestions(
486                 ArrayList<CharSequence> replies, ArrayList<Notification.Action> actions) {
487             this.replies = replies;
488             this.actions = actions;
489         }
490     }
491 
492     private static class Session {
493         public final String resultId;
494         public final Map<CharSequence, Float> repliesScores;
495 
Session(String resultId, Map<CharSequence, Float> repliesScores)496         Session(String resultId, Map<CharSequence, Float> repliesScores) {
497             this.resultId = resultId;
498             this.repliesScores = repliesScores;
499         }
500     }
501 }
502