1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.app; 18 19 import android.accessibilityservice.AccessibilityService.Callbacks; 20 import android.accessibilityservice.AccessibilityService.IAccessibilityServiceClientWrapper; 21 import android.accessibilityservice.AccessibilityServiceInfo; 22 import android.accessibilityservice.IAccessibilityServiceClient; 23 import android.accessibilityservice.IAccessibilityServiceConnection; 24 import android.annotation.NonNull; 25 import android.annotation.Nullable; 26 import android.annotation.TestApi; 27 import android.compat.annotation.UnsupportedAppUsage; 28 import android.graphics.Bitmap; 29 import android.graphics.Point; 30 import android.graphics.Rect; 31 import android.graphics.Region; 32 import android.hardware.display.DisplayManagerGlobal; 33 import android.os.Build; 34 import android.os.Handler; 35 import android.os.HandlerThread; 36 import android.os.IBinder; 37 import android.os.Looper; 38 import android.os.ParcelFileDescriptor; 39 import android.os.Process; 40 import android.os.RemoteException; 41 import android.os.SystemClock; 42 import android.os.UserHandle; 43 import android.util.Log; 44 import android.view.Display; 45 import android.view.InputEvent; 46 import android.view.KeyEvent; 47 import android.view.Surface; 48 import android.view.WindowAnimationFrameStats; 49 import android.view.WindowContentFrameStats; 50 import android.view.accessibility.AccessibilityEvent; 51 import android.view.accessibility.AccessibilityInteractionClient; 52 import android.view.accessibility.AccessibilityNodeInfo; 53 import android.view.accessibility.AccessibilityWindowInfo; 54 import android.view.accessibility.IAccessibilityInteractionConnection; 55 56 import com.android.internal.util.function.pooled.PooledLambda; 57 58 import libcore.io.IoUtils; 59 60 import java.io.IOException; 61 import java.util.ArrayList; 62 import java.util.List; 63 import java.util.concurrent.TimeoutException; 64 65 /** 66 * Class for interacting with the device's UI by simulation user actions and 67 * introspection of the screen content. It relies on the platform accessibility 68 * APIs to introspect the screen and to perform some actions on the remote view 69 * tree. It also allows injecting of arbitrary raw input events simulating user 70 * interaction with keyboards and touch devices. One can think of a UiAutomation 71 * as a special type of {@link android.accessibilityservice.AccessibilityService} 72 * which does not provide hooks for the service life cycle and exposes other 73 * APIs that are useful for UI test automation. 74 * <p> 75 * The APIs exposed by this class are low-level to maximize flexibility when 76 * developing UI test automation tools and libraries. Generally, a UiAutomation 77 * client should be using a higher-level library or implement high-level functions. 78 * For example, performing a tap on the screen requires construction and injecting 79 * of a touch down and up events which have to be delivered to the system by a 80 * call to {@link #injectInputEvent(InputEvent, boolean)}. 81 * </p> 82 * <p> 83 * The APIs exposed by this class operate across applications enabling a client 84 * to write tests that cover use cases spanning over multiple applications. For 85 * example, going to the settings application to change a setting and then 86 * interacting with another application whose behavior depends on that setting. 87 * </p> 88 */ 89 public final class UiAutomation { 90 91 private static final String LOG_TAG = UiAutomation.class.getSimpleName(); 92 93 private static final boolean DEBUG = false; 94 95 private static final int CONNECTION_ID_UNDEFINED = -1; 96 97 private static final long CONNECT_TIMEOUT_MILLIS = 5000; 98 99 /** Rotation constant: Unfreeze rotation (rotating the device changes its rotation state). */ 100 public static final int ROTATION_UNFREEZE = -2; 101 102 /** Rotation constant: Freeze rotation to its current state. */ 103 public static final int ROTATION_FREEZE_CURRENT = -1; 104 105 /** Rotation constant: Freeze rotation to 0 degrees (natural orientation) */ 106 public static final int ROTATION_FREEZE_0 = Surface.ROTATION_0; 107 108 /** Rotation constant: Freeze rotation to 90 degrees . */ 109 public static final int ROTATION_FREEZE_90 = Surface.ROTATION_90; 110 111 /** Rotation constant: Freeze rotation to 180 degrees . */ 112 public static final int ROTATION_FREEZE_180 = Surface.ROTATION_180; 113 114 /** Rotation constant: Freeze rotation to 270 degrees . */ 115 public static final int ROTATION_FREEZE_270 = Surface.ROTATION_270; 116 117 /** 118 * UiAutomation supresses accessibility services by default. This flag specifies that 119 * existing accessibility services should continue to run, and that new ones may start. 120 * This flag is set when obtaining the UiAutomation from 121 * {@link Instrumentation#getUiAutomation(int)}. 122 */ 123 public static final int FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES = 0x00000001; 124 125 private final Object mLock = new Object(); 126 127 private final ArrayList<AccessibilityEvent> mEventQueue = new ArrayList<AccessibilityEvent>(); 128 129 private final Handler mLocalCallbackHandler; 130 131 private final IUiAutomationConnection mUiAutomationConnection; 132 133 private HandlerThread mRemoteCallbackThread; 134 135 private IAccessibilityServiceClient mClient; 136 137 private int mConnectionId = CONNECTION_ID_UNDEFINED; 138 139 private OnAccessibilityEventListener mOnAccessibilityEventListener; 140 141 private boolean mWaitingForEventDelivery; 142 143 private long mLastEventTimeMillis; 144 145 private boolean mIsConnecting; 146 147 private boolean mIsDestroyed; 148 149 private int mFlags; 150 151 /** 152 * Listener for observing the {@link AccessibilityEvent} stream. 153 */ 154 public static interface OnAccessibilityEventListener { 155 156 /** 157 * Callback for receiving an {@link AccessibilityEvent}. 158 * <p> 159 * <strong>Note:</strong> This method is <strong>NOT</strong> executed 160 * on the main test thread. The client is responsible for proper 161 * synchronization. 162 * </p> 163 * <p> 164 * <strong>Note:</strong> It is responsibility of the client 165 * to recycle the received events to minimize object creation. 166 * </p> 167 * 168 * @param event The received event. 169 */ onAccessibilityEvent(AccessibilityEvent event)170 public void onAccessibilityEvent(AccessibilityEvent event); 171 } 172 173 /** 174 * Listener for filtering accessibility events. 175 */ 176 public static interface AccessibilityEventFilter { 177 178 /** 179 * Callback for determining whether an event is accepted or 180 * it is filtered out. 181 * 182 * @param event The event to process. 183 * @return True if the event is accepted, false to filter it out. 184 */ accept(AccessibilityEvent event)185 public boolean accept(AccessibilityEvent event); 186 } 187 188 /** 189 * Creates a new instance that will handle callbacks from the accessibility 190 * layer on the thread of the provided looper and perform requests for privileged 191 * operations on the provided connection. 192 * 193 * @param looper The looper on which to execute accessibility callbacks. 194 * @param connection The connection for performing privileged operations. 195 * 196 * @hide 197 */ 198 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) UiAutomation(Looper looper, IUiAutomationConnection connection)199 public UiAutomation(Looper looper, IUiAutomationConnection connection) { 200 if (looper == null) { 201 throw new IllegalArgumentException("Looper cannot be null!"); 202 } 203 if (connection == null) { 204 throw new IllegalArgumentException("Connection cannot be null!"); 205 } 206 mLocalCallbackHandler = new Handler(looper); 207 mUiAutomationConnection = connection; 208 } 209 210 /** 211 * Connects this UiAutomation to the accessibility introspection APIs with default flags. 212 * 213 * @hide 214 */ 215 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) connect()216 public void connect() { 217 connect(0); 218 } 219 220 /** 221 * Connects this UiAutomation to the accessibility introspection APIs. 222 * 223 * @param flags Any flags to apply to the automation as it gets connected 224 * 225 * @hide 226 */ connect(int flags)227 public void connect(int flags) { 228 synchronized (mLock) { 229 throwIfConnectedLocked(); 230 if (mIsConnecting) { 231 return; 232 } 233 mIsConnecting = true; 234 mRemoteCallbackThread = new HandlerThread("UiAutomation"); 235 mRemoteCallbackThread.start(); 236 mClient = new IAccessibilityServiceClientImpl(mRemoteCallbackThread.getLooper()); 237 } 238 239 try { 240 // Calling out without a lock held. 241 mUiAutomationConnection.connect(mClient, flags); 242 mFlags = flags; 243 } catch (RemoteException re) { 244 throw new RuntimeException("Error while connecting UiAutomation", re); 245 } 246 247 synchronized (mLock) { 248 final long startTimeMillis = SystemClock.uptimeMillis(); 249 try { 250 while (true) { 251 if (isConnectedLocked()) { 252 break; 253 } 254 final long elapsedTimeMillis = SystemClock.uptimeMillis() - startTimeMillis; 255 final long remainingTimeMillis = CONNECT_TIMEOUT_MILLIS - elapsedTimeMillis; 256 if (remainingTimeMillis <= 0) { 257 throw new RuntimeException("Error while connecting UiAutomation"); 258 } 259 try { 260 mLock.wait(remainingTimeMillis); 261 } catch (InterruptedException ie) { 262 /* ignore */ 263 } 264 } 265 } finally { 266 mIsConnecting = false; 267 } 268 } 269 } 270 271 /** 272 * Get the flags used to connect the service. 273 * 274 * @return The flags used to connect 275 * 276 * @hide 277 */ getFlags()278 public int getFlags() { 279 return mFlags; 280 } 281 282 /** 283 * Disconnects this UiAutomation from the accessibility introspection APIs. 284 * 285 * @hide 286 */ 287 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) disconnect()288 public void disconnect() { 289 synchronized (mLock) { 290 if (mIsConnecting) { 291 throw new IllegalStateException( 292 "Cannot call disconnect() while connecting!"); 293 } 294 throwIfNotConnectedLocked(); 295 mConnectionId = CONNECTION_ID_UNDEFINED; 296 } 297 try { 298 // Calling out without a lock held. 299 mUiAutomationConnection.disconnect(); 300 } catch (RemoteException re) { 301 throw new RuntimeException("Error while disconnecting UiAutomation", re); 302 } finally { 303 mRemoteCallbackThread.quit(); 304 mRemoteCallbackThread = null; 305 } 306 } 307 308 /** 309 * The id of the {@link IAccessibilityInteractionConnection} for querying 310 * the screen content. This is here for legacy purposes since some tools use 311 * hidden APIs to introspect the screen. 312 * 313 * @hide 314 */ getConnectionId()315 public int getConnectionId() { 316 synchronized (mLock) { 317 throwIfNotConnectedLocked(); 318 return mConnectionId; 319 } 320 } 321 322 /** 323 * Reports if the object has been destroyed 324 * 325 * @return {code true} if the object has been destroyed. 326 * 327 * @hide 328 */ isDestroyed()329 public boolean isDestroyed() { 330 return mIsDestroyed; 331 } 332 333 /** 334 * Sets a callback for observing the stream of {@link AccessibilityEvent}s. 335 * The callbacks are delivered on the main application thread. 336 * 337 * @param listener The callback. 338 */ setOnAccessibilityEventListener(OnAccessibilityEventListener listener)339 public void setOnAccessibilityEventListener(OnAccessibilityEventListener listener) { 340 synchronized (mLock) { 341 mOnAccessibilityEventListener = listener; 342 } 343 } 344 345 /** 346 * Destroy this UiAutomation. After calling this method, attempting to use the object will 347 * result in errors. 348 * 349 * @hide 350 */ 351 @TestApi destroy()352 public void destroy() { 353 disconnect(); 354 mIsDestroyed = true; 355 } 356 357 /** 358 * Adopt the permission identity of the shell UID for all permissions. This allows 359 * you to call APIs protected permissions which normal apps cannot hold but are 360 * granted to the shell UID. If you already adopted all shell permissions by calling 361 * this method or {@link #adoptShellPermissionIdentity(String...)} a subsequent call 362 * would be a no-op. Note that your permission state becomes that of the shell UID 363 * and it is not a combination of your and the shell UID permissions. 364 * <p> 365 * <strong>Note:<strong/> Calling this method adopts all shell permissions and overrides 366 * any subset of adopted permissions via {@link #adoptShellPermissionIdentity(String...)}. 367 * 368 * @see #adoptShellPermissionIdentity(String...) 369 * @see #dropShellPermissionIdentity() 370 */ adoptShellPermissionIdentity()371 public void adoptShellPermissionIdentity() { 372 synchronized (mLock) { 373 throwIfNotConnectedLocked(); 374 } 375 try { 376 // Calling out without a lock held. 377 mUiAutomationConnection.adoptShellPermissionIdentity(Process.myUid(), null); 378 } catch (RemoteException re) { 379 Log.e(LOG_TAG, "Error executing adopting shell permission identity!", re); 380 } 381 } 382 383 /** 384 * Adopt the permission identity of the shell UID only for the provided permissions. 385 * This allows you to call APIs protected permissions which normal apps cannot hold 386 * but are granted to the shell UID. If you already adopted the specified shell 387 * permissions by calling this method or {@link #adoptShellPermissionIdentity()} a 388 * subsequent call would be a no-op. Note that your permission state becomes that of the 389 * shell UID and it is not a combination of your and the shell UID permissions. 390 * <p> 391 * <strong>Note:<strong/> Calling this method adopts only the specified shell permissions 392 * and overrides all adopted permissions via {@link #adoptShellPermissionIdentity()}. 393 * 394 * @param permissions The permissions to adopt or <code>null</code> to adopt all. 395 * 396 * @see #adoptShellPermissionIdentity() 397 * @see #dropShellPermissionIdentity() 398 */ adoptShellPermissionIdentity(@ullable String... permissions)399 public void adoptShellPermissionIdentity(@Nullable String... permissions) { 400 synchronized (mLock) { 401 throwIfNotConnectedLocked(); 402 } 403 try { 404 // Calling out without a lock held. 405 mUiAutomationConnection.adoptShellPermissionIdentity(Process.myUid(), permissions); 406 } catch (RemoteException re) { 407 Log.e(LOG_TAG, "Error executing adopting shell permission identity!", re); 408 } 409 } 410 411 /** 412 * Drop the shell permission identity adopted by a previous call to 413 * {@link #adoptShellPermissionIdentity()}. If you did not adopt the shell permission 414 * identity this method would be a no-op. 415 * 416 * @see #adoptShellPermissionIdentity() 417 */ dropShellPermissionIdentity()418 public void dropShellPermissionIdentity() { 419 synchronized (mLock) { 420 throwIfNotConnectedLocked(); 421 } 422 try { 423 // Calling out without a lock held. 424 mUiAutomationConnection.dropShellPermissionIdentity(); 425 } catch (RemoteException re) { 426 Log.e(LOG_TAG, "Error executing dropping shell permission identity!", re); 427 } 428 } 429 430 /** 431 * Performs a global action. Such an action can be performed at any moment 432 * regardless of the current application or user location in that application. 433 * For example going back, going home, opening recents, etc. 434 * 435 * @param action The action to perform. 436 * @return Whether the action was successfully performed. 437 * 438 * @see android.accessibilityservice.AccessibilityService#GLOBAL_ACTION_BACK 439 * @see android.accessibilityservice.AccessibilityService#GLOBAL_ACTION_HOME 440 * @see android.accessibilityservice.AccessibilityService#GLOBAL_ACTION_NOTIFICATIONS 441 * @see android.accessibilityservice.AccessibilityService#GLOBAL_ACTION_RECENTS 442 */ performGlobalAction(int action)443 public final boolean performGlobalAction(int action) { 444 final IAccessibilityServiceConnection connection; 445 synchronized (mLock) { 446 throwIfNotConnectedLocked(); 447 connection = AccessibilityInteractionClient.getInstance() 448 .getConnection(mConnectionId); 449 } 450 // Calling out without a lock held. 451 if (connection != null) { 452 try { 453 return connection.performGlobalAction(action); 454 } catch (RemoteException re) { 455 Log.w(LOG_TAG, "Error while calling performGlobalAction", re); 456 } 457 } 458 return false; 459 } 460 461 /** 462 * Find the view that has the specified focus type. The search is performed 463 * across all windows. 464 * <p> 465 * <strong>Note:</strong> In order to access the windows you have to opt-in 466 * to retrieve the interactive windows by setting the 467 * {@link AccessibilityServiceInfo#FLAG_RETRIEVE_INTERACTIVE_WINDOWS} flag. 468 * Otherwise, the search will be performed only in the active window. 469 * </p> 470 * 471 * @param focus The focus to find. One of {@link AccessibilityNodeInfo#FOCUS_INPUT} or 472 * {@link AccessibilityNodeInfo#FOCUS_ACCESSIBILITY}. 473 * @return The node info of the focused view or null. 474 * 475 * @see AccessibilityNodeInfo#FOCUS_INPUT 476 * @see AccessibilityNodeInfo#FOCUS_ACCESSIBILITY 477 */ findFocus(int focus)478 public AccessibilityNodeInfo findFocus(int focus) { 479 return AccessibilityInteractionClient.getInstance().findFocus(mConnectionId, 480 AccessibilityWindowInfo.ANY_WINDOW_ID, AccessibilityNodeInfo.ROOT_NODE_ID, focus); 481 } 482 483 /** 484 * Gets the an {@link AccessibilityServiceInfo} describing this UiAutomation. 485 * This method is useful if one wants to change some of the dynamically 486 * configurable properties at runtime. 487 * 488 * @return The accessibility service info. 489 * 490 * @see AccessibilityServiceInfo 491 */ getServiceInfo()492 public final AccessibilityServiceInfo getServiceInfo() { 493 final IAccessibilityServiceConnection connection; 494 synchronized (mLock) { 495 throwIfNotConnectedLocked(); 496 connection = AccessibilityInteractionClient.getInstance() 497 .getConnection(mConnectionId); 498 } 499 // Calling out without a lock held. 500 if (connection != null) { 501 try { 502 return connection.getServiceInfo(); 503 } catch (RemoteException re) { 504 Log.w(LOG_TAG, "Error while getting AccessibilityServiceInfo", re); 505 } 506 } 507 return null; 508 } 509 510 /** 511 * Sets the {@link AccessibilityServiceInfo} that describes how this 512 * UiAutomation will be handled by the platform accessibility layer. 513 * 514 * @param info The info. 515 * 516 * @see AccessibilityServiceInfo 517 */ setServiceInfo(AccessibilityServiceInfo info)518 public final void setServiceInfo(AccessibilityServiceInfo info) { 519 final IAccessibilityServiceConnection connection; 520 synchronized (mLock) { 521 throwIfNotConnectedLocked(); 522 AccessibilityInteractionClient.getInstance().clearCache(); 523 connection = AccessibilityInteractionClient.getInstance() 524 .getConnection(mConnectionId); 525 } 526 // Calling out without a lock held. 527 if (connection != null) { 528 try { 529 connection.setServiceInfo(info); 530 } catch (RemoteException re) { 531 Log.w(LOG_TAG, "Error while setting AccessibilityServiceInfo", re); 532 } 533 } 534 } 535 536 /** 537 * Gets the windows on the screen. This method returns only the windows 538 * that a sighted user can interact with, as opposed to all windows. 539 * For example, if there is a modal dialog shown and the user cannot touch 540 * anything behind it, then only the modal window will be reported 541 * (assuming it is the top one). For convenience the returned windows 542 * are ordered in a descending layer order, which is the windows that 543 * are higher in the Z-order are reported first. 544 * <p> 545 * <strong>Note:</strong> In order to access the windows you have to opt-in 546 * to retrieve the interactive windows by setting the 547 * {@link AccessibilityServiceInfo#FLAG_RETRIEVE_INTERACTIVE_WINDOWS} flag. 548 * </p> 549 * 550 * @return The windows if there are windows such, otherwise an empty list. 551 */ getWindows()552 public List<AccessibilityWindowInfo> getWindows() { 553 final int connectionId; 554 synchronized (mLock) { 555 throwIfNotConnectedLocked(); 556 connectionId = mConnectionId; 557 } 558 // Calling out without a lock held. 559 return AccessibilityInteractionClient.getInstance() 560 .getWindows(connectionId); 561 } 562 563 /** 564 * Gets the root {@link AccessibilityNodeInfo} in the active window. 565 * 566 * @return The root info. 567 */ getRootInActiveWindow()568 public AccessibilityNodeInfo getRootInActiveWindow() { 569 final int connectionId; 570 synchronized (mLock) { 571 throwIfNotConnectedLocked(); 572 connectionId = mConnectionId; 573 } 574 // Calling out without a lock held. 575 return AccessibilityInteractionClient.getInstance() 576 .getRootInActiveWindow(connectionId); 577 } 578 579 /** 580 * A method for injecting an arbitrary input event. 581 * <p> 582 * <strong>Note:</strong> It is caller's responsibility to recycle the event. 583 * </p> 584 * @param event The event to inject. 585 * @param sync Whether to inject the event synchronously. 586 * @return Whether event injection succeeded. 587 */ injectInputEvent(InputEvent event, boolean sync)588 public boolean injectInputEvent(InputEvent event, boolean sync) { 589 synchronized (mLock) { 590 throwIfNotConnectedLocked(); 591 } 592 try { 593 if (DEBUG) { 594 Log.i(LOG_TAG, "Injecting: " + event + " sync: " + sync); 595 } 596 // Calling out without a lock held. 597 return mUiAutomationConnection.injectInputEvent(event, sync); 598 } catch (RemoteException re) { 599 Log.e(LOG_TAG, "Error while injecting input event!", re); 600 } 601 return false; 602 } 603 604 /** 605 * A request for WindowManagerService to wait until all animations have completed and input 606 * information has been sent from WindowManager to native InputManager. 607 * 608 * @hide 609 */ 610 @TestApi syncInputTransactions()611 public void syncInputTransactions() { 612 synchronized (mLock) { 613 throwIfNotConnectedLocked(); 614 } 615 try { 616 // Calling out without a lock held. 617 mUiAutomationConnection.syncInputTransactions(); 618 } catch (RemoteException re) { 619 Log.e(LOG_TAG, "Error while syncing input transactions!", re); 620 } 621 } 622 623 /** 624 * Sets the device rotation. A client can freeze the rotation in 625 * desired state or freeze the rotation to its current state or 626 * unfreeze the rotation (rotating the device changes its rotation 627 * state). 628 * 629 * @param rotation The desired rotation. 630 * @return Whether the rotation was set successfully. 631 * 632 * @see #ROTATION_FREEZE_0 633 * @see #ROTATION_FREEZE_90 634 * @see #ROTATION_FREEZE_180 635 * @see #ROTATION_FREEZE_270 636 * @see #ROTATION_FREEZE_CURRENT 637 * @see #ROTATION_UNFREEZE 638 */ setRotation(int rotation)639 public boolean setRotation(int rotation) { 640 synchronized (mLock) { 641 throwIfNotConnectedLocked(); 642 } 643 switch (rotation) { 644 case ROTATION_FREEZE_0: 645 case ROTATION_FREEZE_90: 646 case ROTATION_FREEZE_180: 647 case ROTATION_FREEZE_270: 648 case ROTATION_UNFREEZE: 649 case ROTATION_FREEZE_CURRENT: { 650 try { 651 // Calling out without a lock held. 652 mUiAutomationConnection.setRotation(rotation); 653 return true; 654 } catch (RemoteException re) { 655 Log.e(LOG_TAG, "Error while setting rotation!", re); 656 } 657 } return false; 658 default: { 659 throw new IllegalArgumentException("Invalid rotation."); 660 } 661 } 662 } 663 664 /** 665 * Executes a command and waits for a specific accessibility event up to a 666 * given wait timeout. To detect a sequence of events one can implement a 667 * filter that keeps track of seen events of the expected sequence and 668 * returns true after the last event of that sequence is received. 669 * <p> 670 * <strong>Note:</strong> It is caller's responsibility to recycle the returned event. 671 * </p> 672 * @param command The command to execute. 673 * @param filter Filter that recognizes the expected event. 674 * @param timeoutMillis The wait timeout in milliseconds. 675 * 676 * @throws TimeoutException If the expected event is not received within the timeout. 677 */ executeAndWaitForEvent(Runnable command, AccessibilityEventFilter filter, long timeoutMillis)678 public AccessibilityEvent executeAndWaitForEvent(Runnable command, 679 AccessibilityEventFilter filter, long timeoutMillis) throws TimeoutException { 680 // Acquire the lock and prepare for receiving events. 681 synchronized (mLock) { 682 throwIfNotConnectedLocked(); 683 mEventQueue.clear(); 684 // Prepare to wait for an event. 685 mWaitingForEventDelivery = true; 686 } 687 688 // Note: We have to release the lock since calling out with this lock held 689 // can bite. We will correctly filter out events from other interactions, 690 // so starting to collect events before running the action is just fine. 691 692 // We will ignore events from previous interactions. 693 final long executionStartTimeMillis = SystemClock.uptimeMillis(); 694 // Execute the command *without* the lock being held. 695 command.run(); 696 697 List<AccessibilityEvent> receivedEvents = new ArrayList<>(); 698 699 // Acquire the lock and wait for the event. 700 try { 701 // Wait for the event. 702 final long startTimeMillis = SystemClock.uptimeMillis(); 703 while (true) { 704 List<AccessibilityEvent> localEvents = new ArrayList<>(); 705 synchronized (mLock) { 706 localEvents.addAll(mEventQueue); 707 mEventQueue.clear(); 708 } 709 // Drain the event queue 710 while (!localEvents.isEmpty()) { 711 AccessibilityEvent event = localEvents.remove(0); 712 // Ignore events from previous interactions. 713 if (event.getEventTime() < executionStartTimeMillis) { 714 continue; 715 } 716 if (filter.accept(event)) { 717 return event; 718 } 719 receivedEvents.add(event); 720 } 721 // Check if timed out and if not wait. 722 final long elapsedTimeMillis = SystemClock.uptimeMillis() - startTimeMillis; 723 final long remainingTimeMillis = timeoutMillis - elapsedTimeMillis; 724 if (remainingTimeMillis <= 0) { 725 throw new TimeoutException("Expected event not received within: " 726 + timeoutMillis + " ms among: " + receivedEvents); 727 } 728 synchronized (mLock) { 729 if (mEventQueue.isEmpty()) { 730 try { 731 mLock.wait(remainingTimeMillis); 732 } catch (InterruptedException ie) { 733 /* ignore */ 734 } 735 } 736 } 737 } 738 } finally { 739 int size = receivedEvents.size(); 740 for (int i = 0; i < size; i++) { 741 receivedEvents.get(i).recycle(); 742 } 743 744 synchronized (mLock) { 745 mWaitingForEventDelivery = false; 746 mEventQueue.clear(); 747 mLock.notifyAll(); 748 } 749 } 750 } 751 752 /** 753 * Waits for the accessibility event stream to become idle, which is not to 754 * have received an accessibility event within <code>idleTimeoutMillis</code>. 755 * The total time spent to wait for an idle accessibility event stream is bounded 756 * by the <code>globalTimeoutMillis</code>. 757 * 758 * @param idleTimeoutMillis The timeout in milliseconds between two events 759 * to consider the device idle. 760 * @param globalTimeoutMillis The maximal global timeout in milliseconds in 761 * which to wait for an idle state. 762 * 763 * @throws TimeoutException If no idle state was detected within 764 * <code>globalTimeoutMillis.</code> 765 */ waitForIdle(long idleTimeoutMillis, long globalTimeoutMillis)766 public void waitForIdle(long idleTimeoutMillis, long globalTimeoutMillis) 767 throws TimeoutException { 768 synchronized (mLock) { 769 throwIfNotConnectedLocked(); 770 771 final long startTimeMillis = SystemClock.uptimeMillis(); 772 if (mLastEventTimeMillis <= 0) { 773 mLastEventTimeMillis = startTimeMillis; 774 } 775 776 while (true) { 777 final long currentTimeMillis = SystemClock.uptimeMillis(); 778 // Did we get idle state within the global timeout? 779 final long elapsedGlobalTimeMillis = currentTimeMillis - startTimeMillis; 780 final long remainingGlobalTimeMillis = 781 globalTimeoutMillis - elapsedGlobalTimeMillis; 782 if (remainingGlobalTimeMillis <= 0) { 783 throw new TimeoutException("No idle state with idle timeout: " 784 + idleTimeoutMillis + " within global timeout: " 785 + globalTimeoutMillis); 786 } 787 // Did we get an idle state within the idle timeout? 788 final long elapsedIdleTimeMillis = currentTimeMillis - mLastEventTimeMillis; 789 final long remainingIdleTimeMillis = idleTimeoutMillis - elapsedIdleTimeMillis; 790 if (remainingIdleTimeMillis <= 0) { 791 return; 792 } 793 try { 794 mLock.wait(remainingIdleTimeMillis); 795 } catch (InterruptedException ie) { 796 /* ignore */ 797 } 798 } 799 } 800 } 801 802 /** 803 * Takes a screenshot. 804 * 805 * @return The screenshot bitmap on success, null otherwise. 806 */ takeScreenshot()807 public Bitmap takeScreenshot() { 808 synchronized (mLock) { 809 throwIfNotConnectedLocked(); 810 } 811 Display display = DisplayManagerGlobal.getInstance() 812 .getRealDisplay(Display.DEFAULT_DISPLAY); 813 Point displaySize = new Point(); 814 display.getRealSize(displaySize); 815 816 int rotation = display.getRotation(); 817 818 // Take the screenshot 819 Bitmap screenShot = null; 820 try { 821 // Calling out without a lock held. 822 screenShot = mUiAutomationConnection.takeScreenshot( 823 new Rect(0, 0, displaySize.x, displaySize.y), rotation); 824 if (screenShot == null) { 825 return null; 826 } 827 } catch (RemoteException re) { 828 Log.e(LOG_TAG, "Error while taking screnshot!", re); 829 return null; 830 } 831 832 // Optimization 833 screenShot.setHasAlpha(false); 834 835 return screenShot; 836 } 837 838 /** 839 * Sets whether this UiAutomation to run in a "monkey" mode. Applications can query whether 840 * they are executed in a "monkey" mode, i.e. run by a test framework, and avoid doing 841 * potentially undesirable actions such as calling 911 or posting on public forums etc. 842 * 843 * @param enable whether to run in a "monkey" mode or not. Default is not. 844 * @see ActivityManager#isUserAMonkey() 845 */ setRunAsMonkey(boolean enable)846 public void setRunAsMonkey(boolean enable) { 847 synchronized (mLock) { 848 throwIfNotConnectedLocked(); 849 } 850 try { 851 ActivityManager.getService().setUserIsMonkey(enable); 852 } catch (RemoteException re) { 853 Log.e(LOG_TAG, "Error while setting run as monkey!", re); 854 } 855 } 856 857 /** 858 * Clears the frame statistics for the content of a given window. These 859 * statistics contain information about the most recently rendered content 860 * frames. 861 * 862 * @param windowId The window id. 863 * @return Whether the window is present and its frame statistics 864 * were cleared. 865 * 866 * @see android.view.WindowContentFrameStats 867 * @see #getWindowContentFrameStats(int) 868 * @see #getWindows() 869 * @see AccessibilityWindowInfo#getId() AccessibilityWindowInfo.getId() 870 */ clearWindowContentFrameStats(int windowId)871 public boolean clearWindowContentFrameStats(int windowId) { 872 synchronized (mLock) { 873 throwIfNotConnectedLocked(); 874 } 875 try { 876 if (DEBUG) { 877 Log.i(LOG_TAG, "Clearing content frame stats for window: " + windowId); 878 } 879 // Calling out without a lock held. 880 return mUiAutomationConnection.clearWindowContentFrameStats(windowId); 881 } catch (RemoteException re) { 882 Log.e(LOG_TAG, "Error clearing window content frame stats!", re); 883 } 884 return false; 885 } 886 887 /** 888 * Gets the frame statistics for a given window. These statistics contain 889 * information about the most recently rendered content frames. 890 * <p> 891 * A typical usage requires clearing the window frame statistics via {@link 892 * #clearWindowContentFrameStats(int)} followed by an interaction with the UI and 893 * finally getting the window frame statistics via calling this method. 894 * </p> 895 * <pre> 896 * // Assume we have at least one window. 897 * final int windowId = getWindows().get(0).getId(); 898 * 899 * // Start with a clean slate. 900 * uiAutimation.clearWindowContentFrameStats(windowId); 901 * 902 * // Do stuff with the UI. 903 * 904 * // Get the frame statistics. 905 * WindowContentFrameStats stats = uiAutomation.getWindowContentFrameStats(windowId); 906 * </pre> 907 * 908 * @param windowId The window id. 909 * @return The window frame statistics, or null if the window is not present. 910 * 911 * @see android.view.WindowContentFrameStats 912 * @see #clearWindowContentFrameStats(int) 913 * @see #getWindows() 914 * @see AccessibilityWindowInfo#getId() AccessibilityWindowInfo.getId() 915 */ getWindowContentFrameStats(int windowId)916 public WindowContentFrameStats getWindowContentFrameStats(int windowId) { 917 synchronized (mLock) { 918 throwIfNotConnectedLocked(); 919 } 920 try { 921 if (DEBUG) { 922 Log.i(LOG_TAG, "Getting content frame stats for window: " + windowId); 923 } 924 // Calling out without a lock held. 925 return mUiAutomationConnection.getWindowContentFrameStats(windowId); 926 } catch (RemoteException re) { 927 Log.e(LOG_TAG, "Error getting window content frame stats!", re); 928 } 929 return null; 930 } 931 932 /** 933 * Clears the window animation rendering statistics. These statistics contain 934 * information about the most recently rendered window animation frames, i.e. 935 * for window transition animations. 936 * 937 * @see android.view.WindowAnimationFrameStats 938 * @see #getWindowAnimationFrameStats() 939 * @see android.R.styleable#WindowAnimation 940 */ clearWindowAnimationFrameStats()941 public void clearWindowAnimationFrameStats() { 942 synchronized (mLock) { 943 throwIfNotConnectedLocked(); 944 } 945 try { 946 if (DEBUG) { 947 Log.i(LOG_TAG, "Clearing window animation frame stats"); 948 } 949 // Calling out without a lock held. 950 mUiAutomationConnection.clearWindowAnimationFrameStats(); 951 } catch (RemoteException re) { 952 Log.e(LOG_TAG, "Error clearing window animation frame stats!", re); 953 } 954 } 955 956 /** 957 * Gets the window animation frame statistics. These statistics contain 958 * information about the most recently rendered window animation frames, i.e. 959 * for window transition animations. 960 * 961 * <p> 962 * A typical usage requires clearing the window animation frame statistics via 963 * {@link #clearWindowAnimationFrameStats()} followed by an interaction that causes 964 * a window transition which uses a window animation and finally getting the window 965 * animation frame statistics by calling this method. 966 * </p> 967 * <pre> 968 * // Start with a clean slate. 969 * uiAutimation.clearWindowAnimationFrameStats(); 970 * 971 * // Do stuff to trigger a window transition. 972 * 973 * // Get the frame statistics. 974 * WindowAnimationFrameStats stats = uiAutomation.getWindowAnimationFrameStats(); 975 * </pre> 976 * 977 * @return The window animation frame statistics. 978 * 979 * @see android.view.WindowAnimationFrameStats 980 * @see #clearWindowAnimationFrameStats() 981 * @see android.R.styleable#WindowAnimation 982 */ getWindowAnimationFrameStats()983 public WindowAnimationFrameStats getWindowAnimationFrameStats() { 984 synchronized (mLock) { 985 throwIfNotConnectedLocked(); 986 } 987 try { 988 if (DEBUG) { 989 Log.i(LOG_TAG, "Getting window animation frame stats"); 990 } 991 // Calling out without a lock held. 992 return mUiAutomationConnection.getWindowAnimationFrameStats(); 993 } catch (RemoteException re) { 994 Log.e(LOG_TAG, "Error getting window animation frame stats!", re); 995 } 996 return null; 997 } 998 999 /** 1000 * Grants a runtime permission to a package. 1001 * @param packageName The package to which to grant. 1002 * @param permission The permission to grant. 1003 * @throws SecurityException if unable to grant the permission. 1004 */ grantRuntimePermission(String packageName, String permission)1005 public void grantRuntimePermission(String packageName, String permission) { 1006 grantRuntimePermissionAsUser(packageName, permission, android.os.Process.myUserHandle()); 1007 } 1008 1009 /** 1010 * @deprecated replaced by 1011 * {@link #grantRuntimePermissionAsUser(String, String, UserHandle)}. 1012 * @hide 1013 */ 1014 @Deprecated 1015 @TestApi grantRuntimePermission(String packageName, String permission, UserHandle userHandle)1016 public boolean grantRuntimePermission(String packageName, String permission, 1017 UserHandle userHandle) { 1018 grantRuntimePermissionAsUser(packageName, permission, userHandle); 1019 return true; 1020 } 1021 1022 /** 1023 * Grants a runtime permission to a package for a user. 1024 * @param packageName The package to which to grant. 1025 * @param permission The permission to grant. 1026 * @throws SecurityException if unable to grant the permission. 1027 */ grantRuntimePermissionAsUser(String packageName, String permission, UserHandle userHandle)1028 public void grantRuntimePermissionAsUser(String packageName, String permission, 1029 UserHandle userHandle) { 1030 synchronized (mLock) { 1031 throwIfNotConnectedLocked(); 1032 } 1033 try { 1034 if (DEBUG) { 1035 Log.i(LOG_TAG, "Granting runtime permission"); 1036 } 1037 // Calling out without a lock held. 1038 mUiAutomationConnection.grantRuntimePermission(packageName, 1039 permission, userHandle.getIdentifier()); 1040 } catch (Exception e) { 1041 throw new SecurityException("Error granting runtime permission", e); 1042 } 1043 } 1044 1045 /** 1046 * Revokes a runtime permission from a package. 1047 * @param packageName The package to which to grant. 1048 * @param permission The permission to grant. 1049 * @throws SecurityException if unable to revoke the permission. 1050 */ revokeRuntimePermission(String packageName, String permission)1051 public void revokeRuntimePermission(String packageName, String permission) { 1052 revokeRuntimePermissionAsUser(packageName, permission, android.os.Process.myUserHandle()); 1053 } 1054 1055 /** 1056 * @deprecated replaced by 1057 * {@link #revokeRuntimePermissionAsUser(String, String, UserHandle)}. 1058 * @hide 1059 */ 1060 @Deprecated 1061 @TestApi revokeRuntimePermission(String packageName, String permission, UserHandle userHandle)1062 public boolean revokeRuntimePermission(String packageName, String permission, 1063 UserHandle userHandle) { 1064 revokeRuntimePermissionAsUser(packageName, permission, userHandle); 1065 return true; 1066 } 1067 1068 /** 1069 * Revokes a runtime permission from a package. 1070 * @param packageName The package to which to grant. 1071 * @param permission The permission to grant. 1072 * @throws SecurityException if unable to revoke the permission. 1073 */ revokeRuntimePermissionAsUser(String packageName, String permission, UserHandle userHandle)1074 public void revokeRuntimePermissionAsUser(String packageName, String permission, 1075 UserHandle userHandle) { 1076 synchronized (mLock) { 1077 throwIfNotConnectedLocked(); 1078 } 1079 try { 1080 if (DEBUG) { 1081 Log.i(LOG_TAG, "Revoking runtime permission"); 1082 } 1083 // Calling out without a lock held. 1084 mUiAutomationConnection.revokeRuntimePermission(packageName, 1085 permission, userHandle.getIdentifier()); 1086 } catch (Exception e) { 1087 throw new SecurityException("Error granting runtime permission", e); 1088 } 1089 } 1090 1091 /** 1092 * Executes a shell command. This method returns a file descriptor that points 1093 * to the standard output stream. The command execution is similar to running 1094 * "adb shell <command>" from a host connected to the device. 1095 * <p> 1096 * <strong>Note:</strong> It is your responsibility to close the returned file 1097 * descriptor once you are done reading. 1098 * </p> 1099 * 1100 * @param command The command to execute. 1101 * @return A file descriptor to the standard output stream. 1102 * 1103 * @see #adoptShellPermissionIdentity() 1104 */ executeShellCommand(String command)1105 public ParcelFileDescriptor executeShellCommand(String command) { 1106 synchronized (mLock) { 1107 throwIfNotConnectedLocked(); 1108 } 1109 warnIfBetterCommand(command); 1110 1111 ParcelFileDescriptor source = null; 1112 ParcelFileDescriptor sink = null; 1113 1114 try { 1115 ParcelFileDescriptor[] pipe = ParcelFileDescriptor.createPipe(); 1116 source = pipe[0]; 1117 sink = pipe[1]; 1118 1119 // Calling out without a lock held. 1120 mUiAutomationConnection.executeShellCommand(command, sink, null); 1121 } catch (IOException ioe) { 1122 Log.e(LOG_TAG, "Error executing shell command!", ioe); 1123 } catch (RemoteException re) { 1124 Log.e(LOG_TAG, "Error executing shell command!", re); 1125 } finally { 1126 IoUtils.closeQuietly(sink); 1127 } 1128 1129 return source; 1130 } 1131 1132 /** 1133 * Executes a shell command. This method returns two file descriptors, 1134 * one that points to the standard output stream (element at index 0), and one that points 1135 * to the standard input stream (element at index 1). The command execution is similar 1136 * to running "adb shell <command>" from a host connected to the device. 1137 * <p> 1138 * <strong>Note:</strong> It is your responsibility to close the returned file 1139 * descriptors once you are done reading/writing. 1140 * </p> 1141 * 1142 * @param command The command to execute. 1143 * @return File descriptors (out, in) to the standard output/input streams. 1144 * 1145 * @hide 1146 */ 1147 @TestApi executeShellCommandRw(String command)1148 public ParcelFileDescriptor[] executeShellCommandRw(String command) { 1149 synchronized (mLock) { 1150 throwIfNotConnectedLocked(); 1151 } 1152 warnIfBetterCommand(command); 1153 1154 ParcelFileDescriptor source_read = null; 1155 ParcelFileDescriptor sink_read = null; 1156 1157 ParcelFileDescriptor source_write = null; 1158 ParcelFileDescriptor sink_write = null; 1159 1160 try { 1161 ParcelFileDescriptor[] pipe_read = ParcelFileDescriptor.createPipe(); 1162 source_read = pipe_read[0]; 1163 sink_read = pipe_read[1]; 1164 1165 ParcelFileDescriptor[] pipe_write = ParcelFileDescriptor.createPipe(); 1166 source_write = pipe_write[0]; 1167 sink_write = pipe_write[1]; 1168 1169 // Calling out without a lock held. 1170 mUiAutomationConnection.executeShellCommand(command, sink_read, source_write); 1171 } catch (IOException ioe) { 1172 Log.e(LOG_TAG, "Error executing shell command!", ioe); 1173 } catch (RemoteException re) { 1174 Log.e(LOG_TAG, "Error executing shell command!", re); 1175 } finally { 1176 IoUtils.closeQuietly(sink_read); 1177 IoUtils.closeQuietly(source_write); 1178 } 1179 1180 ParcelFileDescriptor[] result = new ParcelFileDescriptor[2]; 1181 result[0] = source_read; 1182 result[1] = sink_write; 1183 return result; 1184 } 1185 isConnectedLocked()1186 private boolean isConnectedLocked() { 1187 return mConnectionId != CONNECTION_ID_UNDEFINED; 1188 } 1189 throwIfConnectedLocked()1190 private void throwIfConnectedLocked() { 1191 if (mConnectionId != CONNECTION_ID_UNDEFINED) { 1192 throw new IllegalStateException("UiAutomation not connected!"); 1193 } 1194 } 1195 throwIfNotConnectedLocked()1196 private void throwIfNotConnectedLocked() { 1197 if (!isConnectedLocked()) { 1198 throw new IllegalStateException("UiAutomation not connected!"); 1199 } 1200 } 1201 warnIfBetterCommand(String cmd)1202 private void warnIfBetterCommand(String cmd) { 1203 if (cmd.startsWith("pm grant ")) { 1204 Log.w(LOG_TAG, "UiAutomation.grantRuntimePermission() " 1205 + "is more robust and should be used instead of 'pm grant'"); 1206 } else if (cmd.startsWith("pm revoke ")) { 1207 Log.w(LOG_TAG, "UiAutomation.revokeRuntimePermission() " 1208 + "is more robust and should be used instead of 'pm revoke'"); 1209 } 1210 } 1211 1212 private class IAccessibilityServiceClientImpl extends IAccessibilityServiceClientWrapper { 1213 IAccessibilityServiceClientImpl(Looper looper)1214 public IAccessibilityServiceClientImpl(Looper looper) { 1215 super(null, looper, new Callbacks() { 1216 @Override 1217 public void init(int connectionId, IBinder windowToken) { 1218 synchronized (mLock) { 1219 mConnectionId = connectionId; 1220 mLock.notifyAll(); 1221 } 1222 } 1223 1224 @Override 1225 public void onServiceConnected() { 1226 /* do nothing */ 1227 } 1228 1229 @Override 1230 public void onInterrupt() { 1231 /* do nothing */ 1232 } 1233 1234 @Override 1235 public boolean onGesture(int gestureId) { 1236 /* do nothing */ 1237 return false; 1238 } 1239 1240 @Override 1241 public void onAccessibilityEvent(AccessibilityEvent event) { 1242 final OnAccessibilityEventListener listener; 1243 synchronized (mLock) { 1244 mLastEventTimeMillis = event.getEventTime(); 1245 if (mWaitingForEventDelivery) { 1246 mEventQueue.add(AccessibilityEvent.obtain(event)); 1247 } 1248 mLock.notifyAll(); 1249 listener = mOnAccessibilityEventListener; 1250 } 1251 if (listener != null) { 1252 // Calling out only without a lock held. 1253 mLocalCallbackHandler.sendMessage(PooledLambda.obtainMessage( 1254 OnAccessibilityEventListener::onAccessibilityEvent, 1255 listener, AccessibilityEvent.obtain(event))); 1256 } 1257 } 1258 1259 @Override 1260 public boolean onKeyEvent(KeyEvent event) { 1261 return false; 1262 } 1263 1264 @Override 1265 public void onMagnificationChanged(int displayId, @NonNull Region region, 1266 float scale, float centerX, float centerY) { 1267 /* do nothing */ 1268 } 1269 1270 @Override 1271 public void onSoftKeyboardShowModeChanged(int showMode) { 1272 /* do nothing */ 1273 } 1274 1275 @Override 1276 public void onPerformGestureResult(int sequence, boolean completedSuccessfully) { 1277 /* do nothing */ 1278 } 1279 1280 @Override 1281 public void onFingerprintCapturingGesturesChanged(boolean active) { 1282 /* do nothing */ 1283 } 1284 1285 @Override 1286 public void onFingerprintGesture(int gesture) { 1287 /* do nothing */ 1288 } 1289 1290 @Override 1291 public void onAccessibilityButtonClicked() { 1292 /* do nothing */ 1293 } 1294 1295 @Override 1296 public void onAccessibilityButtonAvailabilityChanged(boolean available) { 1297 /* do nothing */ 1298 } 1299 }); 1300 } 1301 } 1302 } 1303