1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_FRAMEWORKS_ML_NN_RUNTIME_CALLBACKS_H 18 #define ANDROID_FRAMEWORKS_ML_NN_RUNTIME_CALLBACKS_H 19 20 #include "HalInterfaces.h" 21 22 #include <android-base/thread_annotations.h> 23 #include <condition_variable> 24 #include <functional> 25 #include <mutex> 26 #include <thread> 27 #include <vector> 28 29 /* 30 * The Callback classes are used internally by the NeuralNetworks runtime to 31 * synchronize between different threads. An asynchronous task is launched 32 * paired with a callback object. When a client thread requires the output being 33 * generated by the asynchronous task, the client thread can wait for the result 34 * and be blocked until it has completed. Any wait may safely be called 35 * concurrently, even on the same callback object. When the asynchronous task 36 * has finished its workload, it must immediately call "notify*". If the 37 * asynchronous task has failed to launch, the function that tried to launch the 38 * asynchronous task must immediately call "notify*". This "notify*" call 39 * awakens any client threads waiting on the callback object. 40 * 41 * These classes exist to enable synchronization across HIDL. When 42 * synchronization is only required in the same process, consider using 43 * std::future, std::mutex, std::condition_variable, or std::experimental::latch 44 * instead. 45 */ 46 47 namespace android::nn { 48 49 /** 50 * The PreparedModelCallback class is used to receive the error status of 51 * preparing a model as well as the prepared model from a task executing 52 * asynchronously with respect to the runtime. If a calling thread calls wait 53 * or get* on a PreparedModelCallback object and the corresponding asynchronous 54 * task has not finished preparing the model, the calling thread will block 55 * until the asynchronous task has called notify*. 56 * 57 * If the callback object is notified more than once, only the results of the 58 * first call to notify* are used, and the results from subsequent calls are 59 * discarded. 60 * 61 * This callback object is passed as an argument to IDevice::prepareModel*. 62 */ 63 class PreparedModelCallback : public hal::IPreparedModelCallback { 64 public: 65 /** 66 * IPreparedModelCallback::notify marks the callback object with the return 67 * status of the asynchronous model preparation along with the prepared 68 * model, and allows all prior and future wait calls on the 69 * PreparedModelCallback object to proceed. 70 * 71 * One of IPreparedModelCallback::notify, IPreparedModelCallback::notify_1_2, 72 * or IPreparedModelCallback::notify_1_3 must be called on a given 73 * PreparedModelCallback object. 74 * 75 * If the callback object is notified more than once, only the results of 76 * the first call to notify* are used, and the results from subsequent calls 77 * are discarded. 78 * 79 * @param status Error status returned from asynchronously preparing the 80 * model; will be: 81 * - NONE if the asynchronous preparation was successful 82 * - DEVICE_UNAVAILABLE if driver is offline or busy 83 * - GENERAL_FAILURE if there is an unspecified error 84 * - INVALID_ARGUMENT if the input model is invalid 85 * @param preparedModel Returned model that has been prepared for execution, 86 * nullptr if the model was unable to be prepared. 87 */ 88 hal::Return<void> notify(hal::V1_0::ErrorStatus status, 89 const sp<hal::V1_0::IPreparedModel>& preparedModel) override; 90 91 /** 92 * IPreparedModelCallback::notify_1_2 marks the callback object with the 93 * return status of the asynchronous model preparation along with the 94 * prepared model, and allows all prior and future wait calls on the 95 * PreparedModelCallback object to proceed. 96 * 97 * One of IPreparedModelCallback::notify, IPreparedModelCallback::notify_1_2, 98 * or IPreparedModelCallback::notify_1_3 must be called on a given 99 * PreparedModelCallback object. 100 * 101 * If the callback object is notified more than once, only the results of 102 * the first call to notify* are used, and the results from subsequent calls 103 * are discarded. 104 * 105 * @param status Error status returned from asynchronously preparing the 106 * model; will be: 107 * - NONE if the asynchronous preparation was successful 108 * - DEVICE_UNAVAILABLE if driver is offline or busy 109 * - GENERAL_FAILURE if there is an unspecified error 110 * - INVALID_ARGUMENT if the input model is invalid 111 * @param preparedModel Returned model that has been prepared for execution, 112 * nullptr if the model was unable to be prepared. 113 */ 114 hal::Return<void> notify_1_2(hal::V1_0::ErrorStatus status, 115 const sp<hal::V1_2::IPreparedModel>& preparedModel) override; 116 117 /** 118 * IPreparedModelCallback::notify_1_3 marks the callback object with the 119 * return status of the asynchronous model preparation along with the 120 * prepared model, and allows all prior and future wait calls on the 121 * PreparedModelCallback object to proceed. 122 * 123 * One of IPreparedModelCallback::notify, IPreparedModelCallback::notify_1_2, 124 * or IPreparedModelCallback::notify_1_3 must be called on a given 125 * PreparedModelCallback object. 126 * 127 * If the callback object is notified more than once, only the results of 128 * the first call to notify* are used, and the results from subsequent calls 129 * are discarded. 130 * 131 * @param status Error status returned from asynchronously preparing the 132 * model; will be: 133 * - NONE if the asynchronous preparation was successful 134 * - DEVICE_UNAVAILABLE if driver is offline or busy 135 * - GENERAL_FAILURE if there is an unspecified error 136 * - INVALID_ARGUMENT if the input model is invalid 137 * - MISSED_DEADLINE_* if the deadline could not be met 138 * - RESOURCE_EXHAUSTED_* if the task was aborted by the driver 139 * @param preparedModel Returned model that has been prepared for execution, 140 * nullptr if the model was unable to be prepared. 141 */ 142 hal::Return<void> notify_1_3(hal::V1_3::ErrorStatus status, 143 const sp<hal::V1_3::IPreparedModel>& preparedModel) override; 144 145 /** 146 * Mark the callback object as a dead object. This acts as a call to notify. 147 */ 148 void notifyAsDeadObject(); 149 150 /** 151 * PreparedModelCallback::wait blocks until notify* has been called on the 152 * callback object. 153 */ 154 void wait() const; 155 156 /** 157 * Retrieves the error status returned from the asynchronous task launched 158 * by IDevice::prepareModel*. If IDevice::prepareModel* has not finished 159 * asynchronously preparing the model, this call will block until the 160 * asynchronous task notifies the object. 161 * 162 * @return status Error status returned from asynchronously preparing the 163 * model; will be: 164 * - NONE if the asynchronous preparation was successful 165 * - DEVICE_UNAVAILABLE if driver is offline or busy 166 * - GENERAL_FAILURE if there is an unspecified error 167 * - INVALID_ARGUMENT if the input model is invalid 168 * - MISSED_DEADLINE_* if the deadline could not be met 169 * - RESOURCE_EXHAUSTED_* if the task was aborted by the driver 170 * - DEAD_OBJECT if the driver crashed without returning a result 171 */ 172 hal::V1_3::ErrorStatus getStatus() const; 173 174 /** 175 * Retrieves the model that has been prepared for execution from the 176 * asynchronous task launched by IDevice::prepareModel*. If 177 * IDevice::prepareModel* has not finished asynchronously preparing the 178 * model, this call will block until the asynchronous task notifies the 179 * object. 180 * 181 * @return preparedModel Returned model that has been prepared for 182 * execution, nullptr if the model was unable to be prepared. 183 */ 184 sp<hal::V1_0::IPreparedModel> getPreparedModel() const; 185 186 /** 187 * Queries whether the object is dead. 188 * 189 * @return 'true' if dead, 'false' otherwise. 190 */ 191 bool isDeadObject() const; 192 193 private: 194 hal::Return<void> notifyInternal(bool deadObject, hal::ErrorStatus errorStatus, 195 const sp<hal::V1_0::IPreparedModel>& preparedModel); 196 197 mutable std::mutex mMutex; 198 mutable std::condition_variable mCondition; 199 bool mNotified GUARDED_BY(mMutex) = false; 200 bool mDeadObject = false; 201 hal::ErrorStatus mErrorStatus = hal::ErrorStatus::GENERAL_FAILURE; 202 sp<hal::V1_0::IPreparedModel> mPreparedModel; 203 }; 204 205 /** 206 * The ExecutionCallback class is used to receive the results of the execution 207 * from a task executing asynchronously with respect to the runtime. If a 208 * calling thread calls wait or get* on a ExecutionCallback object and the 209 * corresponding asynchronous task has not finished the execution, the calling 210 * thread will block until the asynchronous task has called one of the notify* 211 * methods. 212 * 213 * If the callback object is notified more than once, only the results of the 214 * first call to notify* are used, and the results from subsequent calls are 215 * discarded. 216 * 217 * This callback object is passed as an argument to IPreparedModel::execute*. 218 */ 219 class ExecutionCallback : public hal::IExecutionCallback { 220 using ExecutionFinish = 221 std::function<hal::ErrorStatus(hal::ErrorStatus, const std::vector<hal::OutputShape>&)>; 222 223 public: 224 /** 225 * IExecutionCallback::notify marks the callback object with the return 226 * status of the asynchronous execution that held this callback and enables 227 * all prior and future wait calls on the ExecutionCallback object to 228 * proceed. 229 * 230 * One of the IExecutionCallback::notify* methods must be called on a given 231 * ExecutionCallback object. 232 * 233 * If the callback object is notified more than once, only the results of 234 * the first call to notify* are used, and the results from subsequent calls 235 * are discarded. 236 * 237 * @param status Error status returned from launching the asynchronous task 238 * (if the launch fails) or from the asynchronous task itself (if the 239 * launch succeeds). Must be: 240 * - NONE if the asynchronous execution was successful 241 * - DEVICE_UNAVAILABLE if driver is offline or busy 242 * - GENERAL_FAILURE if there is an unspecified error 243 * - OUTPUT_INSUFFICIENT_SIZE if provided output buffer is not large 244 * enough to store the resultant values 245 * - INVALID_ARGUMENT if the input request is invalid 246 */ 247 hal::Return<void> notify(hal::V1_0::ErrorStatus status) override; 248 249 /** 250 * IExecutionCallback::notify_1_2 marks the callback object with the results 251 * (error status, dynamic output shapes, and timing information) of the 252 * asynchronous execution that held this callback and enables all prior and 253 * future wait calls on the ExecutionCallback object to proceed. 254 * 255 * One of the IExecutionCallback::notify* methods must be called on a given 256 * ExecutionCallback object. 257 * 258 * If the callback object is notified more than once, only the results of 259 * the first call to notify* are used, and the results from subsequent calls 260 * are discarded. 261 * 262 * @param status Error status returned from launching the asynchronous task 263 * (if the launch fails) or from the asynchronous task itself (if the 264 * launch succeeds). Must be: 265 * - NONE if the asynchronous execution was successful 266 * - DEVICE_UNAVAILABLE if driver is offline or busy 267 * - GENERAL_FAILURE if the asynchronous task resulted in an unspecified 268 * error 269 * - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is 270 * not large enough to store the corresponding output 271 * - INVALID_ARGUMENT if one of the input arguments to prepareModel is 272 * invalid 273 * @param outputShapes A list of shape information of model output operands. 274 * The index into "outputShapes" corresponds to the index of the output 275 * operand in the Request outputs vector. outputShapes must be empty 276 * unless the status is either NONE or OUTPUT_INSUFFICIENT_SIZE. 277 * @param Timing Duration of execution. Unless MeasureTiming::YES was passed 278 * when launching the execution and status is NONE, all times must be 279 * reported as UINT64_MAX. A driver may choose to report any time as 280 * UINT64_MAX, indicating that particular measurement is not available. 281 */ 282 hal::Return<void> notify_1_2(hal::V1_0::ErrorStatus status, 283 const hal::hidl_vec<hal::OutputShape>& outputShapes, 284 const hal::Timing& timing) override; 285 286 /** 287 * IExecutionCallback::notify_1_3 marks the callback object with the results 288 * (error status, dynamic output shapes, and timing information) of the 289 * asynchronous execution that held this callback and enables all prior and 290 * future wait calls on the ExecutionCallback object to proceed. 291 * 292 * One of the IExecutionCallback::notify* methods must be called on a given 293 * ExecutionCallback object. 294 * 295 * If the callback object is notified more than once, only the results of 296 * the first call to notify* are used, and the results from subsequent calls 297 * are discarded. 298 * 299 * @param status Error status returned from launching the asynchronous task 300 * (if the launch fails) or from the asynchronous task itself (if the 301 * launch succeeds). Must be: 302 * - NONE if the asynchronous execution was successful 303 * - DEVICE_UNAVAILABLE if driver is offline or busy 304 * - GENERAL_FAILURE if the asynchronous task resulted in an unspecified 305 * error 306 * - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is 307 * not large enough to store the corresponding output 308 * - INVALID_ARGUMENT if one of the input arguments to prepareModel is 309 * invalid 310 * - MISSED_DEADLINE_* if the deadline could not be met 311 * - RESOURCE_EXHAUSTED_* if the execution was aborted by the driver 312 * @param outputShapes A list of shape information of model output operands. 313 * The index into "outputShapes" corresponds to the index of the output 314 * operand in the Request outputs vector. outputShapes must be empty 315 * unless the status is either NONE or OUTPUT_INSUFFICIENT_SIZE. 316 * @param Timing Duration of execution. Unless MeasureTiming::YES was passed 317 * when launching the execution and status is NONE, all times must be 318 * reported as UINT64_MAX. A driver may choose to report any time as 319 * UINT64_MAX, indicating that particular measurement is not available. 320 */ 321 hal::Return<void> notify_1_3(hal::V1_3::ErrorStatus status, 322 const hal::hidl_vec<hal::OutputShape>& outputShapes, 323 const hal::Timing& timing) override; 324 325 // An overload of the latest notify interface to hide the version from ExecutionBuilder. notify(hal::V1_3::ErrorStatus status,const hal::hidl_vec<hal::OutputShape> & outputShapes,const hal::Timing & timing)326 hal::Return<void> notify(hal::V1_3::ErrorStatus status, 327 const hal::hidl_vec<hal::OutputShape>& outputShapes, 328 const hal::Timing& timing) { 329 return notify_1_3(status, outputShapes, timing); 330 } 331 332 /** 333 * Mark the callback object as a dead object. This acts as a call to notify. 334 */ 335 void notifyAsDeadObject(); 336 337 /** 338 * ExecutionCallback::wait blocks until notify* has been called on the 339 * callback object. 340 */ 341 void wait() const; 342 343 /** 344 * Retrieves the error status returned from the asynchronous task launched 345 * by IPreparedModel::execute* (but not by 346 * IPreparedModel::executeSynchronously*). If IPreparedModel::execute* has 347 * not finished asynchronously executing, this call will block until the 348 * asynchronous task notifies the object. 349 * 350 * @return status Error status returned from launching the asynchronous task 351 * (if the launch fails) or from the asynchronous task itself (if the 352 * launch succeeds). Must be: 353 * - NONE if the asynchronous execution was successful 354 * - DEVICE_UNAVAILABLE if driver is offline or busy 355 * - GENERAL_FAILURE if the asynchronous task resulted in an unspecified 356 * error 357 * - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is 358 * not large enough to store the corresponding output 359 * - INVALID_ARGUMENT if one of the input arguments to prepareModel is 360 * invalid 361 * - MISSED_DEADLINE_* if the deadline could not be met 362 * - RESOURCE_EXHAUSTED_* if the task was aborted by the driver 363 * - DEAD_OBJECT if the driver crashed without returning a result 364 */ 365 hal::V1_3::ErrorStatus getStatus() const; 366 367 /** 368 * Retrieves the output shapes returned from the asynchronous task launched 369 * by either IPreparedModel::execute_1_2 or IPreparedModel::execute_1_3. If 370 * IPreparedModel::execute_1_2 or IPreparedModel::execute_1_3 has not 371 * finished asynchronously executing, this call will block until the 372 * asynchronous task notifies the object. 373 * 374 * If the asynchronous task was launched by IPreparedModel::execute, an 375 * empty vector will be returned. 376 * 377 * @return outputShapes A list of shape information of model output 378 * operands. The index into "outputShapes" corresponds to the index of 379 * the output operand in the Request outputs vector. outputShapes must 380 * be empty unless the status is either NONE or 381 * OUTPUT_INSUFFICIENT_SIZE. outputShaps may be empty if the status is 382 * NONE and all model output operands are fully-specified at execution 383 * time. outputShapes must have the same number of elements as the 384 * number of model output operands if the status is 385 * OUTPUT_INSUFFICIENT_SIZE, or if the status is NONE and the model has 386 * at least one output operand that is not fully-specified. 387 */ 388 const std::vector<hal::OutputShape>& getOutputShapes() const; 389 390 /** 391 * Retrieves the duration of execution of the asynchronous task launched by 392 * by either IPreparedModel::execute_1_2 or IPreparedModel::execute_1_3. If 393 * IPreparedModel::execute_1_2 or IPreparedModel::execute_1_3 has not 394 * finished asynchronously executing, this call will block until the 395 * asynchronous task notifies the object. 396 * 397 * If the asynchronous task was launched by IPreparedModel::execute, every 398 * time must be UINT64_MAX. 399 * 400 * @return timing Duration of the execution. Every time must be UINT64_MAX 401 * unless the status is NONE. 402 */ 403 hal::Timing getTiming() const; 404 405 /** 406 * ExecutionCallback::bindThread binds a thread to the ExecutionCallback 407 * object. The bound thread is later joined by ExecutionCallback::wait or 408 * ExecutionCallback::get*. 409 * 410 * Once a thread is bound with ExecutionCallback::bindThread, the client 411 * code must ensure that ExecutionCallback::wait or ExecutionCallback::get* 412 * has been called before the ExecutionCallback object is destroyed. 413 * 414 * The bound thread must not call any ExecutionCallback method with the 415 * exception of ExecutionCallback::notify*, which it must call when the 416 * thread has finished its computation. 417 * 418 * ExecutionCallback::bindThread can be called at most once on a given 419 * callback object. 420 * 421 * @param asyncThread Thread to be bound to the callback object. The thread 422 * object must represent a thread of execution -- i.e., 423 * std::thread::joinable() must be true. 424 * @return bool True if successful, false if thread was not properly bound. 425 */ 426 bool bindThread(std::thread asyncThread); 427 428 /** 429 * ExecutionCallback::setOnFinish binds a callback to the ExecutionCallback 430 * object that will be executed during one of the ExecutionCallback::notify* 431 * calls but before any calls to wait or get* return. This provided callback 432 * is provided with both the ErrorStatus and the output shapes from 433 * ExecutionCallback::notify*. 434 * 435 * The bound function must not synchronize with or otherwise access the 436 * callback object it is bound to, as this could cause a deadlock. 437 * 438 * This call will not bind the provided callback if any of the following 439 * occur: 440 * (1) the provided callback is invalid (i.e., "(bool) finish" is false) 441 * (2) ExecutionCallback already contains a bound callback 442 * (3) ExecutionCallback has already been notified with results 443 * 444 * @param finish Callback to be executed when ExecutionCallback is notified 445 * with results. 446 */ 447 void setOnFinish(const ExecutionFinish& finish); 448 449 /** 450 * Queries whether the object is dead. 451 * 452 * @return 'true' if dead, 'false' otherwise. 453 */ 454 bool isDeadObject() const; 455 456 private: 457 /* 458 * ExecutionCallback::notifyInternal stores the results of the execution 459 * (status, output shapes, and timing information) in the ExecutionCallback 460 * object and invokes the bound callback function "mOnFinish" (if present) 461 * before any call to wait or get* return. It then enables all prior and 462 * future wait calls on the ExecutionCallback object to proceed. 463 */ 464 hal::Return<void> notifyInternal(bool deadObject, hal::ErrorStatus errorStatus, 465 std::vector<hal::OutputShape> outputShapes, 466 hal::Timing timing); 467 468 // members 469 mutable std::mutex mMutex; 470 mutable std::condition_variable mCondition; 471 mutable std::thread mThread GUARDED_BY(mMutex); 472 ExecutionFinish mOnFinish GUARDED_BY(mMutex); 473 bool mNotified GUARDED_BY(mMutex) = false; 474 bool mDeadObject = false; 475 hal::ErrorStatus mErrorStatus = hal::ErrorStatus::GENERAL_FAILURE; 476 std::vector<hal::OutputShape> mOutputShapes; 477 hal::Timing mTiming = {}; 478 }; 479 480 } // namespace android::nn 481 482 #endif // ANDROID_FRAMEWORKS_ML_NN_RUNTIME_CALLBACKS_H 483