/* * Copyright (C) 2019 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "1.0/Utils.h" #include "1.3/Callbacks.h" #include "1.3/Utils.h" #include "GeneratedTestHarness.h" #include "Utils.h" namespace android::hardware::neuralnetworks::V1_3::vts::functional { using implementation::ExecutionCallback; using implementation::PreparedModelCallback; using test_helper::TestBuffer; using test_helper::TestModel; using V1_1::ExecutionPreference; using V1_2::MeasureTiming; using V1_2::OutputShape; using V1_2::Timing; using HidlToken = hidl_array(V1_2::Constant::BYTE_SIZE_OF_CACHE_TOKEN)>; enum class DeadlineBoundType { NOW, UNLIMITED, SHORT }; constexpr std::array deadlineBounds = { DeadlineBoundType::NOW, DeadlineBoundType::UNLIMITED, DeadlineBoundType::SHORT}; std::string toString(DeadlineBoundType type) { switch (type) { case DeadlineBoundType::NOW: return "NOW"; case DeadlineBoundType::UNLIMITED: return "UNLIMITED"; case DeadlineBoundType::SHORT: return "SHORT"; } LOG(FATAL) << "Unrecognized DeadlineBoundType: " << static_cast(type); return {}; } constexpr auto kShortDuration = std::chrono::milliseconds{5}; using Results = std::tuple, Timing>; using MaybeResults = std::optional; using ExecutionFunction = std::function& preparedModel, const Request& request, const OptionalTimePoint& deadline)>; static OptionalTimePoint makeDeadline(DeadlineBoundType deadlineBoundType) { const auto getNanosecondsSinceEpoch = [](const auto& time) -> uint64_t { const auto timeSinceEpoch = time.time_since_epoch(); return std::chrono::duration_cast(timeSinceEpoch).count(); }; std::chrono::steady_clock::time_point timePoint; switch (deadlineBoundType) { case DeadlineBoundType::NOW: timePoint = std::chrono::steady_clock::now(); break; case DeadlineBoundType::UNLIMITED: timePoint = std::chrono::steady_clock::time_point::max(); break; case DeadlineBoundType::SHORT: timePoint = std::chrono::steady_clock::now() + kShortDuration; break; } OptionalTimePoint deadline; deadline.nanosecondsSinceEpoch(getNanosecondsSinceEpoch(timePoint)); return deadline; } void runPrepareModelTest(const sp& device, const Model& model, Priority priority, std::optional deadlineBound) { OptionalTimePoint deadline; if (deadlineBound.has_value()) { deadline = makeDeadline(deadlineBound.value()); } // see if service can handle model bool fullySupportsModel = false; const Return supportedCall = device->getSupportedOperations_1_3( model, [&fullySupportsModel](ErrorStatus status, const hidl_vec& supported) { ASSERT_EQ(ErrorStatus::NONE, status); ASSERT_NE(0ul, supported.size()); fullySupportsModel = std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); }); ASSERT_TRUE(supportedCall.isOk()); // launch prepare model const sp preparedModelCallback = new PreparedModelCallback(); const Return prepareLaunchStatus = device->prepareModel_1_3( model, ExecutionPreference::FAST_SINGLE_ANSWER, priority, deadline, hidl_vec(), hidl_vec(), HidlToken(), preparedModelCallback); ASSERT_TRUE(prepareLaunchStatus.isOk()); ASSERT_EQ(ErrorStatus::NONE, static_cast(prepareLaunchStatus)); // retrieve prepared model preparedModelCallback->wait(); const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); const sp preparedModelV1_0 = preparedModelCallback->getPreparedModel(); const sp preparedModel = IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr); // The getSupportedOperations_1_3 call returns a list of operations that are // guaranteed not to fail if prepareModel_1_3 is called, and // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed. // If a driver has any doubt that it can prepare an operation, it must // return false. So here, if a driver isn't sure if it can support an // operation, but reports that it successfully prepared the model, the test // can continue. if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { ASSERT_EQ(nullptr, preparedModel.get()); return; } // verify return status if (!deadlineBound.has_value()) { EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); } else { switch (deadlineBound.value()) { case DeadlineBoundType::NOW: case DeadlineBoundType::SHORT: // Either the driver successfully completed the task or it // aborted and returned MISSED_DEADLINE_*. EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE || prepareReturnStatus == ErrorStatus::MISSED_DEADLINE_TRANSIENT || prepareReturnStatus == ErrorStatus::MISSED_DEADLINE_PERSISTENT); break; case DeadlineBoundType::UNLIMITED: // If an unlimited deadline is supplied, we expect the execution to // proceed normally. In this case, check it normally by breaking out // of the switch statement. EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); break; } } ASSERT_EQ(prepareReturnStatus == ErrorStatus::NONE, preparedModel.get() != nullptr); } void runPrepareModelTests(const sp& device, const Model& model) { // test priority for (auto priority : hidl_enum_range{}) { SCOPED_TRACE("priority: " + toString(priority)); if (priority == kDefaultPriority) continue; runPrepareModelTest(device, model, priority, {}); } // test deadline for (auto deadlineBound : deadlineBounds) { SCOPED_TRACE("deadlineBound: " + toString(deadlineBound)); runPrepareModelTest(device, model, kDefaultPriority, deadlineBound); } } static MaybeResults executeAsynchronously(const sp& preparedModel, const Request& request, const OptionalTimePoint& deadline) { SCOPED_TRACE("asynchronous"); const MeasureTiming measure = MeasureTiming::NO; // launch execution const sp callback = new ExecutionCallback(); Return ret = preparedModel->execute_1_3(request, measure, deadline, {}, callback); EXPECT_TRUE(ret.isOk()); EXPECT_EQ(ErrorStatus::NONE, ret.withDefault(ErrorStatus::GENERAL_FAILURE)); if (!ret.isOk() || ret != ErrorStatus::NONE) return std::nullopt; // retrieve execution results callback->wait(); const ErrorStatus status = callback->getStatus(); hidl_vec outputShapes = callback->getOutputShapes(); const Timing timing = callback->getTiming(); // return results return Results{status, std::move(outputShapes), timing}; } static MaybeResults executeSynchronously(const sp& preparedModel, const Request& request, const OptionalTimePoint& deadline) { SCOPED_TRACE("synchronous"); const MeasureTiming measure = MeasureTiming::NO; // configure results callback MaybeResults results; const auto cb = [&results](ErrorStatus status, const hidl_vec& outputShapes, const Timing& timing) { results.emplace(status, outputShapes, timing); }; // run execution const Return ret = preparedModel->executeSynchronously_1_3(request, measure, deadline, {}, cb); EXPECT_TRUE(ret.isOk()); if (!ret.isOk()) return std::nullopt; // return results return results; } void runExecutionTest(const sp& preparedModel, const TestModel& testModel, const Request& request, const ExecutionContext& context, bool synchronous, DeadlineBoundType deadlineBound) { const ExecutionFunction execute = synchronous ? executeSynchronously : executeAsynchronously; const auto deadline = makeDeadline(deadlineBound); // Perform execution and unpack results. const auto results = execute(preparedModel, request, deadline); if (!results.has_value()) return; const auto& [status, outputShapes, timing] = results.value(); // Verify no timing information was returned EXPECT_EQ(UINT64_MAX, timing.timeOnDevice); EXPECT_EQ(UINT64_MAX, timing.timeInDriver); // Validate deadline information if applicable. switch (deadlineBound) { case DeadlineBoundType::NOW: case DeadlineBoundType::SHORT: // Either the driver successfully completed the task or it // aborted and returned MISSED_DEADLINE_*. ASSERT_TRUE(status == ErrorStatus::NONE || status == ErrorStatus::MISSED_DEADLINE_TRANSIENT || status == ErrorStatus::MISSED_DEADLINE_PERSISTENT); break; case DeadlineBoundType::UNLIMITED: // If an unlimited deadline is supplied, we expect the execution to // proceed normally. In this case, check it normally by breaking out // of the switch statement. ASSERT_EQ(ErrorStatus::NONE, status); break; } // If the model output operands are fully specified, outputShapes must be either // either empty, or have the same number of elements as the number of outputs. ASSERT_TRUE(outputShapes.size() == 0 || outputShapes.size() == testModel.main.outputIndexes.size()); // Go through all outputs, check returned output shapes. for (uint32_t i = 0; i < outputShapes.size(); i++) { EXPECT_TRUE(outputShapes[i].isSufficient); const auto& expect = testModel.main.operands[testModel.main.outputIndexes[i]].dimensions; const std::vector actual = outputShapes[i].dimensions; EXPECT_EQ(expect, actual); } // Retrieve execution results. ASSERT_TRUE(nn::compliantWithV1_0(request)); const V1_0::Request request10 = nn::convertToV1_0(request); const std::vector outputs = context.getOutputBuffers(request10); // We want "close-enough" results. if (status == ErrorStatus::NONE) { checkResults(testModel, outputs); } } void runExecutionTests(const sp& preparedModel, const TestModel& testModel, const Request& request, const ExecutionContext& context) { for (bool synchronous : {false, true}) { for (auto deadlineBound : deadlineBounds) { runExecutionTest(preparedModel, testModel, request, context, synchronous, deadlineBound); } } } void runTests(const sp& device, const TestModel& testModel) { // setup const Model model = createModel(testModel); // run prepare model tests runPrepareModelTests(device, model); // prepare model sp preparedModel; createPreparedModel(device, model, &preparedModel); if (preparedModel == nullptr) return; // run execution tests ExecutionContext context; const Request request = nn::convertToV1_3(context.createRequest(testModel)); runExecutionTests(preparedModel, testModel, request, context); } class DeadlineTest : public GeneratedTestBase {}; TEST_P(DeadlineTest, Test) { runTests(kDevice, kTestModel); } INSTANTIATE_GENERATED_TEST(DeadlineTest, [](const TestModel& testModel) { return !testModel.expectFailure; }); } // namespace android::hardware::neuralnetworks::V1_3::vts::functional