1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_3_GENERATED_TEST_HARNESS_H 18 #define ANDROID_HARDWARE_NEURALNETWORKS_V1_3_GENERATED_TEST_HARNESS_H 19 20 #include <android/hardware/neuralnetworks/1.3/IDevice.h> 21 #include <android/hardware/neuralnetworks/1.3/IPreparedModel.h> 22 #include <android/hardware/neuralnetworks/1.3/types.h> 23 #include <functional> 24 #include <vector> 25 #include "1.0/Utils.h" 26 #include "TestHarness.h" 27 #include "VtsHalNeuralnetworks.h" 28 29 namespace android::hardware::neuralnetworks::V1_3::vts::functional { 30 31 using NamedModel = Named<const test_helper::TestModel*>; 32 using GeneratedTestParam = std::tuple<NamedDevice, NamedModel>; 33 34 class GeneratedTestBase : public testing::TestWithParam<GeneratedTestParam> { 35 protected: 36 void SetUp() override; 37 const sp<IDevice> kDevice = getData(std::get<NamedDevice>(GetParam())); 38 const test_helper::TestModel& kTestModel = *getData(std::get<NamedModel>(GetParam())); 39 }; 40 41 using FilterFn = std::function<bool(const test_helper::TestModel&)>; 42 std::vector<NamedModel> getNamedModels(const FilterFn& filter); 43 44 using FilterNameFn = std::function<bool(const std::string&)>; 45 std::vector<NamedModel> getNamedModels(const FilterNameFn& filter); 46 47 std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info); 48 49 #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ 50 GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TestSuite); \ 51 INSTANTIATE_TEST_SUITE_P(TestGenerated, TestSuite, \ 52 testing::Combine(testing::ValuesIn(getNamedDevices()), \ 53 testing::ValuesIn(getNamedModels(filter))), \ 54 printGeneratedTest) 55 56 // Tag for the validation tests, instantiated in VtsHalNeuralnetworks.cpp. 57 // TODO: Clean up the hierarchy for ValidationTest. 58 class ValidationTest : public GeneratedTestBase {}; 59 60 Model createModel(const test_helper::TestModel& testModel); 61 62 void PrepareModel(const sp<IDevice>& device, const Model& model, sp<IPreparedModel>* preparedModel); 63 64 enum class TestKind { 65 // Runs a test model and compares the results to a golden data 66 GENERAL, 67 // Same as GENERAL but sets dimensions for the output tensors to zeros 68 DYNAMIC_SHAPE, 69 // Same as GENERAL but use device memories for inputs and outputs 70 MEMORY_DOMAIN, 71 // Same as GENERAL but use executeFenced for exeuction 72 FENCED_COMPUTE, 73 // Tests if quantized model with TENSOR_QUANT8_ASYMM produces the same result 74 // (OK/SKIPPED/FAILED) as the model with all such tensors converted to 75 // TENSOR_QUANT8_ASYMM_SIGNED. 76 QUANTIZATION_COUPLING, 77 // Runs a test model and verifies that MISSED_DEADLINE_* is returned. 78 INTINITE_LOOP_TIMEOUT 79 }; 80 81 void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>& preparedModel, 82 const test_helper::TestModel& testModel, TestKind testKind); 83 84 void waitForSyncFence(int syncFd); 85 86 } // namespace android::hardware::neuralnetworks::V1_3::vts::functional 87 88 #endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_3_GENERATED_TEST_HARNESS_H 89