1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <android-base/properties.h>
18 #include <gtest/gtest.h>
19
20 #include <algorithm>
21 #include <map>
22 #include <memory>
23 #include <set>
24 #include <string>
25 #include <utility>
26
27 #include "GeneratedTestUtils.h"
28 #include "TestHarness.h"
29 #include "TestNeuralNetworksWrapper.h"
30 #include "fuzzing/OperationManager.h"
31 #include "fuzzing/RandomGraphGenerator.h"
32 #include "fuzzing/RandomGraphGeneratorUtils.h"
33
34 #ifndef NNTEST_CTS
35 #include <memunreachable/memunreachable.h>
36
37 #include <vector>
38
39 #include "HalInterfaces.h"
40 #include "Manager.h"
41 #include "SampleDriverFull.h"
42
43 using android::nn::sample_driver::SampleDriverFull;
44 using namespace android::nn::hal;
45
46 #endif
47
48 namespace android {
49 namespace nn {
50 namespace fuzzing_test {
51
52 using namespace test_helper;
53 using test_wrapper::Result;
54 constexpr char kRefDeviceName[] = "nnapi-reference";
55
56 #ifndef NNTEST_CTS
57 class TestDriverV1_2 : public SampleDriverFull {
58 public:
TestDriverV1_2()59 TestDriverV1_2() : SampleDriverFull(name, {.execTime = 0.9f, .powerUsage = 0.9f}) {}
60 static constexpr char name[] = "TestDriverV1_2";
61 };
62
63 // Like SampleDriverFull, but implementing 1.1
64 class TestDriverV1_1 : public V1_1::IDevice {
65 public:
TestDriverV1_1()66 TestDriverV1_1()
67 : mDriverV1_2(new SampleDriverFull(name, {.execTime = 0.8f, .powerUsage = 0.8f})) {}
68 static constexpr char name[] = "TestDriverV1_1";
getCapabilities_1_1(getCapabilities_1_1_cb _hidl_cb)69 Return<void> getCapabilities_1_1(getCapabilities_1_1_cb _hidl_cb) override {
70 return mDriverV1_2->getCapabilities_1_1(_hidl_cb);
71 }
getSupportedOperations_1_1(const V1_1::Model & model,getSupportedOperations_1_1_cb _hidl_cb)72 Return<void> getSupportedOperations_1_1(const V1_1::Model& model,
73 getSupportedOperations_1_1_cb _hidl_cb) override {
74 return mDriverV1_2->getSupportedOperations_1_1(model, _hidl_cb);
75 }
prepareModel_1_1(const V1_1::Model & model,ExecutionPreference preference,const sp<V1_0::IPreparedModelCallback> & actualCallback)76 Return<V1_0::ErrorStatus> prepareModel_1_1(
77 const V1_1::Model& model, ExecutionPreference preference,
78 const sp<V1_0::IPreparedModelCallback>& actualCallback) override {
79 return mDriverV1_2->prepareModel_1_1(model, preference, actualCallback);
80 }
getStatus()81 Return<DeviceStatus> getStatus() override { return mDriverV1_2->getStatus(); }
getCapabilities(getCapabilities_cb _hidl_cb)82 Return<void> getCapabilities(getCapabilities_cb _hidl_cb) override {
83 return mDriverV1_2->getCapabilities(_hidl_cb);
84 }
getSupportedOperations(const V1_0::Model & model,getSupportedOperations_cb _hidl_cb)85 Return<void> getSupportedOperations(const V1_0::Model& model,
86 getSupportedOperations_cb _hidl_cb) override {
87 return mDriverV1_2->getSupportedOperations(model, _hidl_cb);
88 }
prepareModel(const V1_0::Model & model,const sp<V1_0::IPreparedModelCallback> & actualCallback)89 Return<V1_0::ErrorStatus> prepareModel(
90 const V1_0::Model& model,
91 const sp<V1_0::IPreparedModelCallback>& actualCallback) override {
92 return mDriverV1_2->prepareModel(model, actualCallback);
93 }
94
95 private:
96 const sp<V1_2::IDevice> mDriverV1_2;
97 };
98
99 // Like SampleDriverFull, but implementing 1.0
100 class TestDriverV1_0 : public V1_0::IDevice {
101 public:
TestDriverV1_0()102 TestDriverV1_0()
103 : mDriverV1_2(new SampleDriverFull(name, {.execTime = 0.7f, .powerUsage = 0.7f})) {}
104 static constexpr char name[] = "TestDriverV1_0";
getCapabilities(getCapabilities_cb _hidl_cb)105 Return<void> getCapabilities(getCapabilities_cb _hidl_cb) override {
106 return mDriverV1_2->getCapabilities(_hidl_cb);
107 }
getSupportedOperations(const V1_0::Model & model,getSupportedOperations_cb _hidl_cb)108 Return<void> getSupportedOperations(const V1_0::Model& model,
109 getSupportedOperations_cb _hidl_cb) override {
110 return mDriverV1_2->getSupportedOperations(model, _hidl_cb);
111 }
prepareModel(const V1_0::Model & model,const sp<V1_0::IPreparedModelCallback> & actualCallback)112 Return<V1_0::ErrorStatus> prepareModel(
113 const V1_0::Model& model,
114 const sp<V1_0::IPreparedModelCallback>& actualCallback) override {
115 return mDriverV1_2->prepareModel(model, actualCallback);
116 }
getStatus()117 Return<DeviceStatus> getStatus() override { return mDriverV1_2->getStatus(); }
118
119 private:
120 const sp<V1_2::IDevice> mDriverV1_2;
121 };
122
123 template <class T_TestDriver>
makeTestDevice()124 std::shared_ptr<Device> makeTestDevice() {
125 return DeviceManager::forTest_makeDriverDevice(T_TestDriver::name, new T_TestDriver);
126 }
127
128 #endif
129
130 // NN API fuzzer logging setting comes from system property debug.nn.fuzzer.log and
131 // debug.nn.fuzzer.dumpspec.
132 // * setprop debug.nn.fuzzer.log 1 : enable logging.
133 // * setprop debug.nn.fuzzer.log 0 : silence logging.
134 // * setprop debug.nn.fuzzer.dumpspec 1 : dump the randomly generated graph to a spec file.
135 // * setprop debug.nn.fuzzer.dumpspec 0 : do not dump the graph.
136 //
137 // Logs and spec files are dumped to /data/local/tmp/${testname}.{log,mod.py},
138 // e.g. for test case TestRandomGraph/RandomGraphTest/Large/0,
139 // log : /data/local/tmp/TestRandomGraph_RandomGraphTest_Large_0.log
140 // spec: /data/local/tmp/TestRandomGraph_RandomGraphTest_Large_0.mod.py
141 //
142 class RandomGraphTest : public ::testing::TestWithParam<uint32_t> {
143 public:
SetUpTestCase()144 static void SetUpTestCase() {
145 #ifndef NNTEST_CTS
146 mEnableLog = ::android::base::GetProperty("debug.nn.fuzzer.log", "") == "1";
147 mDumpSpec = ::android::base::GetProperty("debug.nn.fuzzer.dumpspec", "") == "1";
148 mDetectMemoryLeak = ::android::base::GetProperty("debug.nn.fuzzer.detectleak", "") == "1";
149
150 mStandardDevices = DeviceManager::get()->forTest_getDevices();
151 mSyntheticDevices.push_back(makeTestDevice<TestDriverV1_2>());
152 mSyntheticDevices.push_back(makeTestDevice<TestDriverV1_1>());
153 mSyntheticDevices.push_back(makeTestDevice<TestDriverV1_0>());
154 #endif
155 mVndkVersion = ::android::base::GetIntProperty("ro.vndk.version", __ANDROID_API_FUTURE__);
156
157 // Get all the devices and device names.
158 mStandardDevicesFeatureLevel = __ANDROID_API_FUTURE__;
159 uint32_t numDevices = 0;
160 ASSERT_EQ(ANeuralNetworks_getDeviceCount(&numDevices), ANEURALNETWORKS_NO_ERROR);
161 for (uint32_t i = 0; i < numDevices; i++) {
162 ANeuralNetworksDevice* device = nullptr;
163 const char* name = nullptr;
164 int64_t featureLevel;
165 ASSERT_EQ(ANeuralNetworks_getDevice(i, &device), ANEURALNETWORKS_NO_ERROR);
166 ASSERT_EQ(ANeuralNetworksDevice_getName(device, &name), ANEURALNETWORKS_NO_ERROR);
167 ASSERT_EQ(ANeuralNetworksDevice_getFeatureLevel(device, &featureLevel),
168 ANEURALNETWORKS_NO_ERROR);
169 mDevices.emplace(name, device);
170 mStandardDevicesFeatureLevel = std::min(mStandardDevicesFeatureLevel, featureLevel);
171 }
172 }
173
174 protected:
SetUp()175 virtual void SetUp() override {
176 // Initialize logging.
177 const ::testing::TestInfo* const testInfo =
178 ::testing::UnitTest::GetInstance()->current_test_info();
179 mTestName = mTestName + testInfo->test_case_name() + "_" + testInfo->name();
180 std::replace(mTestName.begin(), mTestName.end(), '/', '_');
181 if (mEnableLog) NN_FUZZER_LOG_INIT("/data/local/tmp/" + mTestName + ".log");
182 }
183
TearDown()184 virtual void TearDown() override {
185 NN_FUZZER_LOG_CLOSE;
186 // Dump test results on failure for debugging.
187 if (::testing::Test::HasFailure() || mDumpSpec) {
188 dumpTestResults();
189 }
190 #ifndef NNTEST_CTS
191 if (mDetectMemoryLeak) {
192 ASSERT_TRUE(NoLeaks());
193 }
194 #endif
195 }
196
shouldSkipTest(int64_t featureLevel)197 bool shouldSkipTest(int64_t featureLevel) {
198 static const std::set<std::string> kDisabledTests = {
199 // In this test, the RGG produces a non-sensible graph with extreme large output
200 // gain and highly clamped output range.
201 // TODO: Currently quantized buffer values are uniformly distributed within
202 // [0, 255]. We should investigate on a better buffer value generation
203 // algorithm that represents the real-world cases.
204 "TestRandomGraph_SingleOperationTest_CONV_2D_V1_2_40",
205 "TestRandomGraph_SingleOperationTest_DEPTHWISE_CONV_2D_V1_0_32",
206 };
207 if (kDisabledTests.find(mTestName) != kDisabledTests.end()) return true;
208 for (const auto& op : mTestModel.main.operations) {
209 // Skip if testing BATCH_TO_SPACE_ND with batch dimension == 1.
210 if (op.type == TestOperationType::BATCH_TO_SPACE_ND &&
211 mTestModel.main.operands[op.inputs[0]].dimensions[0] == 1 &&
212 featureLevel <= __ANDROID_API_Q__) {
213 return true;
214 }
215 // L2_NORMALIZATION on axis of all zeros is undefined before R.
216 if (op.type == TestOperationType::L2_NORMALIZATION &&
217 featureLevel <= __ANDROID_API_Q__) {
218 return true;
219 }
220 // Skip the following operations for 1.2 and earlier devices.
221 if ((op.type == TestOperationType::ADD || op.type == TestOperationType::SUB ||
222 op.type == TestOperationType::MAXIMUM || op.type == TestOperationType::MINIMUM ||
223 op.type == TestOperationType::ROI_ALIGN) &&
224 mTestModel.main.operands[op.inputs[0]].type ==
225 TestOperandType::TENSOR_QUANT8_ASYMM &&
226 featureLevel <= __ANDROID_API_Q__) {
227 return true;
228 }
229 // Skip the following operations when the VNDK version is earlier than R.
230 if (mVndkVersion < __ANDROID_API_R__ &&
231 op.type == TestOperationType::HEATMAP_MAX_KEYPOINT) {
232 return true;
233 }
234 }
235 return false;
236 }
237
238 // Compute the golden output results of the test model on nnapi-reference. If possible, the
239 // golden results will be computed from an equivalent float32 model to avoid bias avoid bias
240 // from quantized CPU implementation.
computeGoldenResults()241 void computeGoldenResults() {
242 SCOPED_TRACE("computeGoldenResults");
243
244 // Convert the test model to an equivalent float32 model if possible.
245 auto fpModel = convertToFloat32Model(mTestModel);
246 const TestModel& goldenModel = fpModel.has_value() ? fpModel.value() : mTestModel;
247
248 // Create model.
249 generated_tests::GeneratedModel model;
250 generated_tests::createModel(goldenModel, &model);
251 ASSERT_TRUE(model.isValid());
252 ASSERT_EQ(model.finish(), Result::NO_ERROR);
253
254 // Create compilation for nnapi-reference.
255 ASSERT_TRUE(mDevices.find(kRefDeviceName) != mDevices.end());
256 const auto refDevice = mDevices[kRefDeviceName];
257 auto [result, compilation] = test_wrapper::Compilation::createForDevice(&model, refDevice);
258 ASSERT_EQ(result, Result::NO_ERROR);
259 ASSERT_EQ(compilation.finish(), Result::NO_ERROR);
260
261 // Create request.
262 test_wrapper::Execution execution(&compilation);
263 std::vector<TestBuffer> outputs;
264 generated_tests::createRequest(goldenModel, &execution, &outputs);
265
266 // Compute result.
267 ASSERT_EQ(execution.compute(), Result::NO_ERROR);
268
269 if (fpModel.has_value()) {
270 // Quantize the execution results as golden values.
271 setExpectedOutputsFromFloat32Results(outputs, &mTestModel);
272 } else {
273 for (uint32_t i = 0; i < outputs.size(); i++) {
274 auto outputIndex = mTestModel.main.outputIndexes[i];
275 mTestModel.main.operands[outputIndex].data = outputs[i];
276 }
277 }
278 }
279
280 // Compile and execute the generated graph on a device selected by name.
computeAndVerifyResultsForDevice(const test_wrapper::Model * model,uint32_t numOps,const std::string & name)281 void computeAndVerifyResultsForDevice(const test_wrapper::Model* model, uint32_t numOps,
282 const std::string& name) {
283 SCOPED_TRACE("Device: " + name);
284 std::cout << "[ ] - RUN: " << name << "\n";
285 ASSERT_TRUE(mDevices.find(name) != mDevices.end());
286 const auto device = mDevices[name];
287
288 // Check if the device fully supports the graph.
289 constexpr int kMaxNumberOperations = 1000;
290 ASSERT_TRUE(numOps <= kMaxNumberOperations);
291 bool supported[kMaxNumberOperations] = {false};
292 ASSERT_EQ(ANeuralNetworksModel_getSupportedOperationsForDevices(model->getHandle(), &device,
293 1, supported),
294 ANEURALNETWORKS_NO_ERROR);
295 if (!std::all_of(supported, supported + numOps, [](bool v) { return v; })) {
296 std::cout << "[ ] SKIP: " << name << " does not support the graph.\n";
297 return;
298 }
299
300 // Since this test is introduced in Android Q, we only check the accuracy of output results
301 // if the device has feature level >= Q (API level 29). For pre-Q devices, we allow
302 // them to produce less accurate results, but must not hang or crash.
303 int64_t featureLevel;
304 ASSERT_EQ(ANeuralNetworksDevice_getFeatureLevel(device, &featureLevel),
305 ANEURALNETWORKS_NO_ERROR);
306 if (shouldSkipTest(featureLevel)) return;
307
308 // Create compilation for device.
309 auto [result, compilation] = test_wrapper::Compilation::createForDevice(model, device);
310 ASSERT_EQ(result, Result::NO_ERROR);
311 Result compileReturn = compilation.finish();
312 // Even if the model is fully supported, the compilation may still fail, e.g. each operation
313 // is supported, but model is too big (too many operations and/or too-large constants) for
314 // device.
315 if (compileReturn == Result::OP_FAILED) {
316 std::cout << "[ ] SKIP: " << name << " failed at compilation step.\n";
317 return;
318 }
319 ASSERT_EQ(compileReturn, Result::NO_ERROR);
320
321 // Create request.
322 test_wrapper::Execution execution(&compilation);
323 std::vector<TestBuffer> outputs;
324 generated_tests::createRequest(mTestModel, &execution, &outputs);
325
326 // Compute result.
327 Result executeReturn = execution.compute();
328 // Even if the model is fully supported and the compilation succeeds, the execution may
329 // still fail, e.g. there may be operand shapes that are unknown until execution time, and
330 // at execution time turn out to be too big.
331 if (executeReturn == Result::OP_FAILED) {
332 std::cout << "[ ] SKIP: " << name << " failed at execution step.\n";
333 return;
334 }
335 ASSERT_EQ(executeReturn, Result::NO_ERROR);
336
337 if (featureLevel >= __ANDROID_API_Q__) {
338 checkResults(mTestModel, outputs, mCriteria);
339 mResults.emplace_back(name, std::move(outputs));
340 }
341 }
342
343 // Compile and execute the generated graph normally (i.e., allow runtime to
344 // distribute across devices).
computeAndVerifyResults(const std::string & name,const test_wrapper::Model * model,bool shouldCheckResults)345 void computeAndVerifyResults(const std::string& name, const test_wrapper::Model* model,
346 bool shouldCheckResults) {
347 // Because we're not using the introspection/control API, the CpuDevice
348 // is available as a fallback, and hence we assume that compilation and
349 // execution will succeed.
350 SCOPED_TRACE(name);
351 std::cout << "[ ] - RUN: " << name << "\n";
352
353 // Create compilation.
354 test_wrapper::Compilation compilation(model);
355 ASSERT_EQ(compilation.finish(), Result::NO_ERROR);
356
357 // Create request.
358 test_wrapper::Execution execution(&compilation);
359 std::vector<TestBuffer> outputs;
360 generated_tests::createRequest(mTestModel, &execution, &outputs);
361
362 // Compute and verify result.
363 ASSERT_EQ(execution.compute(), Result::NO_ERROR);
364 if (shouldCheckResults) {
365 checkResults(mTestModel, outputs, mCriteria);
366 mResults.emplace_back(name, std::move(outputs));
367 }
368 }
369
370 // Main test entrance.
testRandomGraph(uint32_t numOperations,uint32_t dimensionRange)371 void testRandomGraph(uint32_t numOperations, uint32_t dimensionRange) {
372 // Generate a random graph.
373 RandomGraph graph;
374 ASSERT_TRUE(graph.generate(kSeed, numOperations, dimensionRange));
375
376 // Create a model from the random graph.
377 mTestModel = graph.createTestModel();
378
379 generated_tests::GeneratedModel model;
380 generated_tests::createModel(mTestModel, &model);
381 ASSERT_TRUE(model.isValid());
382 ASSERT_EQ(model.finish(), Result::NO_ERROR);
383
384 // Compute reference results.
385 computeGoldenResults();
386
387 // Compute on each available device.
388 for (auto& pair : mDevices) {
389 computeAndVerifyResultsForDevice(&model, numOperations, pair.first);
390 }
391
392 if (numOperations > 1) {
393 if (!shouldSkipTest(mStandardDevicesFeatureLevel)) {
394 // Compute normally (i.e., allow runtime to distribute across devices).
395 computeAndVerifyResults("Compute normally", &model,
396 mStandardDevicesFeatureLevel >= __ANDROID_API_Q__);
397 }
398
399 #ifndef NNTEST_CTS
400 {
401 // Stress partitioner by allowing runtime to distribute across
402 // three synthetic devices. The synthetic devices use the
403 // CpuExecutor for execution, so we always check results, even
404 // though some are of feature level < __ANDROID_API_Q__: In this
405 // case, we don't take feature level as an indication of
406 // reliability, as we do with real devices.
407 DeviceManager::get()->forTest_setDevices(mSyntheticDevices);
408 computeAndVerifyResults("Compute across synthetic devices", &model, true);
409 DeviceManager::get()->forTest_setDevices(mStandardDevices);
410 }
411 #endif
412 }
413 }
414
dumpTestResults()415 void dumpTestResults() {
416 std::ofstream os("/data/local/tmp/" + mTestName + ".mod.py");
417 ASSERT_TRUE(os.is_open());
418 os << "# Generated from " << mTestName << ". Do not edit.\n\n";
419 SpecDumper dumper(mTestModel, os);
420 dumper.dumpTestModel();
421 for (const auto& [name, results] : mResults) {
422 dumper.dumpResults(name, results);
423 }
424 }
425
426 enum GraphSize : uint32_t { SINGLE = 1, SMALL = 5, LARGE = 40 };
427 enum DimensionRange : uint32_t { NARROW = 10, WIDE = 1000 };
428
429 static bool mEnableLog;
430 static bool mDumpSpec;
431 static bool mDetectMemoryLeak;
432 static std::map<std::string, ANeuralNetworksDevice*> mDevices;
433
434 const uint32_t kSeed = GetParam();
435 std::string mTestName;
436 TestModel mTestModel;
437 AccuracyCriteria mCriteria;
438
439 // A vector of {name, output_results}.
440 std::vector<std::pair<std::string, std::vector<TestBuffer>>> mResults;
441
442 static int mVndkVersion;
443 static int64_t mStandardDevicesFeatureLevel; // minimum across all devices
444 #ifndef NNTEST_CTS
445 static std::vector<std::shared_ptr<Device>> mStandardDevices;
446 static std::vector<std::shared_ptr<Device>> mSyntheticDevices;
447 #endif
448 };
449
450 bool RandomGraphTest::mEnableLog = false;
451 bool RandomGraphTest::mDumpSpec = false;
452 bool RandomGraphTest::mDetectMemoryLeak = false;
453 std::map<std::string, ANeuralNetworksDevice*> RandomGraphTest::mDevices;
454
455 int RandomGraphTest::mVndkVersion = __ANDROID_API_FUTURE__;
456 int64_t RandomGraphTest::mStandardDevicesFeatureLevel;
457 #ifndef NNTEST_CTS
458 std::vector<std::shared_ptr<Device>> RandomGraphTest::mStandardDevices;
459 std::vector<std::shared_ptr<Device>> RandomGraphTest::mSyntheticDevices;
460 #endif
461
462 // Single-op graph with dimensions in range [1, 1000].
463 class SingleOperationTest : public RandomGraphTest {};
464 #define TEST_SINGLE_OPERATION(operation, halVersion, criteria) \
465 TEST_P(SingleOperationTest, operation##_##halVersion) { \
466 OperationFilter filter = {.opcodes = {TestOperationType::operation}, \
467 .versions = {TestHalVersion::halVersion}}; \
468 OperationManager::get()->applyFilter(filter); \
469 mCriteria = (criteria); \
470 testRandomGraph(GraphSize::SINGLE, DimensionRange::WIDE); \
471 }
472
473 // TODO: Adjust the accuracy criteria based on testing.
474 // We define three sets of accuracy criteria for single-operation tests.
475
476 // This is for operations that only copy buffers around without any computation on buffer values.
477 // Most of these operations fall into categories of reshape or selection, e.g. RESHAPE, GATHER.
478 // Additionally, operations with only logical or comparison arithmetic also use this criteria, e.g.
479 // EQUAL, ARGMAX, TOPK_V2.
480 const AccuracyCriteria kStrictCriteria = {
481 .float32 = {.bias = 1e-7f, .mse = 1e-10f, .atol = 1e-6f, .rtol = 1e-6f},
482 .float16 = {.bias = 1e-4f, .mse = 1e-8f, .atol = 1e-3f, .rtol = 1e-3f},
483 .int32 = {.atol = 1},
484 .quant8Asymm = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
485 .quant8AsymmSigned = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
486 .quant8Symm = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
487 .quant16Asymm = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
488 .quant16Symm = {.bias = 0.1f, .mse = 0.1f, .atol = 1},
489 };
490
491 // This is for operations that only do simple and single computation on buffer values, such as
492 // addition, multiplication, or requantization. Most of these operations fall into categories of
493 // broadcast or elementwise, e.g ADD, FLOOR.
494 const AccuracyCriteria kMediumCriteria = {
495 .float32 = {.bias = 1e-6f, .mse = 1e-8f, .atol = 1e-5f, .rtol = 1e-5f},
496 .float16 = {.bias = 1e-3f, .mse = 1e-5f, .atol = 1e-2f, .rtol = 1e-2f},
497 .int32 = {.atol = 1},
498 .quant8Asymm = {.bias = 1.2, .mse = 1.2, .atol = 2},
499 .quant8AsymmSigned = {.bias = 1.2, .mse = 1.2, .atol = 2},
500 .quant8Symm = {.bias = 1.2, .mse = 1.2, .atol = 2},
501 .quant16Asymm = {.bias = 1.2, .mse = 1.2, .atol = 2},
502 .quant16Symm = {.bias = 1.2, .mse = 1.2, .atol = 2},
503 };
504
505 // This is for operations that involve sophisticated computations on buffer values, either a single
506 // but complex transformation, e.g. LOGISTIC, or multiple transformations with accumulated errors,
507 // e.g. L2_NORMALIZATION, REDUCE_*.
508 const AccuracyCriteria kRelaxedCriteria = {
509 .float32 = {.bias = 3e-5f, .mse = 1e-6f, .atol = 1e-3f, .rtol = 1e-3f},
510 .float16 = {.bias = 5e-3f, .mse = 1e-3f, .atol = 1.0f, .rtol = 1.0f},
511 .int32 = {.atol = 1},
512 .quant8Asymm = {.bias = 1.5, .mse = 1.5, .atol = 10},
513 .quant8AsymmSigned = {.bias = 1.5, .mse = 1.5, .atol = 10},
514 .quant8Symm = {.bias = 1.5, .mse = 1.5, .atol = 10},
515 .quant16Asymm = {.bias = 1.5, .mse = 1.5, .atol = 10},
516 .quant16Symm = {.bias = 1.5, .mse = 1.5, .atol = 10},
517 };
518
519 // This is for convolution operations with potentially large kernel size.
520 const AccuracyCriteria kConvCriteria = {
521 .float32 = {.bias = 4e-4f, .mse = 1e-5f, .atol = 2e-2f, .rtol = 2e-2f},
522 .float16 = {.bias = 5e-2f, .mse = 1e-2f, .atol = 1.0f, .rtol = 1.0f},
523 .int32 = {.atol = 1},
524 .quant8Asymm = {.bias = 1.5, .mse = 1.5, .atol = 10},
525 .quant8AsymmSigned = {.bias = 1.5, .mse = 1.5, .atol = 10},
526 .quant8Symm = {.bias = 1.5, .mse = 1.5, .atol = 10},
527 .quant16Asymm = {.bias = 1.5, .mse = 1.5, .atol = 10},
528 .quant16Symm = {.bias = 1.5, .mse = 1.5, .atol = 10},
529 };
530
531 /*-- NNAPI 1.0 Operations ---------------------------------------------------*/
532
533 // TODO: The following 1.0 operation signatures are currently not defined:
534 // - ANEURALNETWORKS_LSH_PROJECTION
535 // - ANEURALNETWORKS_LSTM
536 // - ANEURALNETWORKS_RNN
537 // - ANEURALNETWORKS_SVDF
538
539 TEST_SINGLE_OPERATION(ADD, V1_0, kMediumCriteria);
540 TEST_SINGLE_OPERATION(MUL, V1_0, kMediumCriteria);
541 TEST_SINGLE_OPERATION(FLOOR, V1_0, kMediumCriteria);
542 TEST_SINGLE_OPERATION(LOGISTIC, V1_0, kRelaxedCriteria);
543 TEST_SINGLE_OPERATION(RELU, V1_0, kMediumCriteria);
544 TEST_SINGLE_OPERATION(RELU1, V1_0, kMediumCriteria);
545 TEST_SINGLE_OPERATION(RELU6, V1_0, kMediumCriteria);
546 TEST_SINGLE_OPERATION(TANH, V1_0, kRelaxedCriteria);
547 TEST_SINGLE_OPERATION(SOFTMAX, V1_0, kRelaxedCriteria);
548 TEST_SINGLE_OPERATION(L2_NORMALIZATION, V1_0, kRelaxedCriteria);
549 TEST_SINGLE_OPERATION(LOCAL_RESPONSE_NORMALIZATION, V1_0, kRelaxedCriteria);
550 TEST_SINGLE_OPERATION(AVERAGE_POOL_2D, V1_0, kRelaxedCriteria);
551 TEST_SINGLE_OPERATION(L2_POOL_2D, V1_0, kRelaxedCriteria);
552 TEST_SINGLE_OPERATION(MAX_POOL_2D, V1_0, kRelaxedCriteria);
553 TEST_SINGLE_OPERATION(CONV_2D, V1_0, kConvCriteria);
554 TEST_SINGLE_OPERATION(DEPTHWISE_CONV_2D, V1_0, kConvCriteria);
555 TEST_SINGLE_OPERATION(CONCATENATION, V1_0, kMediumCriteria);
556 TEST_SINGLE_OPERATION(RESIZE_BILINEAR, V1_0, kRelaxedCriteria);
557 TEST_SINGLE_OPERATION(DEPTH_TO_SPACE, V1_0, kStrictCriteria);
558 TEST_SINGLE_OPERATION(SPACE_TO_DEPTH, V1_0, kStrictCriteria);
559 TEST_SINGLE_OPERATION(EMBEDDING_LOOKUP, V1_0, kStrictCriteria);
560 TEST_SINGLE_OPERATION(HASHTABLE_LOOKUP, V1_0, kStrictCriteria);
561 TEST_SINGLE_OPERATION(FULLY_CONNECTED, V1_0, kRelaxedCriteria);
562 TEST_SINGLE_OPERATION(RESHAPE, V1_0, kStrictCriteria);
563 TEST_SINGLE_OPERATION(DEQUANTIZE, V1_0, kMediumCriteria);
564
565 /*-- NNAPI 1.1 Operations ---------------------------------------------------*/
566
567 TEST_SINGLE_OPERATION(SUB, V1_1, kMediumCriteria);
568 TEST_SINGLE_OPERATION(DIV, V1_1, kRelaxedCriteria);
569 TEST_SINGLE_OPERATION(BATCH_TO_SPACE_ND, V1_1, kStrictCriteria);
570 TEST_SINGLE_OPERATION(SPACE_TO_BATCH_ND, V1_1, kStrictCriteria);
571 TEST_SINGLE_OPERATION(MEAN, V1_1, kRelaxedCriteria);
572 TEST_SINGLE_OPERATION(PAD, V1_1, kStrictCriteria);
573 TEST_SINGLE_OPERATION(TRANSPOSE, V1_1, kStrictCriteria);
574 TEST_SINGLE_OPERATION(SQUEEZE, V1_1, kStrictCriteria);
575 TEST_SINGLE_OPERATION(STRIDED_SLICE, V1_1, kStrictCriteria);
576
577 /*-- NNAPI 1.0 and 1.1 Operations with Extended Behavior in 1.2 -------------*/
578
579 TEST_SINGLE_OPERATION(ADD, V1_2, kMediumCriteria);
580 TEST_SINGLE_OPERATION(MUL, V1_2, kMediumCriteria);
581 TEST_SINGLE_OPERATION(SUB, V1_2, kMediumCriteria);
582 TEST_SINGLE_OPERATION(DIV, V1_2, kRelaxedCriteria);
583 TEST_SINGLE_OPERATION(FLOOR, V1_2, kMediumCriteria);
584 TEST_SINGLE_OPERATION(LOGISTIC, V1_2, kRelaxedCriteria);
585 TEST_SINGLE_OPERATION(RELU, V1_2, kMediumCriteria);
586 TEST_SINGLE_OPERATION(RELU1, V1_2, kMediumCriteria);
587 TEST_SINGLE_OPERATION(RELU6, V1_2, kMediumCriteria);
588 TEST_SINGLE_OPERATION(TANH, V1_2, kRelaxedCriteria);
589 TEST_SINGLE_OPERATION(CONCATENATION, V1_2, kMediumCriteria);
590 TEST_SINGLE_OPERATION(DEPTH_TO_SPACE, V1_2, kStrictCriteria);
591 TEST_SINGLE_OPERATION(SPACE_TO_DEPTH, V1_2, kStrictCriteria);
592 TEST_SINGLE_OPERATION(BATCH_TO_SPACE_ND, V1_2, kStrictCriteria);
593 TEST_SINGLE_OPERATION(SPACE_TO_BATCH_ND, V1_2, kStrictCriteria);
594 TEST_SINGLE_OPERATION(FULLY_CONNECTED, V1_2, kRelaxedCriteria);
595 TEST_SINGLE_OPERATION(RESHAPE, V1_2, kStrictCriteria);
596 TEST_SINGLE_OPERATION(MEAN, V1_2, kRelaxedCriteria);
597 TEST_SINGLE_OPERATION(PAD, V1_2, kStrictCriteria);
598 TEST_SINGLE_OPERATION(TRANSPOSE, V1_2, kStrictCriteria);
599 TEST_SINGLE_OPERATION(CONV_2D, V1_2, kConvCriteria);
600 TEST_SINGLE_OPERATION(DEPTHWISE_CONV_2D, V1_2, kConvCriteria);
601 TEST_SINGLE_OPERATION(AVERAGE_POOL_2D, V1_2, kRelaxedCriteria);
602 TEST_SINGLE_OPERATION(L2_POOL_2D, V1_2, kRelaxedCriteria);
603 TEST_SINGLE_OPERATION(MAX_POOL_2D, V1_2, kRelaxedCriteria);
604 TEST_SINGLE_OPERATION(RESIZE_BILINEAR, V1_2, kRelaxedCriteria);
605 TEST_SINGLE_OPERATION(SOFTMAX, V1_2, kRelaxedCriteria);
606 TEST_SINGLE_OPERATION(L2_NORMALIZATION, V1_2, kRelaxedCriteria);
607 TEST_SINGLE_OPERATION(LOCAL_RESPONSE_NORMALIZATION, V1_2, kRelaxedCriteria);
608 TEST_SINGLE_OPERATION(DEQUANTIZE, V1_2, kMediumCriteria);
609 TEST_SINGLE_OPERATION(SQUEEZE, V1_2, kStrictCriteria);
610 TEST_SINGLE_OPERATION(STRIDED_SLICE, V1_2, kStrictCriteria);
611 TEST_SINGLE_OPERATION(EMBEDDING_LOOKUP, V1_2, kStrictCriteria);
612
613 /*-- NNAPI 1.2 Operations ---------------------------------------------------*/
614
615 // TODO: The following 1.2 operation signatures are currently not defined:
616 // - ANEURALNETWORKS_AXIS_ALIGNED_BBOX_TRANSFORM
617 // - ANEURALNETWORKS_BIDIRECTIONAL_SEQUENCE_LSTM
618 // - ANEURALNETWORKS_BIDIRECTIONAL_SEQUENCE_RNN
619 // - ANEURALNETWORKS_BOX_WITH_NMS_LIMIT
620 // - ANEURALNETWORKS_DETECTION_POSTPROCESSING
621 // - ANEURALNETWORKS_GENERATE_PROPOSALS
622 // - ANEURALNETWORKS_QUANTIZED_16BIT_LSTM
623 // - ANEURALNETWORKS_RANDOM_MULTINOMIAL
624 // - ANEURALNETWORKS_UNIDIRECTIONAL_SEQUENCE_LSTM
625 // - ANEURALNETWORKS_UNIDIRECTIONAL_SEQUENCE_RNN
626
627 TEST_SINGLE_OPERATION(ABS, V1_2, kMediumCriteria);
628 TEST_SINGLE_OPERATION(EXP, V1_2, kRelaxedCriteria);
629 TEST_SINGLE_OPERATION(LOG, V1_2, kRelaxedCriteria);
630 TEST_SINGLE_OPERATION(NEG, V1_2, kMediumCriteria);
631 TEST_SINGLE_OPERATION(RSQRT, V1_2, kRelaxedCriteria);
632 TEST_SINGLE_OPERATION(SIN, V1_2, kRelaxedCriteria);
633 TEST_SINGLE_OPERATION(SQRT, V1_2, kRelaxedCriteria);
634 TEST_SINGLE_OPERATION(ARGMAX, V1_2, kStrictCriteria);
635 TEST_SINGLE_OPERATION(ARGMIN, V1_2, kStrictCriteria);
636 TEST_SINGLE_OPERATION(EQUAL, V1_2, kStrictCriteria);
637 TEST_SINGLE_OPERATION(GREATER, V1_2, kStrictCriteria);
638 TEST_SINGLE_OPERATION(GREATER_EQUAL, V1_2, kStrictCriteria);
639 TEST_SINGLE_OPERATION(LESS, V1_2, kStrictCriteria);
640 TEST_SINGLE_OPERATION(LESS_EQUAL, V1_2, kStrictCriteria);
641 TEST_SINGLE_OPERATION(LOGICAL_AND, V1_2, kStrictCriteria);
642 TEST_SINGLE_OPERATION(LOGICAL_NOT, V1_2, kStrictCriteria);
643 TEST_SINGLE_OPERATION(LOGICAL_OR, V1_2, kStrictCriteria);
644 TEST_SINGLE_OPERATION(NOT_EQUAL, V1_2, kStrictCriteria);
645 TEST_SINGLE_OPERATION(MAXIMUM, V1_2, kMediumCriteria);
646 TEST_SINGLE_OPERATION(MINIMUM, V1_2, kMediumCriteria);
647 TEST_SINGLE_OPERATION(POW, V1_2, kRelaxedCriteria);
648 TEST_SINGLE_OPERATION(PRELU, V1_2, kMediumCriteria);
649 TEST_SINGLE_OPERATION(REDUCE_ALL, V1_2, kRelaxedCriteria);
650 TEST_SINGLE_OPERATION(REDUCE_ANY, V1_2, kRelaxedCriteria);
651 TEST_SINGLE_OPERATION(REDUCE_MAX, V1_2, kRelaxedCriteria);
652 TEST_SINGLE_OPERATION(REDUCE_MIN, V1_2, kRelaxedCriteria);
653 TEST_SINGLE_OPERATION(REDUCE_PROD, V1_2, kRelaxedCriteria);
654 TEST_SINGLE_OPERATION(REDUCE_SUM, V1_2, kRelaxedCriteria);
655 TEST_SINGLE_OPERATION(CHANNEL_SHUFFLE, V1_2, kStrictCriteria);
656 TEST_SINGLE_OPERATION(INSTANCE_NORMALIZATION, V1_2, kRelaxedCriteria);
657 TEST_SINGLE_OPERATION(LOG_SOFTMAX, V1_2, kRelaxedCriteria);
658 TEST_SINGLE_OPERATION(GROUPED_CONV_2D, V1_2, kConvCriteria);
659 TEST_SINGLE_OPERATION(TRANSPOSE_CONV_2D, V1_2, kConvCriteria);
660 TEST_SINGLE_OPERATION(RESIZE_NEAREST_NEIGHBOR, V1_2, kRelaxedCriteria);
661 TEST_SINGLE_OPERATION(PAD_V2, V1_2, kStrictCriteria);
662 TEST_SINGLE_OPERATION(QUANTIZE, V1_2, kMediumCriteria);
663 TEST_SINGLE_OPERATION(CAST, V1_2, kMediumCriteria);
664 TEST_SINGLE_OPERATION(EXPAND_DIMS, V1_2, kStrictCriteria);
665 TEST_SINGLE_OPERATION(TILE, V1_2, kStrictCriteria);
666 TEST_SINGLE_OPERATION(GATHER, V1_2, kStrictCriteria);
667 TEST_SINGLE_OPERATION(SELECT, V1_2, kStrictCriteria);
668 TEST_SINGLE_OPERATION(TOPK_V2, V1_2, kStrictCriteria);
669 TEST_SINGLE_OPERATION(SLICE, V1_2, kStrictCriteria);
670 TEST_SINGLE_OPERATION(SPLIT, V1_2, kMediumCriteria);
671 TEST_SINGLE_OPERATION(ROI_ALIGN, V1_2, kRelaxedCriteria);
672 TEST_SINGLE_OPERATION(ROI_POOLING, V1_2, kRelaxedCriteria);
673 TEST_SINGLE_OPERATION(HEATMAP_MAX_KEYPOINT, V1_2, kRelaxedCriteria);
674
675 /*-- NNAPI 1.0, 1.1, and 1.2 Operations with Extended Behavior in 1.3 -------------*/
676
677 TEST_SINGLE_OPERATION(ADD, V1_3, kMediumCriteria);
678 TEST_SINGLE_OPERATION(AVERAGE_POOL_2D, V1_3, kRelaxedCriteria);
679 TEST_SINGLE_OPERATION(CONCATENATION, V1_3, kMediumCriteria);
680 TEST_SINGLE_OPERATION(CONV_2D, V1_3, kConvCriteria);
681 TEST_SINGLE_OPERATION(DEPTHWISE_CONV_2D, V1_3, kConvCriteria);
682 TEST_SINGLE_OPERATION(DEPTH_TO_SPACE, V1_3, kStrictCriteria);
683 TEST_SINGLE_OPERATION(DEQUANTIZE, V1_3, kMediumCriteria);
684 TEST_SINGLE_OPERATION(EMBEDDING_LOOKUP, V1_3, kStrictCriteria);
685 TEST_SINGLE_OPERATION(FULLY_CONNECTED, V1_3, kRelaxedCriteria);
686 TEST_SINGLE_OPERATION(L2_NORMALIZATION, V1_3, kRelaxedCriteria);
687 TEST_SINGLE_OPERATION(LOGISTIC, V1_3, kRelaxedCriteria);
688 TEST_SINGLE_OPERATION(MAX_POOL_2D, V1_3, kRelaxedCriteria);
689 TEST_SINGLE_OPERATION(MUL, V1_3, kMediumCriteria);
690 TEST_SINGLE_OPERATION(RELU, V1_3, kMediumCriteria);
691 TEST_SINGLE_OPERATION(RELU1, V1_3, kMediumCriteria);
692 TEST_SINGLE_OPERATION(RELU6, V1_3, kMediumCriteria);
693 TEST_SINGLE_OPERATION(RESHAPE, V1_3, kStrictCriteria);
694 TEST_SINGLE_OPERATION(RESIZE_BILINEAR, V1_3, kRelaxedCriteria);
695 TEST_SINGLE_OPERATION(SOFTMAX, V1_3, kRelaxedCriteria);
696 TEST_SINGLE_OPERATION(SPACE_TO_DEPTH, V1_3, kStrictCriteria);
697 TEST_SINGLE_OPERATION(TANH, V1_3, kRelaxedCriteria);
698 TEST_SINGLE_OPERATION(BATCH_TO_SPACE_ND, V1_3, kStrictCriteria);
699 TEST_SINGLE_OPERATION(DIV, V1_3, kMediumCriteria);
700 TEST_SINGLE_OPERATION(MEAN, V1_3, kRelaxedCriteria);
701 TEST_SINGLE_OPERATION(PAD, V1_3, kStrictCriteria);
702 TEST_SINGLE_OPERATION(SPACE_TO_BATCH_ND, V1_3, kStrictCriteria);
703 TEST_SINGLE_OPERATION(SQUEEZE, V1_3, kStrictCriteria);
704 TEST_SINGLE_OPERATION(STRIDED_SLICE, V1_3, kStrictCriteria);
705 TEST_SINGLE_OPERATION(SUB, V1_3, kMediumCriteria);
706 TEST_SINGLE_OPERATION(TRANSPOSE, V1_3, kStrictCriteria);
707 TEST_SINGLE_OPERATION(ABS, V1_3, kMediumCriteria);
708 TEST_SINGLE_OPERATION(ARGMAX, V1_3, kStrictCriteria);
709 TEST_SINGLE_OPERATION(ARGMIN, V1_3, kStrictCriteria);
710 TEST_SINGLE_OPERATION(CAST, V1_3, kMediumCriteria);
711 TEST_SINGLE_OPERATION(CHANNEL_SHUFFLE, V1_3, kStrictCriteria);
712 TEST_SINGLE_OPERATION(EQUAL, V1_3, kStrictCriteria);
713 TEST_SINGLE_OPERATION(EXPAND_DIMS, V1_3, kStrictCriteria);
714 TEST_SINGLE_OPERATION(GATHER, V1_3, kStrictCriteria);
715 TEST_SINGLE_OPERATION(GREATER, V1_3, kStrictCriteria);
716 TEST_SINGLE_OPERATION(GREATER_EQUAL, V1_3, kStrictCriteria);
717 TEST_SINGLE_OPERATION(GROUPED_CONV_2D, V1_3, kConvCriteria);
718 TEST_SINGLE_OPERATION(HEATMAP_MAX_KEYPOINT, V1_3, kRelaxedCriteria);
719 TEST_SINGLE_OPERATION(LESS, V1_3, kStrictCriteria);
720 TEST_SINGLE_OPERATION(LESS_EQUAL, V1_3, kStrictCriteria);
721 TEST_SINGLE_OPERATION(MAXIMUM, V1_3, kMediumCriteria);
722 TEST_SINGLE_OPERATION(MINIMUM, V1_3, kMediumCriteria);
723 TEST_SINGLE_OPERATION(NOT_EQUAL, V1_3, kStrictCriteria);
724 TEST_SINGLE_OPERATION(PAD_V2, V1_3, kStrictCriteria);
725 TEST_SINGLE_OPERATION(PRELU, V1_3, kMediumCriteria);
726 TEST_SINGLE_OPERATION(QUANTIZE, V1_3, kMediumCriteria);
727 TEST_SINGLE_OPERATION(REDUCE_MAX, V1_3, kRelaxedCriteria);
728 TEST_SINGLE_OPERATION(REDUCE_MIN, V1_3, kRelaxedCriteria);
729 TEST_SINGLE_OPERATION(ROI_ALIGN, V1_3, kRelaxedCriteria);
730 TEST_SINGLE_OPERATION(ROI_POOLING, V1_3, kRelaxedCriteria);
731 TEST_SINGLE_OPERATION(SELECT, V1_3, kStrictCriteria);
732 TEST_SINGLE_OPERATION(SLICE, V1_3, kStrictCriteria);
733 TEST_SINGLE_OPERATION(SPLIT, V1_3, kMediumCriteria);
734 TEST_SINGLE_OPERATION(TILE, V1_3, kStrictCriteria);
735 TEST_SINGLE_OPERATION(TOPK_V2, V1_3, kStrictCriteria);
736 TEST_SINGLE_OPERATION(TRANSPOSE_CONV_2D, V1_3, kConvCriteria);
737 TEST_SINGLE_OPERATION(RESIZE_NEAREST_NEIGHBOR, V1_3, kRelaxedCriteria);
738
739 /*-- NNAPI 1.3 Operations ---------------------------------------------------*/
740
741 // TODO: The following 1.3 operation signatures are currently not defined:
742 // - ANEURALNETWORKS_QUANTIZED_LSTM
743 // - ANEURALNETWORKS_IF
744 // - ANEURALNETWORKS_WHILE
745
746 TEST_SINGLE_OPERATION(ELU, V1_3, kMediumCriteria);
747 TEST_SINGLE_OPERATION(HARD_SWISH, V1_3, kMediumCriteria);
748 TEST_SINGLE_OPERATION(FILL, V1_3, kStrictCriteria);
749 TEST_SINGLE_OPERATION(RANK, V1_3, kStrictCriteria);
750
751 const AccuracyCriteria kSmallGraphCriteria = {
752 .float32 = {.bias = 4e-4f, .mse = 1e-5f, .atol = 1e-2f, .rtol = 1e-2f},
753 .float16 = {.bias = 5e-2f, .mse = 1e-2f, .atol = 1.0f, .rtol = 1.0f},
754 .int32 = {.atol = 1},
755 .quant8Asymm = {.bias = 2, .mse = 2, .atol = 12},
756 .quant8AsymmSigned = {.bias = 2, .mse = 2, .atol = 12},
757 .quant8Symm = {.bias = 2, .mse = 2, .atol = 12},
758 .quant16Asymm = {.bias = 2, .mse = 2, .atol = 12},
759 .quant16Symm = {.bias = 2, .mse = 2, .atol = 12},
760 };
761
762 const AccuracyCriteria kLargeGraphCriteria = {
763 .float32 = {.bias = 1e-2f, .mse = 1e-4f, .atol = 1e-1f, .rtol = 1e-1f},
764 .float16 = {.bias = 1e-1f, .mse = 5e-2f, .atol = 1.0f, .rtol = 1.0f},
765 .int32 = {.atol = 1},
766 .quant8Asymm = {.bias = 2, .mse = 2, .atol = 12},
767 .quant8AsymmSigned = {.bias = 2, .mse = 2, .atol = 12},
768 .quant8Symm = {.bias = 2, .mse = 2, .atol = 12},
769 .quant16Asymm = {.bias = 2, .mse = 2, .atol = 12},
770 .quant16Symm = {.bias = 2, .mse = 2, .atol = 12},
771 };
772
773 // Due to the limitation of the random graph generator, graphs generated with mixed-type or
774 // mixed-rank operations are likely to result in a disconnected network. Thus, we filter the
775 // operation signatures by primary data type and rank first, then generate random graph tests for
776 // each combination.
777 //
778 // Two parameterized tests are created for each filter:
779 // * 5-op graph with dimensions in range [1, 1000].
780 // * 40-op graph with dimensions in range [1, 10].
781 //
782 #define TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(dataType, rank) \
783 TEST_P(RandomGraphTest, SmallGraph_##dataType##_Rank##rank) { \
784 OperationFilter filter = {.dataTypes = {TestOperandType::dataType}, .ranks = {rank}}; \
785 OperationManager::get()->applyFilter(filter); \
786 mCriteria = kSmallGraphCriteria; \
787 testRandomGraph(GraphSize::SMALL, DimensionRange::WIDE); \
788 } \
789 TEST_P(RandomGraphTest, LargeGraph_##dataType##_Rank##rank) { \
790 OperationFilter filter = {.dataTypes = {TestOperandType::dataType}, .ranks = {rank}}; \
791 OperationManager::get()->applyFilter(filter); \
792 mCriteria = kLargeGraphCriteria; \
793 testRandomGraph(GraphSize::LARGE, DimensionRange::NARROW); \
794 }
795
796 // Random graph test with TENSOR_QUANT8_ASYMM as the primary data type is currently not defined.
797 // The generated graph with TENSOR_QUANT8_ASYMM as the primary data type will likely to result in
798 // disconnected graphs due to the mismatch between quantized parameters.
799
800 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT32, 4);
801 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT32, 3);
802 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT32, 2);
803 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT32, 1);
804
805 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT16, 4);
806 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT16, 3);
807 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT16, 2);
808 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_FLOAT16, 1);
809
810 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_INT32, 4);
811 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_INT32, 3);
812 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_INT32, 2);
813 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_INT32, 1);
814
815 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_BOOL8, 4);
816 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_BOOL8, 3);
817 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_BOOL8, 2);
818 TEST_RANDOM_GRAPH_WITH_DATA_TYPE_AND_RANK(TENSOR_BOOL8, 1);
819
820 INSTANTIATE_TEST_CASE_P(TestRandomGraph, SingleOperationTest, ::testing::Range(0u, 50u));
821 INSTANTIATE_TEST_CASE_P(TestRandomGraph, RandomGraphTest, ::testing::Range(0u, 50u));
822
823 } // namespace fuzzing_test
824 } // namespace nn
825 } // namespace android
826