1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Operations"
18
19 #include <tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h>
20 #include <tensorflow/lite/kernels/internal/reference/integer_ops/conv.h>
21 #include <tensorflow/lite/kernels/internal/types.h>
22
23 #include <algorithm>
24 #include <iterator>
25 #include <memory>
26 #include <vector>
27
28 #include "CpuOperationUtils.h"
29 #include "HalInterfaces.h"
30 #include "OperationResolver.h"
31 #include "Operations.h"
32 #include "OperationsUtils.h"
33 #include "Tracing.h"
34 #include "Utils.h"
35
36 namespace android {
37 namespace nn {
38 namespace conv_2d {
39
40 constexpr char kOperationName[] = "CONV_2D";
41
42 constexpr uint32_t kNumInputsArray[] = {7, 8, 10, 11, 13};
43 constexpr uint32_t kInputTensor = 0;
44 constexpr uint32_t kFilterTensor = 1;
45 constexpr uint32_t kBiasTensor = 2;
46
47 constexpr uint32_t kNumOutputs = 1;
48 constexpr uint32_t kOutputTensor = 0;
49
50 namespace {
51
52 using namespace hal;
53
54 // If possible we will use this static buffer for the tensor.
55 constexpr size_t kStaticBufferSize = 1605632;
56 char static_scratch_buffer[kStaticBufferSize];
57
58 // executionMutex is used to protect concurrent access of the static_scratch_buffer
59 // and other non-threadsafe resources like gemmlowp::GemmContext.
60 // std::mutex is safe for pthreads on Android.
61 std::mutex executionMutex;
62
63 struct Conv2dParam {
64 int32_t padding_left, padding_right;
65 int32_t padding_top, padding_bottom;
66 int32_t stride_width, stride_height;
67 int32_t dilation_width_factor = 1, dilation_height_factor = 1;
68 int32_t activation;
69 bool useNchw = false;
70
initializeandroid::nn::conv_2d::__anon14bee14a0111::Conv2dParam71 bool initialize(const IOperationExecutionContext* context) {
72 uint32_t inCount = context->getNumInputs();
73 int32_t padding_implicit = 0;
74 bool useImplicitPadding = false;
75 if ((inCount >= 8 && context->getInputType(7) == OperandType::BOOL) || inCount == 7) {
76 padding_implicit = context->getInputValue<int32_t>(3);
77 stride_width = context->getInputValue<int32_t>(4);
78 stride_height = context->getInputValue<int32_t>(5);
79 activation = context->getInputValue<int32_t>(6);
80 if (inCount >= 8) {
81 useNchw = context->getInputValue<bool>(7);
82 }
83 if (inCount == 10) {
84 dilation_width_factor = context->getInputValue<int32_t>(8);
85 dilation_height_factor = context->getInputValue<int32_t>(9);
86 }
87 useImplicitPadding = true;
88 } else if (inCount >= 10 && context->getInputType(7) == OperandType::INT32) {
89 padding_left = context->getInputValue<int32_t>(3);
90 padding_right = context->getInputValue<int32_t>(4);
91 padding_top = context->getInputValue<int32_t>(5);
92 padding_bottom = context->getInputValue<int32_t>(6);
93 stride_width = context->getInputValue<int32_t>(7);
94 stride_height = context->getInputValue<int32_t>(8);
95 activation = context->getInputValue<int32_t>(9);
96 if (inCount >= 11) {
97 useNchw = context->getInputValue<bool>(10);
98 }
99 if (inCount == 13) {
100 dilation_width_factor = context->getInputValue<int32_t>(11);
101 dilation_height_factor = context->getInputValue<int32_t>(12);
102 }
103 } else {
104 NN_RET_CHECK_FAIL() << "Unsupported input spec for operation " << kOperationName;
105 }
106 if (useImplicitPadding) {
107 Shape inputShape = context->getInputShape(kInputTensor);
108 Shape filterShape = context->getInputShape(kFilterTensor);
109 int32_t input_width = getSizeOfDimension(inputShape, useNchw ? 3 : 2);
110 int32_t input_height = getSizeOfDimension(inputShape, useNchw ? 2 : 1);
111 int32_t filter_width = getSizeOfDimension(filterShape, 2);
112 int32_t filter_height = getSizeOfDimension(filterShape, 1);
113 calculateExplicitPadding(input_width, stride_width, dilation_width_factor, filter_width,
114 padding_implicit, &padding_left, &padding_right);
115 calculateExplicitPadding(input_height, stride_height, dilation_height_factor,
116 filter_height, padding_implicit, &padding_top,
117 &padding_bottom);
118 }
119 NN_RET_CHECK_GE(padding_left, 0);
120 NN_RET_CHECK_GE(padding_right, 0);
121 NN_RET_CHECK_GE(padding_top, 0);
122 NN_RET_CHECK_GE(padding_bottom, 0);
123 NN_RET_CHECK_GT(stride_width, 0);
124 NN_RET_CHECK_GT(stride_height, 0);
125 NN_RET_CHECK_GT(dilation_width_factor, 0);
126 NN_RET_CHECK_GT(dilation_height_factor, 0);
127 NN_RET_CHECK_GE(activation, 0);
128 return true;
129 }
130 };
131
132 #define ANDROID_NN_CONV_PARAMETERS(Type) \
133 uint32_t height = getSizeOfDimension(inputShape, 1); \
134 uint32_t width = getSizeOfDimension(inputShape, 2); \
135 uint32_t filterHeight = getSizeOfDimension(filterShape, 1); \
136 uint32_t filterWidth = getSizeOfDimension(filterShape, 2); \
137 uint32_t outHeight = getSizeOfDimension(outputShape, 1); \
138 uint32_t outWidth = getSizeOfDimension(outputShape, 2); \
139 uint32_t inDepth = getSizeOfDimension(inputShape, 3); \
140 \
141 uint32_t paddingHeight = (uint32_t)padding_top; \
142 uint32_t paddingWidth = (uint32_t)padding_left; \
143 \
144 tflite::Dims<4> im2colDim; \
145 im2colDim.sizes[3] = (int)getSizeOfDimension(outputShape, 0); \
146 im2colDim.sizes[2] = (int)getSizeOfDimension(outputShape, 1); \
147 im2colDim.sizes[1] = (int)getSizeOfDimension(outputShape, 2); \
148 im2colDim.sizes[0] = (int)inDepth * filterHeight * filterWidth; \
149 \
150 im2colDim.strides[0] = 1; \
151 for (int i = 1; i < 4; i++) { \
152 im2colDim.strides[i] = im2colDim.strides[i - 1] * im2colDim.sizes[i - 1]; \
153 } \
154 \
155 Type* im2colData = nullptr; \
156 uint64_t im2colByteSize = sizeof(Type); \
157 std::unique_ptr<Type[]> im2colGuard; \
158 for (int i = 0; i < 4; i++) { \
159 im2colByteSize *= im2colDim.sizes[i]; \
160 } \
161 /* http://b/77982879, tflite::optimized_ops::Conv uses int for offsets */ \
162 if (im2colByteSize >= 0x7fffffff) { \
163 LOG(ERROR) << "Conv size is too large, not enough memory"; \
164 return false; \
165 } \
166 if (im2colByteSize <= kStaticBufferSize) { \
167 im2colData = reinterpret_cast<Type*>(static_scratch_buffer); \
168 } else { \
169 im2colData = new (std::nothrow) Type[im2colByteSize / sizeof(Type)]; \
170 if (im2colData == nullptr) { \
171 LOG(ERROR) << "Conv size is too large, not enough memory"; \
172 return false; \
173 } \
174 im2colGuard.reset(im2colData); \
175 }
176
needim2colData(const Shape & filterShape,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor)177 bool needim2colData(const Shape& filterShape, int32_t stride_width, int32_t stride_height,
178 int32_t dilation_width_factor, int32_t dilation_height_factor) {
179 // Within tflite::optimized_ops::Conv, the following tests are performed,
180 // and in the case (!need_dilated_im2col && !need_im2col), then the
181 // method doesn't expect to receive outputData. In debug mode this is
182 // asserted and fails tests, so we need to perform this check as the caller
183 // also. See:
184 // tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h:2655
185 const int filter_width = getSizeOfDimension(filterShape, 2);
186 const int filter_height = getSizeOfDimension(filterShape, 1);
187 const bool need_dilated_im2col = dilation_width_factor != 1 || dilation_height_factor != 1;
188 const bool need_im2col =
189 stride_width != 1 || stride_height != 1 || filter_width != 1 || filter_height != 1;
190 return need_dilated_im2col || need_im2col;
191 }
192
convNhwc(const float * inputData,const Shape & inputShape,const float * filterData,const Shape & filterShape,const float * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,float * outputData,const Shape & outputShape)193 bool convNhwc(const float* inputData, const Shape& inputShape, const float* filterData,
194 const Shape& filterShape, const float* biasData, const Shape& biasShape,
195 int32_t padding_left, int32_t padding_right, int32_t padding_top,
196 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
197 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
198 float* outputData, const Shape& outputShape) {
199 NNTRACE_TRANS("convFloat32");
200
201 ANDROID_NN_CONV_PARAMETERS(float)
202
203 float output_activation_min, output_activation_max;
204 CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max);
205
206 // Prevent concurrent executions that may access the scratch buffer.
207 std::unique_lock<std::mutex> lock(executionMutex);
208 NNTRACE_COMP_SWITCH("optimized_ops::Conv");
209
210 const bool need_im2colData = needim2colData(filterShape, stride_width, stride_height,
211 dilation_width_factor, dilation_height_factor);
212
213 tflite::optimized_ops::Conv(
214 inputData, convertShapeToDims(inputShape), filterData, convertShapeToDims(filterShape),
215 biasData, convertShapeToDims(biasShape), stride_width, stride_height,
216 dilation_width_factor, dilation_height_factor, paddingWidth, paddingHeight,
217 output_activation_min, output_activation_max, outputData,
218 convertShapeToDims(outputShape), need_im2colData ? im2colData : nullptr, im2colDim);
219 return true;
220 }
221
convNhwc(const uint8_t * inputData,const Shape & inputShape,const uint8_t * filterData,const Shape & filterShape,const int32_t * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,uint8_t * outputData,const Shape & outputShape)222 bool convNhwc(const uint8_t* inputData, const Shape& inputShape, const uint8_t* filterData,
223 const Shape& filterShape, const int32_t* biasData, const Shape& biasShape,
224 int32_t padding_left, int32_t padding_right, int32_t padding_top,
225 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
226 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
227 uint8_t* outputData, const Shape& outputShape) {
228 NNTRACE_TRANS("convQuant8");
229
230 ANDROID_NN_CONV_PARAMETERS(uint8_t)
231
232 int32_t inputOffset = -inputShape.offset;
233 int32_t filterOffset = -filterShape.offset;
234 int32_t outputOffset = outputShape.offset;
235
236 double real_multiplier = 0.0;
237 int32_t output_multiplier = 0;
238 int32_t output_shift = 0;
239 int32_t output_activation_min = 0;
240 int32_t output_activation_max = 0;
241
242 NN_RET_CHECK(GetQuantizedConvolutionMultipler(inputShape, filterShape, biasShape, outputShape,
243 &real_multiplier));
244 int exponent;
245 NN_RET_CHECK(QuantizeMultiplier(real_multiplier, &output_multiplier, &exponent));
246 output_shift = -exponent;
247 CalculateActivationRangeUint8(activation, outputShape, &output_activation_min,
248 &output_activation_max);
249
250 static gemmlowp::GemmContext gemm_context;
251
252 // Prevent concurrent executions that may access the scratch buffer and
253 // gemm_context.
254 std::unique_lock<std::mutex> lock(executionMutex);
255 // Alow gemmlowp automatically decide how many threads to use.
256 gemm_context.set_max_num_threads(0);
257
258 NNTRACE_COMP_SWITCH("optimized_ops::Conv");
259
260 const bool need_im2colData = needim2colData(filterShape, stride_width, stride_height,
261 dilation_width_factor, dilation_height_factor);
262
263 tflite::optimized_ops::Conv(inputData, convertShapeToDims(inputShape), inputOffset, filterData,
264 convertShapeToDims(filterShape), filterOffset, biasData,
265 convertShapeToDims(biasShape), stride_width, stride_height,
266 dilation_width_factor, dilation_height_factor, paddingWidth,
267 paddingHeight, outputOffset, output_multiplier, output_shift,
268 output_activation_min, output_activation_max, outputData,
269 convertShapeToDims(outputShape),
270 need_im2colData ? im2colData : nullptr, im2colDim, &gemm_context);
271 return true;
272 }
273
274 // Passing input, filter and output shapes by value, so that we can change the
275 // offsets without modifying the actual shapes.
convNhwc(const int8_t * inputData,Shape inputShape,const int8_t * filterData,Shape filterShape,const int32_t * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,int8_t * outputData,Shape outputShape)276 bool convNhwc(const int8_t* inputData, Shape inputShape, const int8_t* filterData,
277 Shape filterShape, const int32_t* biasData, const Shape& biasShape,
278 int32_t padding_left, int32_t padding_right, int32_t padding_top,
279 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
280 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
281 int8_t* outputData, Shape outputShape) {
282 NNTRACE_TRANS("convQuant8");
283
284 std::vector<uint8_t> unsignedInput(getNumberOfElements(inputShape));
285 convertInt8ToUInt8(inputData, &unsignedInput);
286 inputShape.offset += 128;
287
288 std::vector<uint8_t> unsignedFilter(getNumberOfElements(filterShape));
289 convertInt8ToUInt8(filterData, &unsignedFilter);
290 filterShape.offset += 128;
291
292 std::vector<uint8_t> unsignedOutput(getNumberOfElements(outputShape));
293 outputShape.offset += 128;
294
295 NN_RET_CHECK(convNhwc(unsignedInput.data(), inputShape, unsignedFilter.data(), filterShape,
296 biasData, biasShape, padding_left, padding_right, padding_top,
297 padding_bottom, stride_width, stride_height, dilation_width_factor,
298 dilation_height_factor, activation, unsignedOutput.data(), outputShape));
299
300 convertUInt8ToInt8(unsignedOutput, outputData);
301
302 return true;
303 }
304
convNhwc(const _Float16 * inputData,const Shape & inputShape,const _Float16 * filterData,const Shape & filterShape,const _Float16 * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,_Float16 * outputData,const Shape & outputShape)305 bool convNhwc(const _Float16* inputData, const Shape& inputShape, const _Float16* filterData,
306 const Shape& filterShape, const _Float16* biasData, const Shape& biasShape,
307 int32_t padding_left, int32_t padding_right, int32_t padding_top,
308 int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
309 int32_t dilation_width_factor, int32_t dilation_height_factor, int32_t activation,
310 _Float16* outputData, const Shape& outputShape) {
311 NNTRACE_TRANS("convFloat16");
312
313 std::vector<float> inputData_float32(getNumberOfElements(inputShape));
314 std::vector<float> filterData_float32(getNumberOfElements(filterShape));
315 std::vector<float> biasData_float32(getNumberOfElements(biasShape));
316 std::vector<float> outputData_float32(getNumberOfElements(outputShape));
317
318 convertFloat16ToFloat32(inputData, &inputData_float32);
319 convertFloat16ToFloat32(filterData, &filterData_float32);
320 convertFloat16ToFloat32(biasData, &biasData_float32);
321
322 convNhwc(inputData_float32.data(), inputShape, filterData_float32.data(), filterShape,
323 biasData_float32.data(), biasShape, padding_left, padding_right, padding_top,
324 padding_bottom, stride_width, stride_height, dilation_width_factor,
325 dilation_height_factor, activation, outputData_float32.data(), outputShape);
326 convertFloat32ToFloat16(outputData_float32, outputData);
327
328 return true;
329 }
330
331 template <typename T_Input, typename T_Filter, typename T_Bias>
conv(const T_Input * inputData,const Shape & inputShape,const T_Filter * filterData,const Shape & filterShape,const T_Bias * biasData,const Shape & biasShape,int32_t padding_left,int32_t padding_right,int32_t padding_top,int32_t padding_bottom,int32_t stride_width,int32_t stride_height,int32_t dilation_width_factor,int32_t dilation_height_factor,int32_t activation,bool useNchw,T_Input * outputData,const Shape & outputShape)332 bool conv(const T_Input* inputData, const Shape& inputShape, const T_Filter* filterData,
333 const Shape& filterShape, const T_Bias* biasData, const Shape& biasShape,
334 int32_t padding_left, int32_t padding_right, int32_t padding_top, int32_t padding_bottom,
335 int32_t stride_width, int32_t stride_height, int32_t dilation_width_factor,
336 int32_t dilation_height_factor, int32_t activation, bool useNchw, T_Input* outputData,
337 const Shape& outputShape) {
338 InputWithLayout<T_Input> input(useNchw);
339 OutputWithLayout<T_Input> output(useNchw);
340 NN_RET_CHECK(input.initialize(inputData, inputShape));
341 NN_RET_CHECK(output.initialize(outputData, outputShape));
342 NN_RET_CHECK(convNhwc(input.getNhwcBuffer(), input.getNhwcShape(), filterData, filterShape,
343 biasData, biasShape, padding_left, padding_right, padding_top,
344 padding_bottom, stride_width, stride_height, dilation_width_factor,
345 dilation_height_factor, activation, output.getNhwcBuffer(),
346 output.getNhwcShape()));
347 NN_RET_CHECK(output.commit());
348 return true;
349 }
350
convQuant8PerChannelNhwc(const uint8_t * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t activation,uint8_t * outputData,const Shape & outputShape)351 bool convQuant8PerChannelNhwc(const uint8_t* inputData, const Shape& inputShape,
352 const int8_t* filterData, const Shape& filterShape,
353 const float* filterScales, const int32_t* biasData,
354 const Shape& biasShape, int32_t paddingLeft, int32_t paddingRight,
355 int32_t paddingTop, int32_t paddingBottom, int32_t strideWidth,
356 int32_t strideHeight, int32_t dilationWidthFactor,
357 int32_t dilationHeightFactor, int32_t activation, uint8_t* outputData,
358 const Shape& outputShape) {
359 NNTRACE_TRANS("convQuant8PerChannel");
360
361 uint32_t numBatches = getSizeOfDimension(inputShape, 0);
362 uint32_t inputHeight = getSizeOfDimension(inputShape, 1);
363 uint32_t inputWidth = getSizeOfDimension(inputShape, 2);
364 uint32_t inputDepth = getSizeOfDimension(inputShape, 3);
365 uint32_t filterHeight = getSizeOfDimension(filterShape, 1);
366 uint32_t filterWidth = getSizeOfDimension(filterShape, 2);
367 uint32_t filterDepth = getSizeOfDimension(filterShape, 3);
368 uint32_t outputHeight = getSizeOfDimension(outputShape, 1);
369 uint32_t outputWidth = getSizeOfDimension(outputShape, 2);
370 uint32_t outputDepth = getSizeOfDimension(outputShape, 3);
371
372 int32_t inputOffset = -inputShape.offset;
373 int32_t outputOffset = outputShape.offset;
374
375 auto realMultiplier = std::vector<double>(outputDepth, .0f);
376 auto outputMultiplier = std::vector<int32_t>(outputDepth, 0);
377 auto outputShift = std::vector<int32_t>(outputDepth, .0f);
378
379 for (int i = 0; i < outputDepth; ++i) {
380 Shape filterChannelShape = filterShape;
381 filterChannelShape.scale = filterScales[i];
382 Shape biasChannelShape = biasShape;
383 biasChannelShape.scale = filterScales[i] * inputShape.scale;
384 NN_RET_CHECK(GetQuantizedConvolutionMultipler(
385 inputShape, filterChannelShape, biasChannelShape, outputShape, &realMultiplier[i]));
386 int exponent;
387 NN_RET_CHECK(QuantizeMultiplier(realMultiplier[i], &outputMultiplier[i], &exponent));
388 outputShift[i] = -exponent;
389 }
390
391 int32_t output_activation_min = 0, output_activation_max = 0;
392 CalculateActivationRangeUint8(activation, outputShape, &output_activation_min,
393 &output_activation_max);
394 const uint8_t* inputBase = inputData;
395 uint8_t* outPtr = outputData;
396 for (uint32_t b = 0; b < numBatches; b++) {
397 for (uint32_t h = 0; h < outputHeight; h++) {
398 for (uint32_t w = 0; w < outputWidth; w++) {
399 const int8_t* filterBase = filterData;
400
401 for (uint32_t d = 0; d < outputDepth; d++) {
402 int32_t wInputOrigin = static_cast<int32_t>(w) * strideWidth - paddingLeft;
403 int32_t hInputOrigin = static_cast<int32_t>(h) * strideHeight - paddingTop;
404 int32_t sum = 0.0f;
405
406 for (uint32_t i = 0; i < filterHeight; i++) {
407 for (uint32_t j = 0; j < filterWidth; j++) {
408 for (uint32_t k = 0; k < filterDepth; k++) {
409 int32_t hInput = hInputOrigin +
410 dilationHeightFactor * static_cast<int32_t>(i);
411 int32_t wInput = wInputOrigin +
412 dilationWidthFactor * static_cast<int32_t>(j);
413 uint32_t dInput = k;
414 if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) &&
415 wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) {
416 uint32_t filterIndex =
417 i * filterWidth * filterDepth + j * filterDepth + k;
418 uint32_t inputIndex = hInput * inputWidth * inputDepth +
419 wInput * inputDepth + dInput;
420 sum += (static_cast<int32_t>(filterBase[filterIndex])) *
421 (static_cast<int32_t>(inputBase[inputIndex]) +
422 inputOffset);
423 }
424 }
425 }
426 }
427 sum += biasData[d];
428 sum = tflite::MultiplyByQuantizedMultiplier(sum, outputMultiplier[d],
429 -outputShift[d]);
430 sum += outputOffset;
431 sum = std::max(std::min(sum, output_activation_max), output_activation_min);
432 outPtr[d] = static_cast<uint8_t>(sum);
433 filterBase += filterHeight * filterWidth * filterDepth;
434 }
435 outPtr += outputDepth;
436 }
437 }
438 inputBase += inputHeight * inputWidth * inputDepth;
439 }
440
441 return true;
442 }
443
convQuant8PerChannelNhwc(const int8_t * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t activation,int8_t * outputData,const Shape & outputShape)444 bool convQuant8PerChannelNhwc(const int8_t* inputData, const Shape& inputShape,
445 const int8_t* filterData, const Shape& filterShape,
446 const float* filterScales, const int32_t* biasData,
447 const Shape& biasShape, int32_t paddingLeft, int32_t paddingRight,
448 int32_t paddingTop, int32_t paddingBottom, int32_t strideWidth,
449 int32_t strideHeight, int32_t dilationWidthFactor,
450 int32_t dilationHeightFactor, int32_t activation, int8_t* outputData,
451 const Shape& outputShape) {
452 NNTRACE_TRANS("convQuant8SignedPerChannel");
453
454 uint32_t numBatches = getSizeOfDimension(inputShape, 0);
455 uint32_t inputHeight = getSizeOfDimension(inputShape, 1);
456 uint32_t inputWidth = getSizeOfDimension(inputShape, 2);
457 uint32_t inputDepth = getSizeOfDimension(inputShape, 3);
458 uint32_t filterHeight = getSizeOfDimension(filterShape, 1);
459 uint32_t filterWidth = getSizeOfDimension(filterShape, 2);
460 uint32_t filterDepth = getSizeOfDimension(filterShape, 3);
461 uint32_t outputHeight = getSizeOfDimension(outputShape, 1);
462 uint32_t outputWidth = getSizeOfDimension(outputShape, 2);
463 uint32_t outputDepth = getSizeOfDimension(outputShape, 3);
464
465 int32_t inputOffset = -inputShape.offset;
466 int32_t outputOffset = outputShape.offset;
467
468 auto realMultiplier = std::vector<double>(outputDepth, .0f);
469 auto outputMultiplier = std::vector<int32_t>(outputDepth, 0);
470 auto outputShift = std::vector<int32_t>(outputDepth, .0f);
471
472 for (int i = 0; i < outputDepth; ++i) {
473 Shape filterChannelShape = filterShape;
474 filterChannelShape.scale = filterScales[i];
475 Shape biasChannelShape = biasShape;
476 biasChannelShape.scale = filterScales[i] * inputShape.scale;
477 NN_RET_CHECK(GetQuantizedConvolutionMultipler(
478 inputShape, filterChannelShape, biasChannelShape, outputShape, &realMultiplier[i]));
479 NN_RET_CHECK(QuantizeMultiplier(realMultiplier[i], &outputMultiplier[i], &outputShift[i]));
480 }
481
482 int32_t output_activation_min = 0, output_activation_max = 0;
483 CalculateActivationRangeInt8(activation, outputShape, &output_activation_min,
484 &output_activation_max);
485
486 tflite::ConvParams convParams;
487 convParams.input_offset = -inputShape.offset;
488 convParams.output_offset = outputShape.offset;
489 convParams.stride_height = strideHeight;
490 convParams.stride_width = strideWidth;
491 convParams.dilation_height_factor = dilationHeightFactor;
492 convParams.dilation_width_factor = dilationWidthFactor;
493 convParams.padding_values.height = paddingTop;
494 convParams.padding_values.width = paddingLeft;
495 convParams.quantized_activation_min = output_activation_min;
496 convParams.quantized_activation_max = output_activation_max;
497
498 NNTRACE_COMP_SWITCH("reference_integer_ops::ConvPerChannel");
499 tflite::reference_integer_ops::ConvPerChannel(
500 convParams, outputMultiplier.data(), outputShift.data(),
501 convertShapeToTflshape(inputShape), inputData, convertShapeToTflshape(filterShape),
502 filterData, convertShapeToTflshape(biasShape), biasData,
503 convertShapeToTflshape(outputShape), outputData);
504 return true;
505 }
506
507 template <typename T>
convQuant8PerChannel(const T * inputData,const Shape & inputShape,const int8_t * filterData,const Shape & filterShape,const float * filterScales,const int32_t * biasData,const Shape & biasShape,int32_t paddingLeft,int32_t paddingRight,int32_t paddingTop,int32_t paddingBottom,int32_t strideWidth,int32_t strideHeight,int32_t dilationWidthFactor,int32_t dilationHeightFactor,int32_t activation,bool useNchw,T * outputData,const Shape & outputShape)508 bool convQuant8PerChannel(const T* inputData, const Shape& inputShape, const int8_t* filterData,
509 const Shape& filterShape, const float* filterScales,
510 const int32_t* biasData, const Shape& biasShape, int32_t paddingLeft,
511 int32_t paddingRight, int32_t paddingTop, int32_t paddingBottom,
512 int32_t strideWidth, int32_t strideHeight, int32_t dilationWidthFactor,
513 int32_t dilationHeightFactor, int32_t activation, bool useNchw,
514 T* outputData, const Shape& outputShape) {
515 InputWithLayout<T> input(useNchw);
516 OutputWithLayout<T> output(useNchw);
517 NN_RET_CHECK(input.initialize(inputData, inputShape));
518 NN_RET_CHECK(output.initialize(outputData, outputShape));
519 NN_RET_CHECK(convQuant8PerChannelNhwc(
520 input.getNhwcBuffer(), input.getNhwcShape(), filterData, filterShape, filterScales,
521 biasData, biasShape, paddingLeft, paddingRight, paddingTop, paddingBottom, strideWidth,
522 strideHeight, dilationWidthFactor, dilationHeightFactor, activation,
523 output.getNhwcBuffer(), output.getNhwcShape()));
524 NN_RET_CHECK(output.commit());
525 return true;
526 }
527
528 #undef ANDROID_NN_CONV_PARAMETERS
529
530 } // namespace
531
validate(const IOperationValidationContext * context)532 bool validate(const IOperationValidationContext* context) {
533 const uint32_t numInputs = context->getNumInputs();
534 NN_RET_CHECK(
535 std::binary_search(std::begin(kNumInputsArray), std::end(kNumInputsArray), numInputs));
536 NN_RET_CHECK_EQ(context->getNumOutputs(), kNumOutputs);
537 const auto inputRank = getNumberOfDimensions(context->getInputShape(kInputTensor));
538 const auto filterRank = getNumberOfDimensions(context->getInputShape(kFilterTensor));
539 if (inputRank != 0) {
540 NN_RET_CHECK_EQ(inputRank, 4);
541 }
542 if (filterRank != 0) {
543 NN_RET_CHECK_EQ(filterRank, 4);
544 }
545 auto inputCount = context->getNumInputs();
546 auto inputType = context->getInputType(kInputTensor);
547 auto filterType = context->getInputType(kFilterTensor);
548 std::vector<OperandType> inExpectedTypes;
549 if (inputType == OperandType::TENSOR_FLOAT32) {
550 inExpectedTypes = {OperandType::TENSOR_FLOAT32, OperandType::TENSOR_FLOAT32,
551 OperandType::TENSOR_FLOAT32, OperandType::INT32,
552 OperandType::INT32, OperandType::INT32,
553 OperandType::INT32};
554 } else if (inputType == OperandType::TENSOR_FLOAT16) {
555 inExpectedTypes = {OperandType::TENSOR_FLOAT16, OperandType::TENSOR_FLOAT16,
556 OperandType::TENSOR_FLOAT16, OperandType::INT32,
557 OperandType::INT32, OperandType::INT32,
558 OperandType::INT32};
559 } else if (inputType == OperandType::TENSOR_QUANT8_ASYMM ||
560 inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
561 NN_RET_CHECK(filterType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL ||
562 filterType == inputType)
563 << "Unsupported filter tensor type for operation " << kOperationName;
564 inExpectedTypes = {inputType, filterType, OperandType::TENSOR_INT32,
565 OperandType::INT32, OperandType::INT32, OperandType::INT32,
566 OperandType::INT32};
567
568 if (filterType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
569 NN_RET_CHECK_EQ(context->getInputExtraParams(kFilterTensor).channelQuant().channelDim,
570 0)
571 << "Unsupported filter tensor channel dimension for operation "
572 << kOperationName;
573 }
574 } else {
575 NN_RET_CHECK_FAIL() << "Unsupported input tensor type for operation " << kOperationName;
576 }
577
578 // NeuralNetworks.h specifies that ANEURALNETWORKS_CONV_2D's output must
579 // meet "outputScale > inputScale * filterScale" for the operand type
580 // ANEURALNETWORKS_TENSOR_QUANT8_ASYMM before API level 29. For other
581 // operand types (e.g., ANEURALNETWORKS_TENSOR_FLOAT32), this constraint
582 // does not apply, so by default the constraint is met.
583 bool meetsQuantizedScaleConstraintBeforeV1_2 = true;
584 if (inputType == OperandType::TENSOR_QUANT8_ASYMM) {
585 const float inputScale = context->getInputShape(kInputTensor).scale;
586 const float filterScale = context->getInputShape(kFilterTensor).scale;
587 const float outputScale = context->getInputShape(kOutputTensor).scale;
588 meetsQuantizedScaleConstraintBeforeV1_2 = (outputScale > inputScale * filterScale);
589 }
590
591 bool withExplicitPadding = false;
592 bool withLayout = false;
593 bool withDilation = false;
594 if (inputCount >= 8) {
595 if (context->getInputType(7) == OperandType::INT32 && inputCount >= 10) {
596 std::vector<OperandType> explicitScalarTypes(3, OperandType::INT32);
597 inExpectedTypes.insert(inExpectedTypes.end(), explicitScalarTypes.begin(),
598 explicitScalarTypes.end());
599 withExplicitPadding = true;
600 }
601 int inputOffset = withExplicitPadding ? 3 : 0;
602 if (inputCount >= 8 + inputOffset) {
603 inExpectedTypes.push_back(OperandType::BOOL);
604 withLayout = true;
605 }
606 NN_RET_CHECK_NE(inputCount, 9 + inputOffset)
607 << "Provided only one dilation factor value, two values are requred for operation "
608 << kOperationName;
609 if (inputCount == 10 + inputOffset) {
610 inExpectedTypes.push_back(OperandType::INT32);
611 inExpectedTypes.push_back(OperandType::INT32);
612 withDilation = true;
613 }
614 }
615
616 if (inputType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
617 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_3));
618 } else if (inputType == OperandType::TENSOR_FLOAT16 ||
619 filterType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL || withLayout ||
620 withDilation || !meetsQuantizedScaleConstraintBeforeV1_2) {
621 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_2));
622 } else {
623 NN_RET_CHECK(validateHalVersion(context, HalVersion::V1_0));
624 }
625 return validateInputTypes(context, inExpectedTypes) &&
626 validateOutputTypes(context, {inputType});
627 }
628
prepare(IOperationExecutionContext * context)629 bool prepare(IOperationExecutionContext* context) {
630 Shape input = context->getInputShape(kInputTensor);
631 Shape filter = context->getInputShape(kFilterTensor);
632 Shape bias = context->getInputShape(kBiasTensor);
633
634 if (filter.type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
635 NN_RET_CHECK(input.type == OperandType::TENSOR_QUANT8_ASYMM ||
636 input.type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED);
637 } else {
638 NN_RET_CHECK(input.type == filter.type);
639 }
640 if (input.type == OperandType::TENSOR_QUANT8_ASYMM ||
641 input.type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
642 NN_RET_CHECK(bias.type == OperandType::TENSOR_INT32);
643 } else {
644 NN_RET_CHECK(input.type == bias.type);
645 }
646 NN_RET_CHECK_EQ(getNumberOfDimensions(input), 4);
647 NN_RET_CHECK_EQ(getNumberOfDimensions(filter), 4);
648 NN_RET_CHECK_EQ(getNumberOfDimensions(bias), 1);
649
650 Conv2dParam param;
651 NN_RET_CHECK(param.initialize(context));
652
653 uint32_t batches = getSizeOfDimension(input, 0);
654 uint32_t height = getSizeOfDimension(input, param.useNchw ? 2 : 1);
655 uint32_t width = getSizeOfDimension(input, param.useNchw ? 3 : 2);
656 uint32_t channels_in = getSizeOfDimension(input, param.useNchw ? 1 : 3);
657 uint32_t channels_out = getSizeOfDimension(filter, 0);
658 uint32_t filterHeight = getSizeOfDimension(filter, 1);
659 uint32_t filterWidth = getSizeOfDimension(filter, 2);
660 // Only batches can be zero.
661 NN_RET_CHECK_EQ(channels_in, getSizeOfDimension(filter, 3));
662 NN_RET_CHECK_EQ(channels_out, getSizeOfDimension(bias, 0));
663 NN_RET_CHECK_GT(height, 0);
664 NN_RET_CHECK_GT(width, 0);
665 NN_RET_CHECK_GT(channels_in, 0);
666 NN_RET_CHECK_GT(channels_out, 0);
667
668 int32_t effectiveFilterWidth = (filterWidth - 1) * param.dilation_width_factor + 1;
669 int32_t effectiveFilterHeight = (filterHeight - 1) * param.dilation_height_factor + 1;
670 NN_RET_CHECK_GT(effectiveFilterWidth, param.padding_left);
671 NN_RET_CHECK_GT(effectiveFilterWidth, param.padding_right);
672 NN_RET_CHECK_GT(effectiveFilterHeight, param.padding_top);
673 NN_RET_CHECK_GT(effectiveFilterHeight, param.padding_bottom);
674
675 uint32_t outWidth =
676 computeOutSize(width, filterWidth, param.stride_width, param.dilation_width_factor,
677 param.padding_left, param.padding_right);
678 uint32_t outHeight =
679 computeOutSize(height, filterHeight, param.stride_height, param.dilation_height_factor,
680 param.padding_top, param.padding_bottom);
681
682 Shape output = context->getOutputShape(kOutputTensor);
683 output.type = input.type;
684 if (param.useNchw) {
685 output.dimensions = {batches, channels_out, outHeight, outWidth};
686 } else {
687 output.dimensions = {batches, outHeight, outWidth, channels_out};
688 }
689 return context->setOutputShape(kOutputTensor, output);
690 }
691
execute(IOperationExecutionContext * context)692 bool execute(IOperationExecutionContext* context) {
693 // Bypass execution in the case of zero-sized input.
694 if (getNumberOfElements(context->getOutputShape(kOutputTensor)) == 0) return true;
695 Conv2dParam param;
696 NN_RET_CHECK(param.initialize(context));
697 switch (context->getInputType(kInputTensor)) {
698 case OperandType::TENSOR_FLOAT32:
699 return conv(context->getInputBuffer<float>(kInputTensor),
700 context->getInputShape(kInputTensor),
701 context->getInputBuffer<float>(kFilterTensor),
702 context->getInputShape(kFilterTensor),
703 context->getInputBuffer<float>(kBiasTensor),
704 context->getInputShape(kBiasTensor), param.padding_left,
705 param.padding_right, param.padding_top, param.padding_bottom,
706 param.stride_width, param.stride_height, param.dilation_width_factor,
707 param.dilation_height_factor, param.activation, param.useNchw,
708 context->getOutputBuffer<float>(kOutputTensor),
709 context->getOutputShape(kOutputTensor));
710 case OperandType::TENSOR_FLOAT16:
711 return conv(context->getInputBuffer<_Float16>(kInputTensor),
712 context->getInputShape(kInputTensor),
713 context->getInputBuffer<_Float16>(kFilterTensor),
714 context->getInputShape(kFilterTensor),
715 context->getInputBuffer<_Float16>(kBiasTensor),
716 context->getInputShape(kBiasTensor), param.padding_left,
717 param.padding_right, param.padding_top, param.padding_bottom,
718 param.stride_width, param.stride_height, param.dilation_width_factor,
719 param.dilation_height_factor, param.activation, param.useNchw,
720 context->getOutputBuffer<_Float16>(kOutputTensor),
721 context->getOutputShape(kOutputTensor));
722 case OperandType::TENSOR_QUANT8_ASYMM:
723 if (context->getInputType(kFilterTensor) ==
724 OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
725 return convQuant8PerChannel(
726 context->getInputBuffer<uint8_t>(kInputTensor),
727 context->getInputShape(kInputTensor),
728 context->getInputBuffer<int8_t>(kFilterTensor),
729 context->getInputShape(kFilterTensor),
730 context->getInputExtraParams(kFilterTensor).channelQuant().scales.data(),
731 context->getInputBuffer<int32_t>(kBiasTensor),
732 context->getInputShape(kBiasTensor), param.padding_left,
733 param.padding_right, param.padding_top, param.padding_bottom,
734 param.stride_width, param.stride_height, param.dilation_width_factor,
735 param.dilation_height_factor, param.activation, param.useNchw,
736 context->getOutputBuffer<uint8_t>(kOutputTensor),
737 context->getOutputShape(kOutputTensor));
738 } else if (context->getInputType(kFilterTensor) == OperandType::TENSOR_QUANT8_ASYMM) {
739 return conv(context->getInputBuffer<uint8_t>(kInputTensor),
740 context->getInputShape(kInputTensor),
741 context->getInputBuffer<uint8_t>(kFilterTensor),
742 context->getInputShape(kFilterTensor),
743 context->getInputBuffer<int32_t>(kBiasTensor),
744 context->getInputShape(kBiasTensor), param.padding_left,
745 param.padding_right, param.padding_top, param.padding_bottom,
746 param.stride_width, param.stride_height, param.dilation_width_factor,
747 param.dilation_height_factor, param.activation, param.useNchw,
748 context->getOutputBuffer<uint8_t>(kOutputTensor),
749 context->getOutputShape(kOutputTensor));
750 } else {
751 NN_RET_CHECK_FAIL() << "Unsupported filter type for operation " << kOperationName;
752 }
753 case OperandType::TENSOR_QUANT8_ASYMM_SIGNED:
754 if (context->getInputType(kFilterTensor) ==
755 OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
756 return convQuant8PerChannel(
757 context->getInputBuffer<int8_t>(kInputTensor),
758 context->getInputShape(kInputTensor),
759 context->getInputBuffer<int8_t>(kFilterTensor),
760 context->getInputShape(kFilterTensor),
761 context->getInputExtraParams(kFilterTensor).channelQuant().scales.data(),
762 context->getInputBuffer<int32_t>(kBiasTensor),
763 context->getInputShape(kBiasTensor), param.padding_left,
764 param.padding_right, param.padding_top, param.padding_bottom,
765 param.stride_width, param.stride_height, param.dilation_width_factor,
766 param.dilation_height_factor, param.activation, param.useNchw,
767 context->getOutputBuffer<int8_t>(kOutputTensor),
768 context->getOutputShape(kOutputTensor));
769 } else if (context->getInputType(kFilterTensor) ==
770 OperandType::TENSOR_QUANT8_ASYMM_SIGNED) {
771 return conv(context->getInputBuffer<int8_t>(kInputTensor),
772 context->getInputShape(kInputTensor),
773 context->getInputBuffer<int8_t>(kFilterTensor),
774 context->getInputShape(kFilterTensor),
775 context->getInputBuffer<int32_t>(kBiasTensor),
776 context->getInputShape(kBiasTensor), param.padding_left,
777 param.padding_right, param.padding_top, param.padding_bottom,
778 param.stride_width, param.stride_height, param.dilation_width_factor,
779 param.dilation_height_factor, param.activation, param.useNchw,
780 context->getOutputBuffer<int8_t>(kOutputTensor),
781 context->getOutputShape(kOutputTensor));
782 } else {
783 NN_RET_CHECK_FAIL() << "Unsupported filter type for operation " << kOperationName;
784 }
785 default:
786 NN_RET_CHECK_FAIL() << "Unsupported tensor type for operation " << kOperationName;
787 }
788 }
789
790 } // namespace conv_2d
791
792 NN_REGISTER_OPERATION(CONV_2D, conv_2d::kOperationName, conv_2d::validate, conv_2d::prepare,
793 conv_2d::execute, .allowZeroSizedInput = true);
794
795 } // namespace nn
796 } // namespace android
797