1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "code_generator_arm_vixl.h"
18 #include "mirror/array-inl.h"
19 
20 namespace vixl32 = vixl::aarch32;
21 using namespace vixl32;  // NOLINT(build/namespaces)
22 
23 namespace art {
24 namespace arm {
25 
26 using helpers::DRegisterFrom;
27 using helpers::Int64ConstantFrom;
28 using helpers::InputDRegisterAt;
29 using helpers::InputRegisterAt;
30 using helpers::OutputDRegister;
31 using helpers::OutputRegister;
32 using helpers::RegisterFrom;
33 
34 #define __ GetVIXLAssembler()->
35 
VisitVecReplicateScalar(HVecReplicateScalar * instruction)36 void LocationsBuilderARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
37   LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
38   switch (instruction->GetPackedType()) {
39     case DataType::Type::kBool:
40     case DataType::Type::kUint8:
41     case DataType::Type::kInt8:
42     case DataType::Type::kUint16:
43     case DataType::Type::kInt16:
44     case DataType::Type::kInt32:
45       locations->SetInAt(0, Location::RequiresRegister());
46       locations->SetOut(Location::RequiresFpuRegister());
47       break;
48     default:
49       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
50       UNREACHABLE();
51   }
52 }
53 
VisitVecReplicateScalar(HVecReplicateScalar * instruction)54 void InstructionCodeGeneratorARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
55   LocationSummary* locations = instruction->GetLocations();
56   vixl32::DRegister dst = DRegisterFrom(locations->Out());
57   switch (instruction->GetPackedType()) {
58     case DataType::Type::kBool:
59     case DataType::Type::kUint8:
60     case DataType::Type::kInt8:
61       DCHECK_EQ(8u, instruction->GetVectorLength());
62       __ Vdup(Untyped8, dst, InputRegisterAt(instruction, 0));
63       break;
64     case DataType::Type::kUint16:
65     case DataType::Type::kInt16:
66       DCHECK_EQ(4u, instruction->GetVectorLength());
67       __ Vdup(Untyped16, dst, InputRegisterAt(instruction, 0));
68       break;
69     case DataType::Type::kInt32:
70       DCHECK_EQ(2u, instruction->GetVectorLength());
71       __ Vdup(Untyped32, dst, InputRegisterAt(instruction, 0));
72       break;
73     default:
74       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
75       UNREACHABLE();
76   }
77 }
78 
VisitVecExtractScalar(HVecExtractScalar * instruction)79 void LocationsBuilderARMVIXL::VisitVecExtractScalar(HVecExtractScalar* instruction) {
80   LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
81   switch (instruction->GetPackedType()) {
82     case DataType::Type::kInt32:
83       locations->SetInAt(0, Location::RequiresFpuRegister());
84       locations->SetOut(Location::RequiresRegister());
85       break;
86     default:
87       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
88       UNREACHABLE();
89   }
90 }
91 
VisitVecExtractScalar(HVecExtractScalar * instruction)92 void InstructionCodeGeneratorARMVIXL::VisitVecExtractScalar(HVecExtractScalar* instruction) {
93   LocationSummary* locations = instruction->GetLocations();
94   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
95   switch (instruction->GetPackedType()) {
96     case DataType::Type::kInt32:
97       DCHECK_EQ(2u, instruction->GetVectorLength());
98       __ Vmov(OutputRegister(instruction), DRegisterLane(src, 0));
99       break;
100     default:
101       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
102       UNREACHABLE();
103   }
104 }
105 
106 // Helper to set up locations for vector unary operations.
CreateVecUnOpLocations(ArenaAllocator * allocator,HVecUnaryOperation * instruction)107 static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation* instruction) {
108   LocationSummary* locations = new (allocator) LocationSummary(instruction);
109   switch (instruction->GetPackedType()) {
110     case DataType::Type::kBool:
111       locations->SetInAt(0, Location::RequiresFpuRegister());
112       locations->SetOut(Location::RequiresFpuRegister(),
113                         instruction->IsVecNot() ? Location::kOutputOverlap
114                                                 : Location::kNoOutputOverlap);
115       break;
116     case DataType::Type::kUint8:
117     case DataType::Type::kInt8:
118     case DataType::Type::kUint16:
119     case DataType::Type::kInt16:
120     case DataType::Type::kInt32:
121       locations->SetInAt(0, Location::RequiresFpuRegister());
122       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
123       break;
124     default:
125       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
126       UNREACHABLE();
127   }
128 }
129 
VisitVecReduce(HVecReduce * instruction)130 void LocationsBuilderARMVIXL::VisitVecReduce(HVecReduce* instruction) {
131   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
132 }
133 
VisitVecReduce(HVecReduce * instruction)134 void InstructionCodeGeneratorARMVIXL::VisitVecReduce(HVecReduce* instruction) {
135   LocationSummary* locations = instruction->GetLocations();
136   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
137   vixl32::DRegister dst = DRegisterFrom(locations->Out());
138   switch (instruction->GetPackedType()) {
139     case DataType::Type::kInt32:
140       DCHECK_EQ(2u, instruction->GetVectorLength());
141       switch (instruction->GetReductionKind()) {
142         case HVecReduce::kSum:
143           __ Vpadd(DataTypeValue::I32, dst, src, src);
144           break;
145         case HVecReduce::kMin:
146           __ Vpmin(DataTypeValue::S32, dst, src, src);
147           break;
148         case HVecReduce::kMax:
149           __ Vpmax(DataTypeValue::S32, dst, src, src);
150           break;
151       }
152       break;
153     default:
154       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
155       UNREACHABLE();
156   }
157 }
158 
VisitVecCnv(HVecCnv * instruction)159 void LocationsBuilderARMVIXL::VisitVecCnv(HVecCnv* instruction) {
160   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
161 }
162 
VisitVecCnv(HVecCnv * instruction)163 void InstructionCodeGeneratorARMVIXL::VisitVecCnv(HVecCnv* instruction) {
164   LOG(FATAL) << "No SIMD for " << instruction->GetId();
165 }
166 
VisitVecNeg(HVecNeg * instruction)167 void LocationsBuilderARMVIXL::VisitVecNeg(HVecNeg* instruction) {
168   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
169 }
170 
VisitVecNeg(HVecNeg * instruction)171 void InstructionCodeGeneratorARMVIXL::VisitVecNeg(HVecNeg* instruction) {
172   LocationSummary* locations = instruction->GetLocations();
173   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
174   vixl32::DRegister dst = DRegisterFrom(locations->Out());
175   switch (instruction->GetPackedType()) {
176     case DataType::Type::kUint8:
177     case DataType::Type::kInt8:
178       DCHECK_EQ(8u, instruction->GetVectorLength());
179       __ Vneg(DataTypeValue::S8, dst, src);
180       break;
181     case DataType::Type::kUint16:
182     case DataType::Type::kInt16:
183       DCHECK_EQ(4u, instruction->GetVectorLength());
184       __ Vneg(DataTypeValue::S16, dst, src);
185       break;
186     case DataType::Type::kInt32:
187       DCHECK_EQ(2u, instruction->GetVectorLength());
188       __ Vneg(DataTypeValue::S32, dst, src);
189       break;
190     default:
191       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
192       UNREACHABLE();
193   }
194 }
195 
VisitVecAbs(HVecAbs * instruction)196 void LocationsBuilderARMVIXL::VisitVecAbs(HVecAbs* instruction) {
197   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
198 }
199 
VisitVecAbs(HVecAbs * instruction)200 void InstructionCodeGeneratorARMVIXL::VisitVecAbs(HVecAbs* instruction) {
201   LocationSummary* locations = instruction->GetLocations();
202   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
203   vixl32::DRegister dst = DRegisterFrom(locations->Out());
204   switch (instruction->GetPackedType()) {
205     case DataType::Type::kInt8:
206       DCHECK_EQ(8u, instruction->GetVectorLength());
207       __ Vabs(DataTypeValue::S8, dst, src);
208       break;
209     case DataType::Type::kInt16:
210       DCHECK_EQ(4u, instruction->GetVectorLength());
211       __ Vabs(DataTypeValue::S16, dst, src);
212       break;
213     case DataType::Type::kInt32:
214       DCHECK_EQ(2u, instruction->GetVectorLength());
215       __ Vabs(DataTypeValue::S32, dst, src);
216       break;
217     default:
218       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
219       UNREACHABLE();
220   }
221 }
222 
VisitVecNot(HVecNot * instruction)223 void LocationsBuilderARMVIXL::VisitVecNot(HVecNot* instruction) {
224   CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
225 }
226 
VisitVecNot(HVecNot * instruction)227 void InstructionCodeGeneratorARMVIXL::VisitVecNot(HVecNot* instruction) {
228   LocationSummary* locations = instruction->GetLocations();
229   vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
230   vixl32::DRegister dst = DRegisterFrom(locations->Out());
231   switch (instruction->GetPackedType()) {
232     case DataType::Type::kBool:  // special case boolean-not
233       DCHECK_EQ(8u, instruction->GetVectorLength());
234       __ Vmov(I8, dst, 1);
235       __ Veor(dst, dst, src);
236       break;
237     case DataType::Type::kUint8:
238     case DataType::Type::kInt8:
239     case DataType::Type::kUint16:
240     case DataType::Type::kInt16:
241     case DataType::Type::kInt32:
242       __ Vmvn(I8, dst, src);  // lanes do not matter
243       break;
244     default:
245       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
246       UNREACHABLE();
247   }
248 }
249 
250 // Helper to set up locations for vector binary operations.
CreateVecBinOpLocations(ArenaAllocator * allocator,HVecBinaryOperation * instruction)251 static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) {
252   LocationSummary* locations = new (allocator) LocationSummary(instruction);
253   switch (instruction->GetPackedType()) {
254     case DataType::Type::kBool:
255     case DataType::Type::kUint8:
256     case DataType::Type::kInt8:
257     case DataType::Type::kUint16:
258     case DataType::Type::kInt16:
259     case DataType::Type::kInt32:
260       locations->SetInAt(0, Location::RequiresFpuRegister());
261       locations->SetInAt(1, Location::RequiresFpuRegister());
262       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
263       break;
264     default:
265       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
266       UNREACHABLE();
267   }
268 }
269 
VisitVecAdd(HVecAdd * instruction)270 void LocationsBuilderARMVIXL::VisitVecAdd(HVecAdd* instruction) {
271   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
272 }
273 
VisitVecAdd(HVecAdd * instruction)274 void InstructionCodeGeneratorARMVIXL::VisitVecAdd(HVecAdd* instruction) {
275   LocationSummary* locations = instruction->GetLocations();
276   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
277   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
278   vixl32::DRegister dst = DRegisterFrom(locations->Out());
279   switch (instruction->GetPackedType()) {
280     case DataType::Type::kUint8:
281     case DataType::Type::kInt8:
282       DCHECK_EQ(8u, instruction->GetVectorLength());
283       __ Vadd(I8, dst, lhs, rhs);
284       break;
285     case DataType::Type::kUint16:
286     case DataType::Type::kInt16:
287       DCHECK_EQ(4u, instruction->GetVectorLength());
288       __ Vadd(I16, dst, lhs, rhs);
289       break;
290     case DataType::Type::kInt32:
291       DCHECK_EQ(2u, instruction->GetVectorLength());
292       __ Vadd(I32, dst, lhs, rhs);
293       break;
294     default:
295       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
296       UNREACHABLE();
297   }
298 }
299 
VisitVecSaturationAdd(HVecSaturationAdd * instruction)300 void LocationsBuilderARMVIXL::VisitVecSaturationAdd(HVecSaturationAdd* instruction) {
301   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
302 }
303 
VisitVecSaturationAdd(HVecSaturationAdd * instruction)304 void InstructionCodeGeneratorARMVIXL::VisitVecSaturationAdd(HVecSaturationAdd* instruction) {
305   LocationSummary* locations = instruction->GetLocations();
306   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
307   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
308   vixl32::DRegister dst = DRegisterFrom(locations->Out());
309   switch (instruction->GetPackedType()) {
310     case DataType::Type::kUint8:
311       DCHECK_EQ(8u, instruction->GetVectorLength());
312       __ Vqadd(DataTypeValue::U8, dst, lhs, rhs);
313       break;
314     case DataType::Type::kInt8:
315       DCHECK_EQ(8u, instruction->GetVectorLength());
316       __ Vqadd(DataTypeValue::S8, dst, lhs, rhs);
317       break;
318     case DataType::Type::kUint16:
319       DCHECK_EQ(4u, instruction->GetVectorLength());
320       __ Vqadd(DataTypeValue::U16, dst, lhs, rhs);
321       break;
322     case DataType::Type::kInt16:
323       DCHECK_EQ(4u, instruction->GetVectorLength());
324       __ Vqadd(DataTypeValue::S16, dst, lhs, rhs);
325       break;
326     default:
327       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
328       UNREACHABLE();
329   }
330 }
331 
VisitVecHalvingAdd(HVecHalvingAdd * instruction)332 void LocationsBuilderARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
333   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
334 }
335 
VisitVecHalvingAdd(HVecHalvingAdd * instruction)336 void InstructionCodeGeneratorARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
337   LocationSummary* locations = instruction->GetLocations();
338   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
339   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
340   vixl32::DRegister dst = DRegisterFrom(locations->Out());
341   switch (instruction->GetPackedType()) {
342     case DataType::Type::kUint8:
343       DCHECK_EQ(8u, instruction->GetVectorLength());
344       instruction->IsRounded()
345           ? __ Vrhadd(DataTypeValue::U8, dst, lhs, rhs)
346           : __ Vhadd(DataTypeValue::U8, dst, lhs, rhs);
347       break;
348     case DataType::Type::kInt8:
349       DCHECK_EQ(8u, instruction->GetVectorLength());
350       instruction->IsRounded()
351           ? __ Vrhadd(DataTypeValue::S8, dst, lhs, rhs)
352           : __ Vhadd(DataTypeValue::S8, dst, lhs, rhs);
353       break;
354     case DataType::Type::kUint16:
355       DCHECK_EQ(4u, instruction->GetVectorLength());
356       instruction->IsRounded()
357           ? __ Vrhadd(DataTypeValue::U16, dst, lhs, rhs)
358           : __ Vhadd(DataTypeValue::U16, dst, lhs, rhs);
359       break;
360     case DataType::Type::kInt16:
361       DCHECK_EQ(4u, instruction->GetVectorLength());
362       instruction->IsRounded()
363           ? __ Vrhadd(DataTypeValue::S16, dst, lhs, rhs)
364           : __ Vhadd(DataTypeValue::S16, dst, lhs, rhs);
365       break;
366     default:
367       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
368       UNREACHABLE();
369   }
370 }
371 
VisitVecSub(HVecSub * instruction)372 void LocationsBuilderARMVIXL::VisitVecSub(HVecSub* instruction) {
373   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
374 }
375 
VisitVecSub(HVecSub * instruction)376 void InstructionCodeGeneratorARMVIXL::VisitVecSub(HVecSub* instruction) {
377   LocationSummary* locations = instruction->GetLocations();
378   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
379   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
380   vixl32::DRegister dst = DRegisterFrom(locations->Out());
381   switch (instruction->GetPackedType()) {
382     case DataType::Type::kUint8:
383     case DataType::Type::kInt8:
384       DCHECK_EQ(8u, instruction->GetVectorLength());
385       __ Vsub(I8, dst, lhs, rhs);
386       break;
387     case DataType::Type::kUint16:
388     case DataType::Type::kInt16:
389       DCHECK_EQ(4u, instruction->GetVectorLength());
390       __ Vsub(I16, dst, lhs, rhs);
391       break;
392     case DataType::Type::kInt32:
393       DCHECK_EQ(2u, instruction->GetVectorLength());
394       __ Vsub(I32, dst, lhs, rhs);
395       break;
396     default:
397       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
398       UNREACHABLE();
399   }
400 }
401 
VisitVecSaturationSub(HVecSaturationSub * instruction)402 void LocationsBuilderARMVIXL::VisitVecSaturationSub(HVecSaturationSub* instruction) {
403   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
404 }
405 
VisitVecSaturationSub(HVecSaturationSub * instruction)406 void InstructionCodeGeneratorARMVIXL::VisitVecSaturationSub(HVecSaturationSub* instruction) {
407   LocationSummary* locations = instruction->GetLocations();
408   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
409   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
410   vixl32::DRegister dst = DRegisterFrom(locations->Out());
411   switch (instruction->GetPackedType()) {
412     case DataType::Type::kUint8:
413       DCHECK_EQ(8u, instruction->GetVectorLength());
414       __ Vqsub(DataTypeValue::U8, dst, lhs, rhs);
415       break;
416     case DataType::Type::kInt8:
417       DCHECK_EQ(8u, instruction->GetVectorLength());
418       __ Vqsub(DataTypeValue::S8, dst, lhs, rhs);
419       break;
420     case DataType::Type::kUint16:
421       DCHECK_EQ(4u, instruction->GetVectorLength());
422       __ Vqsub(DataTypeValue::U16, dst, lhs, rhs);
423       break;
424     case DataType::Type::kInt16:
425       DCHECK_EQ(4u, instruction->GetVectorLength());
426       __ Vqsub(DataTypeValue::S16, dst, lhs, rhs);
427       break;
428     default:
429       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
430       UNREACHABLE();
431   }
432 }
433 
VisitVecMul(HVecMul * instruction)434 void LocationsBuilderARMVIXL::VisitVecMul(HVecMul* instruction) {
435   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
436 }
437 
VisitVecMul(HVecMul * instruction)438 void InstructionCodeGeneratorARMVIXL::VisitVecMul(HVecMul* instruction) {
439   LocationSummary* locations = instruction->GetLocations();
440   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
441   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
442   vixl32::DRegister dst = DRegisterFrom(locations->Out());
443   switch (instruction->GetPackedType()) {
444     case DataType::Type::kUint8:
445     case DataType::Type::kInt8:
446       DCHECK_EQ(8u, instruction->GetVectorLength());
447       __ Vmul(I8, dst, lhs, rhs);
448       break;
449     case DataType::Type::kUint16:
450     case DataType::Type::kInt16:
451       DCHECK_EQ(4u, instruction->GetVectorLength());
452       __ Vmul(I16, dst, lhs, rhs);
453       break;
454     case DataType::Type::kInt32:
455       DCHECK_EQ(2u, instruction->GetVectorLength());
456       __ Vmul(I32, dst, lhs, rhs);
457       break;
458     default:
459       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
460       UNREACHABLE();
461   }
462 }
463 
VisitVecDiv(HVecDiv * instruction)464 void LocationsBuilderARMVIXL::VisitVecDiv(HVecDiv* instruction) {
465   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
466 }
467 
VisitVecDiv(HVecDiv * instruction)468 void InstructionCodeGeneratorARMVIXL::VisitVecDiv(HVecDiv* instruction) {
469   LOG(FATAL) << "No SIMD for " << instruction->GetId();
470 }
471 
VisitVecMin(HVecMin * instruction)472 void LocationsBuilderARMVIXL::VisitVecMin(HVecMin* instruction) {
473   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
474 }
475 
VisitVecMin(HVecMin * instruction)476 void InstructionCodeGeneratorARMVIXL::VisitVecMin(HVecMin* instruction) {
477   LocationSummary* locations = instruction->GetLocations();
478   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
479   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
480   vixl32::DRegister dst = DRegisterFrom(locations->Out());
481   switch (instruction->GetPackedType()) {
482     case DataType::Type::kUint8:
483       DCHECK_EQ(8u, instruction->GetVectorLength());
484       __ Vmin(DataTypeValue::U8, dst, lhs, rhs);
485       break;
486     case DataType::Type::kInt8:
487       DCHECK_EQ(8u, instruction->GetVectorLength());
488       __ Vmin(DataTypeValue::S8, dst, lhs, rhs);
489       break;
490     case DataType::Type::kUint16:
491       DCHECK_EQ(4u, instruction->GetVectorLength());
492       __ Vmin(DataTypeValue::U16, dst, lhs, rhs);
493       break;
494     case DataType::Type::kInt16:
495       DCHECK_EQ(4u, instruction->GetVectorLength());
496       __ Vmin(DataTypeValue::S16, dst, lhs, rhs);
497       break;
498     case DataType::Type::kUint32:
499       DCHECK_EQ(2u, instruction->GetVectorLength());
500       __ Vmin(DataTypeValue::U32, dst, lhs, rhs);
501       break;
502     case DataType::Type::kInt32:
503       DCHECK_EQ(2u, instruction->GetVectorLength());
504       __ Vmin(DataTypeValue::S32, dst, lhs, rhs);
505       break;
506     default:
507       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
508       UNREACHABLE();
509   }
510 }
511 
VisitVecMax(HVecMax * instruction)512 void LocationsBuilderARMVIXL::VisitVecMax(HVecMax* instruction) {
513   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
514 }
515 
VisitVecMax(HVecMax * instruction)516 void InstructionCodeGeneratorARMVIXL::VisitVecMax(HVecMax* instruction) {
517   LocationSummary* locations = instruction->GetLocations();
518   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
519   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
520   vixl32::DRegister dst = DRegisterFrom(locations->Out());
521   switch (instruction->GetPackedType()) {
522     case DataType::Type::kUint8:
523       DCHECK_EQ(8u, instruction->GetVectorLength());
524       __ Vmax(DataTypeValue::U8, dst, lhs, rhs);
525       break;
526     case DataType::Type::kInt8:
527       DCHECK_EQ(8u, instruction->GetVectorLength());
528       __ Vmax(DataTypeValue::S8, dst, lhs, rhs);
529       break;
530     case DataType::Type::kUint16:
531       DCHECK_EQ(4u, instruction->GetVectorLength());
532       __ Vmax(DataTypeValue::U16, dst, lhs, rhs);
533       break;
534     case DataType::Type::kInt16:
535       DCHECK_EQ(4u, instruction->GetVectorLength());
536       __ Vmax(DataTypeValue::S16, dst, lhs, rhs);
537       break;
538     case DataType::Type::kUint32:
539       DCHECK_EQ(2u, instruction->GetVectorLength());
540       __ Vmax(DataTypeValue::U32, dst, lhs, rhs);
541       break;
542     case DataType::Type::kInt32:
543       DCHECK_EQ(2u, instruction->GetVectorLength());
544       __ Vmax(DataTypeValue::S32, dst, lhs, rhs);
545       break;
546     default:
547       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
548       UNREACHABLE();
549   }
550 }
551 
VisitVecAnd(HVecAnd * instruction)552 void LocationsBuilderARMVIXL::VisitVecAnd(HVecAnd* instruction) {
553   // TODO: Allow constants supported by VAND (immediate).
554   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
555 }
556 
VisitVecAnd(HVecAnd * instruction)557 void InstructionCodeGeneratorARMVIXL::VisitVecAnd(HVecAnd* instruction) {
558   LocationSummary* locations = instruction->GetLocations();
559   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
560   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
561   vixl32::DRegister dst = DRegisterFrom(locations->Out());
562   switch (instruction->GetPackedType()) {
563     case DataType::Type::kBool:
564     case DataType::Type::kUint8:
565     case DataType::Type::kInt8:
566     case DataType::Type::kUint16:
567     case DataType::Type::kInt16:
568     case DataType::Type::kInt32:
569       __ Vand(I8, dst, lhs, rhs);
570       break;
571     default:
572       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
573       UNREACHABLE();
574   }
575 }
576 
VisitVecAndNot(HVecAndNot * instruction)577 void LocationsBuilderARMVIXL::VisitVecAndNot(HVecAndNot* instruction) {
578   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
579 }
580 
VisitVecAndNot(HVecAndNot * instruction)581 void InstructionCodeGeneratorARMVIXL::VisitVecAndNot(HVecAndNot* instruction) {
582   LOG(FATAL) << "No SIMD for " << instruction->GetId();
583 }
584 
VisitVecOr(HVecOr * instruction)585 void LocationsBuilderARMVIXL::VisitVecOr(HVecOr* instruction) {
586   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
587 }
588 
VisitVecOr(HVecOr * instruction)589 void InstructionCodeGeneratorARMVIXL::VisitVecOr(HVecOr* instruction) {
590   LocationSummary* locations = instruction->GetLocations();
591   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
592   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
593   vixl32::DRegister dst = DRegisterFrom(locations->Out());
594   switch (instruction->GetPackedType()) {
595     case DataType::Type::kBool:
596     case DataType::Type::kUint8:
597     case DataType::Type::kInt8:
598     case DataType::Type::kUint16:
599     case DataType::Type::kInt16:
600     case DataType::Type::kInt32:
601       __ Vorr(I8, dst, lhs, rhs);
602       break;
603     default:
604       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
605       UNREACHABLE();
606   }
607 }
608 
VisitVecXor(HVecXor * instruction)609 void LocationsBuilderARMVIXL::VisitVecXor(HVecXor* instruction) {
610   CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
611 }
612 
VisitVecXor(HVecXor * instruction)613 void InstructionCodeGeneratorARMVIXL::VisitVecXor(HVecXor* instruction) {
614   LocationSummary* locations = instruction->GetLocations();
615   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
616   vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
617   vixl32::DRegister dst = DRegisterFrom(locations->Out());
618   switch (instruction->GetPackedType()) {
619     case DataType::Type::kBool:
620     case DataType::Type::kUint8:
621     case DataType::Type::kInt8:
622     case DataType::Type::kUint16:
623     case DataType::Type::kInt16:
624     case DataType::Type::kInt32:
625       __ Veor(I8, dst, lhs, rhs);
626       break;
627     default:
628       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
629       UNREACHABLE();
630   }
631 }
632 
633 // Helper to set up locations for vector shift operations.
CreateVecShiftLocations(ArenaAllocator * allocator,HVecBinaryOperation * instruction)634 static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) {
635   LocationSummary* locations = new (allocator) LocationSummary(instruction);
636   switch (instruction->GetPackedType()) {
637     case DataType::Type::kUint8:
638     case DataType::Type::kInt8:
639     case DataType::Type::kUint16:
640     case DataType::Type::kInt16:
641     case DataType::Type::kInt32:
642       locations->SetInAt(0, Location::RequiresFpuRegister());
643       locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant()));
644       locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
645       break;
646     default:
647       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
648       UNREACHABLE();
649   }
650 }
651 
VisitVecShl(HVecShl * instruction)652 void LocationsBuilderARMVIXL::VisitVecShl(HVecShl* instruction) {
653   CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
654 }
655 
VisitVecShl(HVecShl * instruction)656 void InstructionCodeGeneratorARMVIXL::VisitVecShl(HVecShl* instruction) {
657   LocationSummary* locations = instruction->GetLocations();
658   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
659   vixl32::DRegister dst = DRegisterFrom(locations->Out());
660   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
661   switch (instruction->GetPackedType()) {
662     case DataType::Type::kUint8:
663     case DataType::Type::kInt8:
664       DCHECK_EQ(8u, instruction->GetVectorLength());
665       __ Vshl(I8, dst, lhs, value);
666       break;
667     case DataType::Type::kUint16:
668     case DataType::Type::kInt16:
669       DCHECK_EQ(4u, instruction->GetVectorLength());
670       __ Vshl(I16, dst, lhs, value);
671       break;
672     case DataType::Type::kInt32:
673       DCHECK_EQ(2u, instruction->GetVectorLength());
674       __ Vshl(I32, dst, lhs, value);
675       break;
676     default:
677       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
678       UNREACHABLE();
679   }
680 }
681 
VisitVecShr(HVecShr * instruction)682 void LocationsBuilderARMVIXL::VisitVecShr(HVecShr* instruction) {
683   CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
684 }
685 
VisitVecShr(HVecShr * instruction)686 void InstructionCodeGeneratorARMVIXL::VisitVecShr(HVecShr* instruction) {
687   LocationSummary* locations = instruction->GetLocations();
688   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
689   vixl32::DRegister dst = DRegisterFrom(locations->Out());
690   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
691   switch (instruction->GetPackedType()) {
692     case DataType::Type::kUint8:
693     case DataType::Type::kInt8:
694       DCHECK_EQ(8u, instruction->GetVectorLength());
695       __ Vshr(DataTypeValue::S8, dst, lhs, value);
696       break;
697     case DataType::Type::kUint16:
698     case DataType::Type::kInt16:
699       DCHECK_EQ(4u, instruction->GetVectorLength());
700       __ Vshr(DataTypeValue::S16, dst, lhs, value);
701       break;
702     case DataType::Type::kInt32:
703       DCHECK_EQ(2u, instruction->GetVectorLength());
704       __ Vshr(DataTypeValue::S32, dst, lhs, value);
705       break;
706     default:
707       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
708       UNREACHABLE();
709   }
710 }
711 
VisitVecUShr(HVecUShr * instruction)712 void LocationsBuilderARMVIXL::VisitVecUShr(HVecUShr* instruction) {
713   CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
714 }
715 
VisitVecUShr(HVecUShr * instruction)716 void InstructionCodeGeneratorARMVIXL::VisitVecUShr(HVecUShr* instruction) {
717   LocationSummary* locations = instruction->GetLocations();
718   vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
719   vixl32::DRegister dst = DRegisterFrom(locations->Out());
720   int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
721   switch (instruction->GetPackedType()) {
722     case DataType::Type::kUint8:
723     case DataType::Type::kInt8:
724       DCHECK_EQ(8u, instruction->GetVectorLength());
725       __ Vshr(DataTypeValue::U8, dst, lhs, value);
726       break;
727     case DataType::Type::kUint16:
728     case DataType::Type::kInt16:
729       DCHECK_EQ(4u, instruction->GetVectorLength());
730       __ Vshr(DataTypeValue::U16, dst, lhs, value);
731       break;
732     case DataType::Type::kInt32:
733       DCHECK_EQ(2u, instruction->GetVectorLength());
734       __ Vshr(DataTypeValue::U32, dst, lhs, value);
735       break;
736     default:
737       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
738       UNREACHABLE();
739   }
740 }
741 
VisitVecSetScalars(HVecSetScalars * instruction)742 void LocationsBuilderARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) {
743   LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
744 
745   DCHECK_EQ(1u, instruction->InputCount());  // only one input currently implemented
746 
747   HInstruction* input = instruction->InputAt(0);
748   bool is_zero = IsZeroBitPattern(input);
749 
750   switch (instruction->GetPackedType()) {
751     case DataType::Type::kInt32:
752       locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant())
753                                     : Location::RequiresRegister());
754       locations->SetOut(Location::RequiresFpuRegister());
755       break;
756     default:
757       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
758       UNREACHABLE();
759   }
760 }
761 
VisitVecSetScalars(HVecSetScalars * instruction)762 void InstructionCodeGeneratorARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) {
763   LocationSummary* locations = instruction->GetLocations();
764   vixl32::DRegister dst = DRegisterFrom(locations->Out());
765 
766   DCHECK_EQ(1u, instruction->InputCount());  // only one input currently implemented
767 
768   // Zero out all other elements first.
769   __ Vmov(I32, dst, 0);
770 
771   // Shorthand for any type of zero.
772   if (IsZeroBitPattern(instruction->InputAt(0))) {
773     return;
774   }
775 
776   // Set required elements.
777   switch (instruction->GetPackedType()) {
778     case DataType::Type::kInt32:
779       DCHECK_EQ(2u, instruction->GetVectorLength());
780       __ Vmov(Untyped32, DRegisterLane(dst, 0), InputRegisterAt(instruction, 0));
781       break;
782     default:
783       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
784       UNREACHABLE();
785   }
786 }
787 
788 // Helper to set up locations for vector accumulations.
CreateVecAccumLocations(ArenaAllocator * allocator,HVecOperation * instruction)789 static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* instruction) {
790   LocationSummary* locations = new (allocator) LocationSummary(instruction);
791   switch (instruction->GetPackedType()) {
792     case DataType::Type::kUint8:
793     case DataType::Type::kInt8:
794     case DataType::Type::kUint16:
795     case DataType::Type::kInt16:
796     case DataType::Type::kInt32:
797     case DataType::Type::kInt64:
798       locations->SetInAt(0, Location::RequiresFpuRegister());
799       locations->SetInAt(1, Location::RequiresFpuRegister());
800       locations->SetInAt(2, Location::RequiresFpuRegister());
801       locations->SetOut(Location::SameAsFirstInput());
802       break;
803     default:
804       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
805       UNREACHABLE();
806   }
807 }
808 
VisitVecMultiplyAccumulate(HVecMultiplyAccumulate * instruction)809 void LocationsBuilderARMVIXL::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) {
810   CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction);
811 }
812 
VisitVecMultiplyAccumulate(HVecMultiplyAccumulate * instruction)813 void InstructionCodeGeneratorARMVIXL::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) {
814   LOG(FATAL) << "No SIMD for " << instruction->GetId();
815 }
816 
VisitVecSADAccumulate(HVecSADAccumulate * instruction)817 void LocationsBuilderARMVIXL::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
818   CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction);
819 }
820 
VisitVecSADAccumulate(HVecSADAccumulate * instruction)821 void InstructionCodeGeneratorARMVIXL::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
822   LocationSummary* locations = instruction->GetLocations();
823   vixl32::DRegister acc = DRegisterFrom(locations->InAt(0));
824   vixl32::DRegister left = DRegisterFrom(locations->InAt(1));
825   vixl32::DRegister right = DRegisterFrom(locations->InAt(2));
826 
827   DCHECK(locations->InAt(0).Equals(locations->Out()));
828 
829   // Handle all feasible acc_T += sad(a_S, b_S) type combinations (T x S).
830   HVecOperation* a = instruction->InputAt(1)->AsVecOperation();
831   HVecOperation* b = instruction->InputAt(2)->AsVecOperation();
832   DCHECK_EQ(a->GetPackedType(), b->GetPackedType());
833   switch (a->GetPackedType()) {
834     case DataType::Type::kInt32:
835       DCHECK_EQ(2u, a->GetVectorLength());
836       switch (instruction->GetPackedType()) {
837         case DataType::Type::kInt32: {
838           DCHECK_EQ(2u, instruction->GetVectorLength());
839           UseScratchRegisterScope temps(GetVIXLAssembler());
840           vixl32::DRegister tmp = temps.AcquireD();
841           __ Vsub(DataTypeValue::I32, tmp, left, right);
842           __ Vabs(DataTypeValue::S32, tmp, tmp);
843           __ Vadd(DataTypeValue::I32, acc, acc, tmp);
844           break;
845         }
846         default:
847           LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
848           UNREACHABLE();
849       }
850       break;
851     default:
852       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
853       UNREACHABLE();
854   }
855 }
856 
VisitVecDotProd(HVecDotProd * instruction)857 void LocationsBuilderARMVIXL::VisitVecDotProd(HVecDotProd* instruction) {
858   LOG(FATAL) << "No SIMD for " << instruction->GetId();
859 }
860 
VisitVecDotProd(HVecDotProd * instruction)861 void InstructionCodeGeneratorARMVIXL::VisitVecDotProd(HVecDotProd* instruction) {
862   LOG(FATAL) << "No SIMD for " << instruction->GetId();
863 }
864 
865 // Return whether the vector memory access operation is guaranteed to be word-aligned (ARM word
866 // size equals to 4).
IsWordAligned(HVecMemoryOperation * instruction)867 static bool IsWordAligned(HVecMemoryOperation* instruction) {
868   return instruction->GetAlignment().IsAlignedAt(4u);
869 }
870 
871 // Helper to set up locations for vector memory operations.
CreateVecMemLocations(ArenaAllocator * allocator,HVecMemoryOperation * instruction,bool is_load)872 static void CreateVecMemLocations(ArenaAllocator* allocator,
873                                   HVecMemoryOperation* instruction,
874                                   bool is_load) {
875   LocationSummary* locations = new (allocator) LocationSummary(instruction);
876   switch (instruction->GetPackedType()) {
877     case DataType::Type::kBool:
878     case DataType::Type::kUint8:
879     case DataType::Type::kInt8:
880     case DataType::Type::kUint16:
881     case DataType::Type::kInt16:
882     case DataType::Type::kInt32:
883       locations->SetInAt(0, Location::RequiresRegister());
884       locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
885       if (is_load) {
886         locations->SetOut(Location::RequiresFpuRegister());
887       } else {
888         locations->SetInAt(2, Location::RequiresFpuRegister());
889       }
890       break;
891     default:
892       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
893       UNREACHABLE();
894   }
895 }
896 
897 // Helper to set up locations for vector memory operations. Returns the memory operand and,
898 // if used, sets the output parameter scratch to a temporary register used in this operand,
899 // so that the client can release it right after the memory operand use.
VecAddress(HVecMemoryOperation * instruction,UseScratchRegisterScope * temps_scope,vixl32::Register * scratch)900 MemOperand InstructionCodeGeneratorARMVIXL::VecAddress(
901         HVecMemoryOperation* instruction,
902         UseScratchRegisterScope* temps_scope,
903         /*out*/ vixl32::Register* scratch) {
904   LocationSummary* locations = instruction->GetLocations();
905   vixl32::Register base = InputRegisterAt(instruction, 0);
906 
907   Location index = locations->InAt(1);
908   size_t size = DataType::Size(instruction->GetPackedType());
909   uint32_t offset = mirror::Array::DataOffset(size).Uint32Value();
910   size_t shift = ComponentSizeShiftWidth(size);
911 
912   // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet.
913   DCHECK(!instruction->InputAt(0)->IsIntermediateAddress());
914 
915   if (index.IsConstant()) {
916     offset += Int64ConstantFrom(index) << shift;
917     return MemOperand(base, offset);
918   } else {
919     *scratch = temps_scope->Acquire();
920     __ Add(*scratch, base, Operand(RegisterFrom(index), ShiftType::LSL, shift));
921 
922     return MemOperand(*scratch, offset);
923   }
924 }
925 
VecAddressUnaligned(HVecMemoryOperation * instruction,UseScratchRegisterScope * temps_scope,vixl32::Register * scratch)926 AlignedMemOperand InstructionCodeGeneratorARMVIXL::VecAddressUnaligned(
927         HVecMemoryOperation* instruction,
928         UseScratchRegisterScope* temps_scope,
929         /*out*/ vixl32::Register* scratch) {
930   LocationSummary* locations = instruction->GetLocations();
931   vixl32::Register base = InputRegisterAt(instruction, 0);
932 
933   Location index = locations->InAt(1);
934   size_t size = DataType::Size(instruction->GetPackedType());
935   uint32_t offset = mirror::Array::DataOffset(size).Uint32Value();
936   size_t shift = ComponentSizeShiftWidth(size);
937 
938   // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet.
939   DCHECK(!instruction->InputAt(0)->IsIntermediateAddress());
940 
941   if (index.IsConstant()) {
942     offset += Int64ConstantFrom(index) << shift;
943     __ Add(*scratch, base, offset);
944   } else {
945     *scratch = temps_scope->Acquire();
946     __ Add(*scratch, base, offset);
947     __ Add(*scratch, *scratch, Operand(RegisterFrom(index), ShiftType::LSL, shift));
948   }
949   return AlignedMemOperand(*scratch, kNoAlignment);
950 }
951 
VisitVecLoad(HVecLoad * instruction)952 void LocationsBuilderARMVIXL::VisitVecLoad(HVecLoad* instruction) {
953   CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ true);
954 }
955 
VisitVecLoad(HVecLoad * instruction)956 void InstructionCodeGeneratorARMVIXL::VisitVecLoad(HVecLoad* instruction) {
957   vixl32::DRegister reg = OutputDRegister(instruction);
958   UseScratchRegisterScope temps(GetVIXLAssembler());
959   vixl32::Register scratch;
960 
961   DCHECK(instruction->GetPackedType() != DataType::Type::kUint16 || !instruction->IsStringCharAt());
962 
963   switch (instruction->GetPackedType()) {
964     case DataType::Type::kBool:
965     case DataType::Type::kUint8:
966     case DataType::Type::kInt8:
967       DCHECK_EQ(8u, instruction->GetVectorLength());
968       if (IsWordAligned(instruction)) {
969         __ Vldr(reg, VecAddress(instruction, &temps, &scratch));
970       } else {
971         __ Vld1(Untyped8,
972             NeonRegisterList(reg, kMultipleLanes),
973             VecAddressUnaligned(instruction, &temps, &scratch));
974       }
975       break;
976     case DataType::Type::kUint16:
977     case DataType::Type::kInt16:
978       DCHECK_EQ(4u, instruction->GetVectorLength());
979       if (IsWordAligned(instruction)) {
980         __ Vldr(reg, VecAddress(instruction, &temps, &scratch));
981       } else {
982         __ Vld1(Untyped16,
983             NeonRegisterList(reg, kMultipleLanes),
984             VecAddressUnaligned(instruction, &temps, &scratch));
985       }
986       break;
987     case DataType::Type::kInt32:
988       DCHECK_EQ(2u, instruction->GetVectorLength());
989       if (IsWordAligned(instruction)) {
990         __ Vldr(reg, VecAddress(instruction, &temps, &scratch));
991       } else {
992         __ Vld1(Untyped32,
993             NeonRegisterList(reg, kMultipleLanes),
994             VecAddressUnaligned(instruction, &temps, &scratch));
995       }
996       break;
997     default:
998       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
999       UNREACHABLE();
1000   }
1001 }
1002 
VisitVecStore(HVecStore * instruction)1003 void LocationsBuilderARMVIXL::VisitVecStore(HVecStore* instruction) {
1004   CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ false);
1005 }
1006 
VisitVecStore(HVecStore * instruction)1007 void InstructionCodeGeneratorARMVIXL::VisitVecStore(HVecStore* instruction) {
1008   vixl32::DRegister reg = InputDRegisterAt(instruction, 2);
1009   UseScratchRegisterScope temps(GetVIXLAssembler());
1010   vixl32::Register scratch;
1011   switch (instruction->GetPackedType()) {
1012     case DataType::Type::kBool:
1013     case DataType::Type::kUint8:
1014     case DataType::Type::kInt8:
1015       DCHECK_EQ(8u, instruction->GetVectorLength());
1016       if (IsWordAligned(instruction)) {
1017         __ Vstr(reg, VecAddress(instruction, &temps, &scratch));
1018       } else {
1019         __ Vst1(Untyped8,
1020                 NeonRegisterList(reg, kMultipleLanes),
1021                 VecAddressUnaligned(instruction, &temps, &scratch));
1022       }
1023       break;
1024     case DataType::Type::kUint16:
1025     case DataType::Type::kInt16:
1026       DCHECK_EQ(4u, instruction->GetVectorLength());
1027       if (IsWordAligned(instruction)) {
1028         __ Vstr(reg, VecAddress(instruction, &temps, &scratch));
1029       } else {
1030         __ Vst1(Untyped16,
1031                 NeonRegisterList(reg, kMultipleLanes),
1032                 VecAddressUnaligned(instruction, &temps, &scratch));
1033       }
1034       break;
1035     case DataType::Type::kInt32:
1036       DCHECK_EQ(2u, instruction->GetVectorLength());
1037       if (IsWordAligned(instruction)) {
1038         __ Vstr(reg, VecAddress(instruction, &temps, &scratch));
1039       } else {
1040         __ Vst1(Untyped32,
1041                 NeonRegisterList(reg, kMultipleLanes),
1042                 VecAddressUnaligned(instruction, &temps, &scratch));
1043       }
1044       break;
1045     default:
1046       LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType();
1047       UNREACHABLE();
1048   }
1049 }
1050 
VisitVecPredSetAll(HVecPredSetAll * instruction)1051 void LocationsBuilderARMVIXL::VisitVecPredSetAll(HVecPredSetAll* instruction) {
1052   LOG(FATAL) << "No SIMD for " << instruction->GetId();
1053   UNREACHABLE();
1054 }
1055 
VisitVecPredSetAll(HVecPredSetAll * instruction)1056 void InstructionCodeGeneratorARMVIXL::VisitVecPredSetAll(HVecPredSetAll* instruction) {
1057   LOG(FATAL) << "No SIMD for " << instruction->GetId();
1058   UNREACHABLE();
1059 }
1060 
VisitVecPredWhile(HVecPredWhile * instruction)1061 void LocationsBuilderARMVIXL::VisitVecPredWhile(HVecPredWhile* instruction) {
1062   LOG(FATAL) << "No SIMD for " << instruction->GetId();
1063   UNREACHABLE();
1064 }
1065 
VisitVecPredWhile(HVecPredWhile * instruction)1066 void InstructionCodeGeneratorARMVIXL::VisitVecPredWhile(HVecPredWhile* instruction) {
1067   LOG(FATAL) << "No SIMD for " << instruction->GetId();
1068   UNREACHABLE();
1069 }
1070 
VisitVecPredCondition(HVecPredCondition * instruction)1071 void LocationsBuilderARMVIXL::VisitVecPredCondition(HVecPredCondition* instruction) {
1072   LOG(FATAL) << "No SIMD for " << instruction->GetId();
1073   UNREACHABLE();
1074 }
1075 
VisitVecPredCondition(HVecPredCondition * instruction)1076 void InstructionCodeGeneratorARMVIXL::VisitVecPredCondition(HVecPredCondition* instruction) {
1077   LOG(FATAL) << "No SIMD for " << instruction->GetId();
1078   UNREACHABLE();
1079 }
1080 
1081 #undef __
1082 
1083 }  // namespace arm
1084 }  // namespace art
1085