1 /*
2 * Copyright (C) 2016 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "jni_macro_assembler_arm64.h"
18
19 #include "entrypoints/quick/quick_entrypoints.h"
20 #include "managed_register_arm64.h"
21 #include "offsets.h"
22 #include "thread.h"
23
24 using namespace vixl::aarch64; // NOLINT(build/namespaces)
25
26 namespace art {
27 namespace arm64 {
28
29 #ifdef ___
30 #error "ARM64 Assembler macro already defined."
31 #else
32 #define ___ asm_.GetVIXLAssembler()->
33 #endif
34
35 #define reg_x(X) Arm64Assembler::reg_x(X)
36 #define reg_w(W) Arm64Assembler::reg_w(W)
37 #define reg_d(D) Arm64Assembler::reg_d(D)
38 #define reg_s(S) Arm64Assembler::reg_s(S)
39
40 // The AAPCS64 requires 16-byte alignement. This is the same as the Managed ABI stack alignment.
41 static constexpr size_t kAapcs64StackAlignment = 16u;
42 static_assert(kAapcs64StackAlignment == kStackAlignment);
43
~Arm64JNIMacroAssembler()44 Arm64JNIMacroAssembler::~Arm64JNIMacroAssembler() {
45 }
46
FinalizeCode()47 void Arm64JNIMacroAssembler::FinalizeCode() {
48 for (const std::unique_ptr<Arm64Exception>& exception : exception_blocks_) {
49 EmitExceptionPoll(exception.get());
50 }
51 ___ FinalizeCode();
52 }
53
GetCurrentThread(ManagedRegister dest)54 void Arm64JNIMacroAssembler::GetCurrentThread(ManagedRegister dest) {
55 ___ Mov(reg_x(dest.AsArm64().AsXRegister()), reg_x(TR));
56 }
57
GetCurrentThread(FrameOffset offset)58 void Arm64JNIMacroAssembler::GetCurrentThread(FrameOffset offset) {
59 StoreToOffset(TR, SP, offset.Int32Value());
60 }
61
62 // See Arm64 PCS Section 5.2.2.1.
IncreaseFrameSize(size_t adjust)63 void Arm64JNIMacroAssembler::IncreaseFrameSize(size_t adjust) {
64 if (adjust != 0u) {
65 CHECK_ALIGNED(adjust, kStackAlignment);
66 AddConstant(SP, -adjust);
67 cfi().AdjustCFAOffset(adjust);
68 }
69 }
70
71 // See Arm64 PCS Section 5.2.2.1.
DecreaseFrameSize(size_t adjust)72 void Arm64JNIMacroAssembler::DecreaseFrameSize(size_t adjust) {
73 if (adjust != 0u) {
74 CHECK_ALIGNED(adjust, kStackAlignment);
75 AddConstant(SP, adjust);
76 cfi().AdjustCFAOffset(-adjust);
77 }
78 }
79
AddConstant(XRegister rd,int32_t value,Condition cond)80 void Arm64JNIMacroAssembler::AddConstant(XRegister rd, int32_t value, Condition cond) {
81 AddConstant(rd, rd, value, cond);
82 }
83
AddConstant(XRegister rd,XRegister rn,int32_t value,Condition cond)84 void Arm64JNIMacroAssembler::AddConstant(XRegister rd,
85 XRegister rn,
86 int32_t value,
87 Condition cond) {
88 if ((cond == al) || (cond == nv)) {
89 // VIXL macro-assembler handles all variants.
90 ___ Add(reg_x(rd), reg_x(rn), value);
91 } else {
92 // temp = rd + value
93 // rd = cond ? temp : rn
94 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
95 temps.Exclude(reg_x(rd), reg_x(rn));
96 Register temp = temps.AcquireX();
97 ___ Add(temp, reg_x(rn), value);
98 ___ Csel(reg_x(rd), temp, reg_x(rd), cond);
99 }
100 }
101
StoreWToOffset(StoreOperandType type,WRegister source,XRegister base,int32_t offset)102 void Arm64JNIMacroAssembler::StoreWToOffset(StoreOperandType type,
103 WRegister source,
104 XRegister base,
105 int32_t offset) {
106 switch (type) {
107 case kStoreByte:
108 ___ Strb(reg_w(source), MEM_OP(reg_x(base), offset));
109 break;
110 case kStoreHalfword:
111 ___ Strh(reg_w(source), MEM_OP(reg_x(base), offset));
112 break;
113 case kStoreWord:
114 ___ Str(reg_w(source), MEM_OP(reg_x(base), offset));
115 break;
116 default:
117 LOG(FATAL) << "UNREACHABLE";
118 }
119 }
120
StoreToOffset(XRegister source,XRegister base,int32_t offset)121 void Arm64JNIMacroAssembler::StoreToOffset(XRegister source, XRegister base, int32_t offset) {
122 CHECK_NE(source, SP);
123 ___ Str(reg_x(source), MEM_OP(reg_x(base), offset));
124 }
125
StoreSToOffset(SRegister source,XRegister base,int32_t offset)126 void Arm64JNIMacroAssembler::StoreSToOffset(SRegister source, XRegister base, int32_t offset) {
127 ___ Str(reg_s(source), MEM_OP(reg_x(base), offset));
128 }
129
StoreDToOffset(DRegister source,XRegister base,int32_t offset)130 void Arm64JNIMacroAssembler::StoreDToOffset(DRegister source, XRegister base, int32_t offset) {
131 ___ Str(reg_d(source), MEM_OP(reg_x(base), offset));
132 }
133
Store(FrameOffset offs,ManagedRegister m_src,size_t size)134 void Arm64JNIMacroAssembler::Store(FrameOffset offs, ManagedRegister m_src, size_t size) {
135 Arm64ManagedRegister src = m_src.AsArm64();
136 if (src.IsNoRegister()) {
137 CHECK_EQ(0u, size);
138 } else if (src.IsWRegister()) {
139 CHECK_EQ(4u, size);
140 StoreWToOffset(kStoreWord, src.AsWRegister(), SP, offs.Int32Value());
141 } else if (src.IsXRegister()) {
142 CHECK_EQ(8u, size);
143 StoreToOffset(src.AsXRegister(), SP, offs.Int32Value());
144 } else if (src.IsSRegister()) {
145 StoreSToOffset(src.AsSRegister(), SP, offs.Int32Value());
146 } else {
147 CHECK(src.IsDRegister()) << src;
148 StoreDToOffset(src.AsDRegister(), SP, offs.Int32Value());
149 }
150 }
151
StoreRef(FrameOffset offs,ManagedRegister m_src)152 void Arm64JNIMacroAssembler::StoreRef(FrameOffset offs, ManagedRegister m_src) {
153 Arm64ManagedRegister src = m_src.AsArm64();
154 CHECK(src.IsXRegister()) << src;
155 StoreWToOffset(kStoreWord, src.AsOverlappingWRegister(), SP,
156 offs.Int32Value());
157 }
158
StoreRawPtr(FrameOffset offs,ManagedRegister m_src)159 void Arm64JNIMacroAssembler::StoreRawPtr(FrameOffset offs, ManagedRegister m_src) {
160 Arm64ManagedRegister src = m_src.AsArm64();
161 CHECK(src.IsXRegister()) << src;
162 StoreToOffset(src.AsXRegister(), SP, offs.Int32Value());
163 }
164
StoreImmediateToFrame(FrameOffset offs,uint32_t imm)165 void Arm64JNIMacroAssembler::StoreImmediateToFrame(FrameOffset offs, uint32_t imm) {
166 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
167 Register scratch = temps.AcquireW();
168 ___ Mov(scratch, imm);
169 ___ Str(scratch, MEM_OP(reg_x(SP), offs.Int32Value()));
170 }
171
StoreStackOffsetToThread(ThreadOffset64 tr_offs,FrameOffset fr_offs)172 void Arm64JNIMacroAssembler::StoreStackOffsetToThread(ThreadOffset64 tr_offs, FrameOffset fr_offs) {
173 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
174 Register scratch = temps.AcquireX();
175 ___ Add(scratch, reg_x(SP), fr_offs.Int32Value());
176 ___ Str(scratch, MEM_OP(reg_x(TR), tr_offs.Int32Value()));
177 }
178
StoreStackPointerToThread(ThreadOffset64 tr_offs)179 void Arm64JNIMacroAssembler::StoreStackPointerToThread(ThreadOffset64 tr_offs) {
180 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
181 Register scratch = temps.AcquireX();
182 ___ Mov(scratch, reg_x(SP));
183 ___ Str(scratch, MEM_OP(reg_x(TR), tr_offs.Int32Value()));
184 }
185
StoreSpanning(FrameOffset dest_off ATTRIBUTE_UNUSED,ManagedRegister m_source ATTRIBUTE_UNUSED,FrameOffset in_off ATTRIBUTE_UNUSED)186 void Arm64JNIMacroAssembler::StoreSpanning(FrameOffset dest_off ATTRIBUTE_UNUSED,
187 ManagedRegister m_source ATTRIBUTE_UNUSED,
188 FrameOffset in_off ATTRIBUTE_UNUSED) {
189 UNIMPLEMENTED(FATAL); // This case is not applicable to ARM64.
190 }
191
192 // Load routines.
LoadImmediate(XRegister dest,int32_t value,Condition cond)193 void Arm64JNIMacroAssembler::LoadImmediate(XRegister dest, int32_t value, Condition cond) {
194 if ((cond == al) || (cond == nv)) {
195 ___ Mov(reg_x(dest), value);
196 } else {
197 // temp = value
198 // rd = cond ? temp : rd
199 if (value != 0) {
200 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
201 temps.Exclude(reg_x(dest));
202 Register temp = temps.AcquireX();
203 ___ Mov(temp, value);
204 ___ Csel(reg_x(dest), temp, reg_x(dest), cond);
205 } else {
206 ___ Csel(reg_x(dest), reg_x(XZR), reg_x(dest), cond);
207 }
208 }
209 }
210
LoadWFromOffset(LoadOperandType type,WRegister dest,XRegister base,int32_t offset)211 void Arm64JNIMacroAssembler::LoadWFromOffset(LoadOperandType type,
212 WRegister dest,
213 XRegister base,
214 int32_t offset) {
215 switch (type) {
216 case kLoadSignedByte:
217 ___ Ldrsb(reg_w(dest), MEM_OP(reg_x(base), offset));
218 break;
219 case kLoadSignedHalfword:
220 ___ Ldrsh(reg_w(dest), MEM_OP(reg_x(base), offset));
221 break;
222 case kLoadUnsignedByte:
223 ___ Ldrb(reg_w(dest), MEM_OP(reg_x(base), offset));
224 break;
225 case kLoadUnsignedHalfword:
226 ___ Ldrh(reg_w(dest), MEM_OP(reg_x(base), offset));
227 break;
228 case kLoadWord:
229 ___ Ldr(reg_w(dest), MEM_OP(reg_x(base), offset));
230 break;
231 default:
232 LOG(FATAL) << "UNREACHABLE";
233 }
234 }
235
236 // Note: We can extend this member by adding load type info - see
237 // sign extended A64 load variants.
LoadFromOffset(XRegister dest,XRegister base,int32_t offset)238 void Arm64JNIMacroAssembler::LoadFromOffset(XRegister dest, XRegister base, int32_t offset) {
239 CHECK_NE(dest, SP);
240 ___ Ldr(reg_x(dest), MEM_OP(reg_x(base), offset));
241 }
242
LoadSFromOffset(SRegister dest,XRegister base,int32_t offset)243 void Arm64JNIMacroAssembler::LoadSFromOffset(SRegister dest, XRegister base, int32_t offset) {
244 ___ Ldr(reg_s(dest), MEM_OP(reg_x(base), offset));
245 }
246
LoadDFromOffset(DRegister dest,XRegister base,int32_t offset)247 void Arm64JNIMacroAssembler::LoadDFromOffset(DRegister dest, XRegister base, int32_t offset) {
248 ___ Ldr(reg_d(dest), MEM_OP(reg_x(base), offset));
249 }
250
Load(Arm64ManagedRegister dest,XRegister base,int32_t offset,size_t size)251 void Arm64JNIMacroAssembler::Load(Arm64ManagedRegister dest,
252 XRegister base,
253 int32_t offset,
254 size_t size) {
255 if (dest.IsNoRegister()) {
256 CHECK_EQ(0u, size) << dest;
257 } else if (dest.IsWRegister()) {
258 CHECK_EQ(4u, size) << dest;
259 ___ Ldr(reg_w(dest.AsWRegister()), MEM_OP(reg_x(base), offset));
260 } else if (dest.IsXRegister()) {
261 CHECK_NE(dest.AsXRegister(), SP) << dest;
262
263 if (size == 1u) {
264 ___ Ldrb(reg_w(dest.AsOverlappingWRegister()), MEM_OP(reg_x(base), offset));
265 } else if (size == 4u) {
266 ___ Ldr(reg_w(dest.AsOverlappingWRegister()), MEM_OP(reg_x(base), offset));
267 } else {
268 CHECK_EQ(8u, size) << dest;
269 ___ Ldr(reg_x(dest.AsXRegister()), MEM_OP(reg_x(base), offset));
270 }
271 } else if (dest.IsSRegister()) {
272 ___ Ldr(reg_s(dest.AsSRegister()), MEM_OP(reg_x(base), offset));
273 } else {
274 CHECK(dest.IsDRegister()) << dest;
275 ___ Ldr(reg_d(dest.AsDRegister()), MEM_OP(reg_x(base), offset));
276 }
277 }
278
Load(ManagedRegister m_dst,FrameOffset src,size_t size)279 void Arm64JNIMacroAssembler::Load(ManagedRegister m_dst, FrameOffset src, size_t size) {
280 return Load(m_dst.AsArm64(), SP, src.Int32Value(), size);
281 }
282
LoadFromThread(ManagedRegister m_dst,ThreadOffset64 src,size_t size)283 void Arm64JNIMacroAssembler::LoadFromThread(ManagedRegister m_dst,
284 ThreadOffset64 src,
285 size_t size) {
286 return Load(m_dst.AsArm64(), TR, src.Int32Value(), size);
287 }
288
LoadRef(ManagedRegister m_dst,FrameOffset offs)289 void Arm64JNIMacroAssembler::LoadRef(ManagedRegister m_dst, FrameOffset offs) {
290 Arm64ManagedRegister dst = m_dst.AsArm64();
291 CHECK(dst.IsXRegister()) << dst;
292 LoadWFromOffset(kLoadWord, dst.AsOverlappingWRegister(), SP, offs.Int32Value());
293 }
294
LoadRef(ManagedRegister m_dst,ManagedRegister m_base,MemberOffset offs,bool unpoison_reference)295 void Arm64JNIMacroAssembler::LoadRef(ManagedRegister m_dst,
296 ManagedRegister m_base,
297 MemberOffset offs,
298 bool unpoison_reference) {
299 Arm64ManagedRegister dst = m_dst.AsArm64();
300 Arm64ManagedRegister base = m_base.AsArm64();
301 CHECK(dst.IsXRegister() && base.IsXRegister());
302 LoadWFromOffset(kLoadWord, dst.AsOverlappingWRegister(), base.AsXRegister(),
303 offs.Int32Value());
304 if (unpoison_reference) {
305 WRegister ref_reg = dst.AsOverlappingWRegister();
306 asm_.MaybeUnpoisonHeapReference(reg_w(ref_reg));
307 }
308 }
309
LoadRawPtr(ManagedRegister m_dst,ManagedRegister m_base,Offset offs)310 void Arm64JNIMacroAssembler::LoadRawPtr(ManagedRegister m_dst,
311 ManagedRegister m_base,
312 Offset offs) {
313 Arm64ManagedRegister dst = m_dst.AsArm64();
314 Arm64ManagedRegister base = m_base.AsArm64();
315 CHECK(dst.IsXRegister() && base.IsXRegister());
316 // Remove dst and base form the temp list - higher level API uses IP1, IP0.
317 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
318 temps.Exclude(reg_x(dst.AsXRegister()), reg_x(base.AsXRegister()));
319 ___ Ldr(reg_x(dst.AsXRegister()), MEM_OP(reg_x(base.AsXRegister()), offs.Int32Value()));
320 }
321
LoadRawPtrFromThread(ManagedRegister m_dst,ThreadOffset64 offs)322 void Arm64JNIMacroAssembler::LoadRawPtrFromThread(ManagedRegister m_dst, ThreadOffset64 offs) {
323 Arm64ManagedRegister dst = m_dst.AsArm64();
324 CHECK(dst.IsXRegister()) << dst;
325 LoadFromOffset(dst.AsXRegister(), TR, offs.Int32Value());
326 }
327
328 // Copying routines.
MoveArguments(ArrayRef<ArgumentLocation> dests,ArrayRef<ArgumentLocation> srcs)329 void Arm64JNIMacroAssembler::MoveArguments(ArrayRef<ArgumentLocation> dests,
330 ArrayRef<ArgumentLocation> srcs) {
331 DCHECK_EQ(dests.size(), srcs.size());
332 auto get_mask = [](ManagedRegister reg) -> uint64_t {
333 Arm64ManagedRegister arm64_reg = reg.AsArm64();
334 if (arm64_reg.IsXRegister()) {
335 size_t core_reg_number = static_cast<size_t>(arm64_reg.AsXRegister());
336 DCHECK_LT(core_reg_number, 31u); // xSP, xZR not allowed.
337 return UINT64_C(1) << core_reg_number;
338 } else if (arm64_reg.IsWRegister()) {
339 size_t core_reg_number = static_cast<size_t>(arm64_reg.AsWRegister());
340 DCHECK_LT(core_reg_number, 31u); // wSP, wZR not allowed.
341 return UINT64_C(1) << core_reg_number;
342 } else if (arm64_reg.IsDRegister()) {
343 size_t fp_reg_number = static_cast<size_t>(arm64_reg.AsDRegister());
344 DCHECK_LT(fp_reg_number, 32u);
345 return (UINT64_C(1) << 32u) << fp_reg_number;
346 } else {
347 DCHECK(arm64_reg.IsSRegister());
348 size_t fp_reg_number = static_cast<size_t>(arm64_reg.AsSRegister());
349 DCHECK_LT(fp_reg_number, 32u);
350 return (UINT64_C(1) << 32u) << fp_reg_number;
351 }
352 };
353 // Collect registers to move while storing/copying args to stack slots.
354 // More than 8 core or FP reg args are very rare, so we do not optimize
355 // for that case by using LDP/STP.
356 // TODO: LDP/STP will be useful for normal and @FastNative where we need
357 // to spill even the leading arguments.
358 uint64_t src_regs = 0u;
359 uint64_t dest_regs = 0u;
360 for (size_t i = 0, arg_count = srcs.size(); i != arg_count; ++i) {
361 const ArgumentLocation& src = srcs[i];
362 const ArgumentLocation& dest = dests[i];
363 DCHECK_EQ(src.GetSize(), dest.GetSize());
364 if (dest.IsRegister()) {
365 if (src.IsRegister() && src.GetRegister().Equals(dest.GetRegister())) {
366 // Nothing to do.
367 } else {
368 if (src.IsRegister()) {
369 src_regs |= get_mask(src.GetRegister());
370 }
371 dest_regs |= get_mask(dest.GetRegister());
372 }
373 } else {
374 if (src.IsRegister()) {
375 Store(dest.GetFrameOffset(), src.GetRegister(), dest.GetSize());
376 } else {
377 Copy(dest.GetFrameOffset(), src.GetFrameOffset(), dest.GetSize());
378 }
379 }
380 }
381 // Fill destination registers.
382 // There should be no cycles, so this simple algorithm should make progress.
383 while (dest_regs != 0u) {
384 uint64_t old_dest_regs = dest_regs;
385 for (size_t i = 0, arg_count = srcs.size(); i != arg_count; ++i) {
386 const ArgumentLocation& src = srcs[i];
387 const ArgumentLocation& dest = dests[i];
388 if (!dest.IsRegister()) {
389 continue; // Stored in first loop above.
390 }
391 uint64_t dest_reg_mask = get_mask(dest.GetRegister());
392 if ((dest_reg_mask & dest_regs) == 0u) {
393 continue; // Equals source, or already filled in one of previous iterations.
394 }
395 if ((dest_reg_mask & src_regs) != 0u) {
396 continue; // Cannot clobber this register yet.
397 }
398 if (src.IsRegister()) {
399 Move(dest.GetRegister(), src.GetRegister(), dest.GetSize());
400 src_regs &= ~get_mask(src.GetRegister()); // Allow clobbering source register.
401 } else {
402 Load(dest.GetRegister(), src.GetFrameOffset(), dest.GetSize());
403 }
404 dest_regs &= ~get_mask(dest.GetRegister()); // Destination register was filled.
405 }
406 CHECK_NE(old_dest_regs, dest_regs);
407 DCHECK_EQ(0u, dest_regs & ~old_dest_regs);
408 }
409 }
410
Move(ManagedRegister m_dst,ManagedRegister m_src,size_t size)411 void Arm64JNIMacroAssembler::Move(ManagedRegister m_dst, ManagedRegister m_src, size_t size) {
412 Arm64ManagedRegister dst = m_dst.AsArm64();
413 if (kIsDebugBuild) {
414 // Check that the destination is not a scratch register.
415 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
416 if (dst.IsXRegister()) {
417 CHECK(!temps.IsAvailable(reg_x(dst.AsXRegister())));
418 } else if (dst.IsWRegister()) {
419 CHECK(!temps.IsAvailable(reg_w(dst.AsWRegister())));
420 } else if (dst.IsSRegister()) {
421 CHECK(!temps.IsAvailable(reg_s(dst.AsSRegister())));
422 } else {
423 CHECK(!temps.IsAvailable(reg_d(dst.AsDRegister())));
424 }
425 }
426 Arm64ManagedRegister src = m_src.AsArm64();
427 if (!dst.Equals(src)) {
428 if (dst.IsXRegister()) {
429 if (size == 4) {
430 CHECK(src.IsWRegister());
431 ___ Mov(reg_w(dst.AsOverlappingWRegister()), reg_w(src.AsWRegister()));
432 } else {
433 if (src.IsXRegister()) {
434 ___ Mov(reg_x(dst.AsXRegister()), reg_x(src.AsXRegister()));
435 } else {
436 ___ Mov(reg_x(dst.AsXRegister()), reg_x(src.AsOverlappingXRegister()));
437 }
438 }
439 } else if (dst.IsWRegister()) {
440 CHECK(src.IsWRegister()) << src;
441 ___ Mov(reg_w(dst.AsWRegister()), reg_w(src.AsWRegister()));
442 } else if (dst.IsSRegister()) {
443 CHECK(src.IsSRegister()) << src;
444 ___ Fmov(reg_s(dst.AsSRegister()), reg_s(src.AsSRegister()));
445 } else {
446 CHECK(dst.IsDRegister()) << dst;
447 CHECK(src.IsDRegister()) << src;
448 ___ Fmov(reg_d(dst.AsDRegister()), reg_d(src.AsDRegister()));
449 }
450 }
451 }
452
CopyRawPtrFromThread(FrameOffset fr_offs,ThreadOffset64 tr_offs)453 void Arm64JNIMacroAssembler::CopyRawPtrFromThread(FrameOffset fr_offs, ThreadOffset64 tr_offs) {
454 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
455 Register scratch = temps.AcquireX();
456 ___ Ldr(scratch, MEM_OP(reg_x(TR), tr_offs.Int32Value()));
457 ___ Str(scratch, MEM_OP(sp, fr_offs.Int32Value()));
458 }
459
CopyRawPtrToThread(ThreadOffset64 tr_offs,FrameOffset fr_offs,ManagedRegister m_scratch)460 void Arm64JNIMacroAssembler::CopyRawPtrToThread(ThreadOffset64 tr_offs,
461 FrameOffset fr_offs,
462 ManagedRegister m_scratch) {
463 Arm64ManagedRegister scratch = m_scratch.AsArm64();
464 CHECK(scratch.IsXRegister()) << scratch;
465 LoadFromOffset(scratch.AsXRegister(), SP, fr_offs.Int32Value());
466 StoreToOffset(scratch.AsXRegister(), TR, tr_offs.Int32Value());
467 }
468
CopyRef(FrameOffset dest,FrameOffset src)469 void Arm64JNIMacroAssembler::CopyRef(FrameOffset dest, FrameOffset src) {
470 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
471 Register scratch = temps.AcquireW();
472 ___ Ldr(scratch, MEM_OP(reg_x(SP), src.Int32Value()));
473 ___ Str(scratch, MEM_OP(reg_x(SP), dest.Int32Value()));
474 }
475
CopyRef(FrameOffset dest,ManagedRegister base,MemberOffset offs,bool unpoison_reference)476 void Arm64JNIMacroAssembler::CopyRef(FrameOffset dest,
477 ManagedRegister base,
478 MemberOffset offs,
479 bool unpoison_reference) {
480 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
481 Register scratch = temps.AcquireW();
482 ___ Ldr(scratch, MEM_OP(reg_x(base.AsArm64().AsXRegister()), offs.Int32Value()));
483 if (unpoison_reference) {
484 asm_.MaybeUnpoisonHeapReference(scratch);
485 }
486 ___ Str(scratch, MEM_OP(reg_x(SP), dest.Int32Value()));
487 }
488
Copy(FrameOffset dest,FrameOffset src,size_t size)489 void Arm64JNIMacroAssembler::Copy(FrameOffset dest, FrameOffset src, size_t size) {
490 DCHECK(size == 4 || size == 8) << size;
491 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
492 Register scratch = (size == 8) ? temps.AcquireX() : temps.AcquireW();
493 ___ Ldr(scratch, MEM_OP(reg_x(SP), src.Int32Value()));
494 ___ Str(scratch, MEM_OP(reg_x(SP), dest.Int32Value()));
495 }
496
Copy(FrameOffset dest,ManagedRegister src_base,Offset src_offset,ManagedRegister m_scratch,size_t size)497 void Arm64JNIMacroAssembler::Copy(FrameOffset dest,
498 ManagedRegister src_base,
499 Offset src_offset,
500 ManagedRegister m_scratch,
501 size_t size) {
502 Arm64ManagedRegister scratch = m_scratch.AsArm64();
503 Arm64ManagedRegister base = src_base.AsArm64();
504 CHECK(base.IsXRegister()) << base;
505 CHECK(scratch.IsXRegister() || scratch.IsWRegister()) << scratch;
506 CHECK(size == 4 || size == 8) << size;
507 if (size == 4) {
508 LoadWFromOffset(kLoadWord, scratch.AsWRegister(), base.AsXRegister(),
509 src_offset.Int32Value());
510 StoreWToOffset(kStoreWord, scratch.AsWRegister(), SP, dest.Int32Value());
511 } else if (size == 8) {
512 LoadFromOffset(scratch.AsXRegister(), base.AsXRegister(), src_offset.Int32Value());
513 StoreToOffset(scratch.AsXRegister(), SP, dest.Int32Value());
514 } else {
515 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
516 }
517 }
518
Copy(ManagedRegister m_dest_base,Offset dest_offs,FrameOffset src,ManagedRegister m_scratch,size_t size)519 void Arm64JNIMacroAssembler::Copy(ManagedRegister m_dest_base,
520 Offset dest_offs,
521 FrameOffset src,
522 ManagedRegister m_scratch,
523 size_t size) {
524 Arm64ManagedRegister scratch = m_scratch.AsArm64();
525 Arm64ManagedRegister base = m_dest_base.AsArm64();
526 CHECK(base.IsXRegister()) << base;
527 CHECK(scratch.IsXRegister() || scratch.IsWRegister()) << scratch;
528 CHECK(size == 4 || size == 8) << size;
529 if (size == 4) {
530 LoadWFromOffset(kLoadWord, scratch.AsWRegister(), SP, src.Int32Value());
531 StoreWToOffset(kStoreWord, scratch.AsWRegister(), base.AsXRegister(),
532 dest_offs.Int32Value());
533 } else if (size == 8) {
534 LoadFromOffset(scratch.AsXRegister(), SP, src.Int32Value());
535 StoreToOffset(scratch.AsXRegister(), base.AsXRegister(), dest_offs.Int32Value());
536 } else {
537 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
538 }
539 }
540
Copy(FrameOffset,FrameOffset,Offset,ManagedRegister,size_t)541 void Arm64JNIMacroAssembler::Copy(FrameOffset /*dst*/,
542 FrameOffset /*src_base*/,
543 Offset /*src_offset*/,
544 ManagedRegister /*mscratch*/,
545 size_t /*size*/) {
546 UNIMPLEMENTED(FATAL) << "Unimplemented Copy() variant";
547 }
548
Copy(ManagedRegister m_dest,Offset dest_offset,ManagedRegister m_src,Offset src_offset,ManagedRegister m_scratch,size_t size)549 void Arm64JNIMacroAssembler::Copy(ManagedRegister m_dest,
550 Offset dest_offset,
551 ManagedRegister m_src,
552 Offset src_offset,
553 ManagedRegister m_scratch,
554 size_t size) {
555 Arm64ManagedRegister scratch = m_scratch.AsArm64();
556 Arm64ManagedRegister src = m_src.AsArm64();
557 Arm64ManagedRegister dest = m_dest.AsArm64();
558 CHECK(dest.IsXRegister()) << dest;
559 CHECK(src.IsXRegister()) << src;
560 CHECK(scratch.IsXRegister() || scratch.IsWRegister()) << scratch;
561 CHECK(size == 4 || size == 8) << size;
562 if (size == 4) {
563 if (scratch.IsWRegister()) {
564 LoadWFromOffset(kLoadWord, scratch.AsWRegister(), src.AsXRegister(),
565 src_offset.Int32Value());
566 StoreWToOffset(kStoreWord, scratch.AsWRegister(), dest.AsXRegister(),
567 dest_offset.Int32Value());
568 } else {
569 LoadWFromOffset(kLoadWord, scratch.AsOverlappingWRegister(), src.AsXRegister(),
570 src_offset.Int32Value());
571 StoreWToOffset(kStoreWord, scratch.AsOverlappingWRegister(), dest.AsXRegister(),
572 dest_offset.Int32Value());
573 }
574 } else if (size == 8) {
575 LoadFromOffset(scratch.AsXRegister(), src.AsXRegister(), src_offset.Int32Value());
576 StoreToOffset(scratch.AsXRegister(), dest.AsXRegister(), dest_offset.Int32Value());
577 } else {
578 UNIMPLEMENTED(FATAL) << "We only support Copy() of size 4 and 8";
579 }
580 }
581
Copy(FrameOffset,Offset,FrameOffset,Offset,ManagedRegister,size_t)582 void Arm64JNIMacroAssembler::Copy(FrameOffset /*dst*/,
583 Offset /*dest_offset*/,
584 FrameOffset /*src*/,
585 Offset /*src_offset*/,
586 ManagedRegister /*scratch*/,
587 size_t /*size*/) {
588 UNIMPLEMENTED(FATAL) << "Unimplemented Copy() variant";
589 }
590
MemoryBarrier(ManagedRegister m_scratch ATTRIBUTE_UNUSED)591 void Arm64JNIMacroAssembler::MemoryBarrier(ManagedRegister m_scratch ATTRIBUTE_UNUSED) {
592 // TODO: Should we check that m_scratch is IP? - see arm.
593 ___ Dmb(InnerShareable, BarrierAll);
594 }
595
SignExtend(ManagedRegister mreg,size_t size)596 void Arm64JNIMacroAssembler::SignExtend(ManagedRegister mreg, size_t size) {
597 Arm64ManagedRegister reg = mreg.AsArm64();
598 CHECK(size == 1 || size == 2) << size;
599 CHECK(reg.IsWRegister()) << reg;
600 if (size == 1) {
601 ___ Sxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
602 } else {
603 ___ Sxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
604 }
605 }
606
ZeroExtend(ManagedRegister mreg,size_t size)607 void Arm64JNIMacroAssembler::ZeroExtend(ManagedRegister mreg, size_t size) {
608 Arm64ManagedRegister reg = mreg.AsArm64();
609 CHECK(size == 1 || size == 2) << size;
610 CHECK(reg.IsWRegister()) << reg;
611 if (size == 1) {
612 ___ Uxtb(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
613 } else {
614 ___ Uxth(reg_w(reg.AsWRegister()), reg_w(reg.AsWRegister()));
615 }
616 }
617
VerifyObject(ManagedRegister,bool)618 void Arm64JNIMacroAssembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) {
619 // TODO: not validating references.
620 }
621
VerifyObject(FrameOffset,bool)622 void Arm64JNIMacroAssembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) {
623 // TODO: not validating references.
624 }
625
Jump(ManagedRegister m_base,Offset offs)626 void Arm64JNIMacroAssembler::Jump(ManagedRegister m_base, Offset offs) {
627 Arm64ManagedRegister base = m_base.AsArm64();
628 CHECK(base.IsXRegister()) << base;
629 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
630 Register scratch = temps.AcquireX();
631 ___ Ldr(scratch, MEM_OP(reg_x(base.AsXRegister()), offs.Int32Value()));
632 ___ Br(scratch);
633 }
634
Call(ManagedRegister m_base,Offset offs)635 void Arm64JNIMacroAssembler::Call(ManagedRegister m_base, Offset offs) {
636 Arm64ManagedRegister base = m_base.AsArm64();
637 CHECK(base.IsXRegister()) << base;
638 ___ Ldr(lr, MEM_OP(reg_x(base.AsXRegister()), offs.Int32Value()));
639 ___ Blr(lr);
640 }
641
Call(FrameOffset base,Offset offs)642 void Arm64JNIMacroAssembler::Call(FrameOffset base, Offset offs) {
643 // Call *(*(SP + base) + offset)
644 ___ Ldr(lr, MEM_OP(reg_x(SP), base.Int32Value()));
645 ___ Ldr(lr, MEM_OP(lr, offs.Int32Value()));
646 ___ Blr(lr);
647 }
648
CallFromThread(ThreadOffset64 offset ATTRIBUTE_UNUSED)649 void Arm64JNIMacroAssembler::CallFromThread(ThreadOffset64 offset ATTRIBUTE_UNUSED) {
650 UNIMPLEMENTED(FATAL) << "Unimplemented Call() variant";
651 }
652
CreateHandleScopeEntry(ManagedRegister m_out_reg,FrameOffset handle_scope_offs,ManagedRegister m_in_reg,bool null_allowed)653 void Arm64JNIMacroAssembler::CreateHandleScopeEntry(ManagedRegister m_out_reg,
654 FrameOffset handle_scope_offs,
655 ManagedRegister m_in_reg,
656 bool null_allowed) {
657 Arm64ManagedRegister out_reg = m_out_reg.AsArm64();
658 Arm64ManagedRegister in_reg = m_in_reg.AsArm64();
659 // For now we only hold stale handle scope entries in x registers.
660 CHECK(in_reg.IsNoRegister() || in_reg.IsXRegister()) << in_reg;
661 CHECK(out_reg.IsXRegister()) << out_reg;
662 if (null_allowed) {
663 // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is
664 // the address in the handle scope holding the reference.
665 // e.g. out_reg = (handle == 0) ? 0 : (SP+handle_offset)
666 if (in_reg.IsNoRegister()) {
667 LoadWFromOffset(kLoadWord, out_reg.AsOverlappingWRegister(), SP,
668 handle_scope_offs.Int32Value());
669 in_reg = out_reg;
670 }
671 ___ Cmp(reg_w(in_reg.AsOverlappingWRegister()), 0);
672 if (!out_reg.Equals(in_reg)) {
673 LoadImmediate(out_reg.AsXRegister(), 0, eq);
674 }
675 AddConstant(out_reg.AsXRegister(), SP, handle_scope_offs.Int32Value(), ne);
676 } else {
677 AddConstant(out_reg.AsXRegister(), SP, handle_scope_offs.Int32Value(), al);
678 }
679 }
680
CreateHandleScopeEntry(FrameOffset out_off,FrameOffset handle_scope_offset,bool null_allowed)681 void Arm64JNIMacroAssembler::CreateHandleScopeEntry(FrameOffset out_off,
682 FrameOffset handle_scope_offset,
683 bool null_allowed) {
684 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
685 Register scratch = temps.AcquireX();
686 if (null_allowed) {
687 Register scratch2 = temps.AcquireW();
688 ___ Ldr(scratch2, MEM_OP(reg_x(SP), handle_scope_offset.Int32Value()));
689 ___ Add(scratch, reg_x(SP), handle_scope_offset.Int32Value());
690 // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is
691 // the address in the handle scope holding the reference.
692 // e.g. scratch = (scratch == 0) ? 0 : (SP+handle_scope_offset)
693 ___ Cmp(scratch2, 0);
694 ___ Csel(scratch, scratch, xzr, ne);
695 } else {
696 ___ Add(scratch, reg_x(SP), handle_scope_offset.Int32Value());
697 }
698 ___ Str(scratch, MEM_OP(reg_x(SP), out_off.Int32Value()));
699 }
700
LoadReferenceFromHandleScope(ManagedRegister m_out_reg,ManagedRegister m_in_reg)701 void Arm64JNIMacroAssembler::LoadReferenceFromHandleScope(ManagedRegister m_out_reg,
702 ManagedRegister m_in_reg) {
703 Arm64ManagedRegister out_reg = m_out_reg.AsArm64();
704 Arm64ManagedRegister in_reg = m_in_reg.AsArm64();
705 CHECK(out_reg.IsXRegister()) << out_reg;
706 CHECK(in_reg.IsXRegister()) << in_reg;
707 vixl::aarch64::Label exit;
708 if (!out_reg.Equals(in_reg)) {
709 // FIXME: Who sets the flags here?
710 LoadImmediate(out_reg.AsXRegister(), 0, eq);
711 }
712 ___ Cbz(reg_x(in_reg.AsXRegister()), &exit);
713 LoadFromOffset(out_reg.AsXRegister(), in_reg.AsXRegister(), 0);
714 ___ Bind(&exit);
715 }
716
ExceptionPoll(size_t stack_adjust)717 void Arm64JNIMacroAssembler::ExceptionPoll(size_t stack_adjust) {
718 CHECK_ALIGNED(stack_adjust, kStackAlignment);
719 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
720 Register scratch = temps.AcquireX();
721 exception_blocks_.emplace_back(new Arm64Exception(scratch, stack_adjust));
722 ___ Ldr(scratch, MEM_OP(reg_x(TR), Thread::ExceptionOffset<kArm64PointerSize>().Int32Value()));
723 ___ Cbnz(scratch, exception_blocks_.back()->Entry());
724 }
725
CreateLabel()726 std::unique_ptr<JNIMacroLabel> Arm64JNIMacroAssembler::CreateLabel() {
727 return std::unique_ptr<JNIMacroLabel>(new Arm64JNIMacroLabel());
728 }
729
Jump(JNIMacroLabel * label)730 void Arm64JNIMacroAssembler::Jump(JNIMacroLabel* label) {
731 CHECK(label != nullptr);
732 ___ B(Arm64JNIMacroLabel::Cast(label)->AsArm64());
733 }
734
TestGcMarking(JNIMacroLabel * label,JNIMacroUnaryCondition cond)735 void Arm64JNIMacroAssembler::TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) {
736 CHECK(label != nullptr);
737
738 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
739 DCHECK_EQ(Thread::IsGcMarkingSize(), 4u);
740 Register scratch = temps.AcquireW();
741 ___ Ldr(scratch, MEM_OP(reg_x(TR), Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value()));
742 switch (cond) {
743 case JNIMacroUnaryCondition::kZero:
744 ___ Cbz(scratch, Arm64JNIMacroLabel::Cast(label)->AsArm64());
745 break;
746 case JNIMacroUnaryCondition::kNotZero:
747 ___ Cbnz(scratch, Arm64JNIMacroLabel::Cast(label)->AsArm64());
748 break;
749 default:
750 LOG(FATAL) << "Not implemented unary condition: " << static_cast<int>(cond);
751 UNREACHABLE();
752 }
753 }
754
Bind(JNIMacroLabel * label)755 void Arm64JNIMacroAssembler::Bind(JNIMacroLabel* label) {
756 CHECK(label != nullptr);
757 ___ Bind(Arm64JNIMacroLabel::Cast(label)->AsArm64());
758 }
759
EmitExceptionPoll(Arm64Exception * exception)760 void Arm64JNIMacroAssembler::EmitExceptionPoll(Arm64Exception* exception) {
761 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
762 temps.Exclude(exception->scratch_);
763
764 // Bind exception poll entry.
765 ___ Bind(exception->Entry());
766 if (exception->stack_adjust_ != 0) { // Fix up the frame.
767 DecreaseFrameSize(exception->stack_adjust_);
768 }
769 // Pass exception object as argument.
770 // Don't care about preserving X0 as this won't return.
771 ___ Mov(reg_x(X0), exception->scratch_);
772 ___ Ldr(lr,
773 MEM_OP(reg_x(TR),
774 QUICK_ENTRYPOINT_OFFSET(kArm64PointerSize, pDeliverException).Int32Value()));
775
776 ___ Blr(lr);
777 // Call should never return.
778 ___ Brk();
779 }
780
BuildFrame(size_t frame_size,ManagedRegister method_reg,ArrayRef<const ManagedRegister> callee_save_regs)781 void Arm64JNIMacroAssembler::BuildFrame(size_t frame_size,
782 ManagedRegister method_reg,
783 ArrayRef<const ManagedRegister> callee_save_regs) {
784 // Setup VIXL CPURegList for callee-saves.
785 CPURegList core_reg_list(CPURegister::kRegister, kXRegSize, 0);
786 CPURegList fp_reg_list(CPURegister::kVRegister, kDRegSize, 0);
787 for (auto r : callee_save_regs) {
788 Arm64ManagedRegister reg = r.AsArm64();
789 if (reg.IsXRegister()) {
790 core_reg_list.Combine(reg_x(reg.AsXRegister()).GetCode());
791 } else {
792 DCHECK(reg.IsDRegister());
793 fp_reg_list.Combine(reg_d(reg.AsDRegister()).GetCode());
794 }
795 }
796 size_t core_reg_size = core_reg_list.GetTotalSizeInBytes();
797 size_t fp_reg_size = fp_reg_list.GetTotalSizeInBytes();
798
799 // Increase frame to required size.
800 DCHECK_ALIGNED(frame_size, kStackAlignment);
801 // Must at least have space for Method* if we're going to spill it.
802 DCHECK_GE(frame_size,
803 core_reg_size + fp_reg_size + (method_reg.IsRegister() ? kXRegSizeInBytes : 0u));
804 IncreaseFrameSize(frame_size);
805
806 // Save callee-saves.
807 asm_.SpillRegisters(core_reg_list, frame_size - core_reg_size);
808 asm_.SpillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size);
809
810 if (method_reg.IsRegister()) {
811 // Write ArtMethod*
812 DCHECK(X0 == method_reg.AsArm64().AsXRegister());
813 StoreToOffset(X0, SP, 0);
814 }
815 }
816
RemoveFrame(size_t frame_size,ArrayRef<const ManagedRegister> callee_save_regs,bool may_suspend)817 void Arm64JNIMacroAssembler::RemoveFrame(size_t frame_size,
818 ArrayRef<const ManagedRegister> callee_save_regs,
819 bool may_suspend) {
820 // Setup VIXL CPURegList for callee-saves.
821 CPURegList core_reg_list(CPURegister::kRegister, kXRegSize, 0);
822 CPURegList fp_reg_list(CPURegister::kVRegister, kDRegSize, 0);
823 for (auto r : callee_save_regs) {
824 Arm64ManagedRegister reg = r.AsArm64();
825 if (reg.IsXRegister()) {
826 core_reg_list.Combine(reg_x(reg.AsXRegister()).GetCode());
827 } else {
828 DCHECK(reg.IsDRegister());
829 fp_reg_list.Combine(reg_d(reg.AsDRegister()).GetCode());
830 }
831 }
832 size_t core_reg_size = core_reg_list.GetTotalSizeInBytes();
833 size_t fp_reg_size = fp_reg_list.GetTotalSizeInBytes();
834
835 // For now we only check that the size of the frame is large enough to hold spills and method
836 // reference.
837 DCHECK_GE(frame_size, core_reg_size + fp_reg_size);
838 DCHECK_ALIGNED(frame_size, kAapcs64StackAlignment);
839
840 cfi().RememberState();
841
842 // Restore callee-saves.
843 asm_.UnspillRegisters(core_reg_list, frame_size - core_reg_size);
844 asm_.UnspillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size);
845
846 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
847 vixl::aarch64::Register mr = reg_x(MR); // Marking Register.
848 vixl::aarch64::Register tr = reg_x(TR); // Thread Register.
849
850 if (may_suspend) {
851 // The method may be suspended; refresh the Marking Register.
852 ___ Ldr(mr.W(), MemOperand(tr, Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value()));
853 } else {
854 // The method shall not be suspended; no need to refresh the Marking Register.
855
856 // The Marking Register is a callee-save register and thus has been
857 // preserved by native code following the AAPCS64 calling convention.
858
859 // The following condition is a compile-time one, so it does not have a run-time cost.
860 if (kIsDebugBuild) {
861 // The following condition is a run-time one; it is executed after the
862 // previous compile-time test, to avoid penalizing non-debug builds.
863 if (emit_run_time_checks_in_debug_mode_) {
864 // Emit a run-time check verifying that the Marking Register is up-to-date.
865 UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
866 Register temp = temps.AcquireW();
867 // Ensure we are not clobbering a callee-save register that was restored before.
868 DCHECK(!core_reg_list.IncludesAliasOf(temp.X()))
869 << "core_reg_list should not contain scratch register X" << temp.GetCode();
870 asm_.GenerateMarkingRegisterCheck(temp);
871 }
872 }
873 }
874 }
875
876 // Decrease frame size to start of callee saved regs.
877 DecreaseFrameSize(frame_size);
878
879 // Return to LR.
880 ___ Ret();
881
882 // The CFI should be restored for any code that follows the exit block.
883 cfi().RestoreState();
884 cfi().DefCFAOffset(frame_size);
885 }
886
887 #undef ___
888
889 } // namespace arm64
890 } // namespace art
891