1%def header(): 2/* 3 * Copyright (C) 2019 The Android Open Source Project 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18/* 19 * This is a #include, not a %include, because we want the C pre-processor 20 * to expand the macros into assembler assignment statements. 21 */ 22#include "asm_support.h" 23#include "arch/x86_64/asm_support_x86_64.S" 24 25/** 26 * x86_64 ABI general notes: 27 * 28 * Caller save set: 29 * rax, rdx, rcx, rsi, rdi, r8-r11, st(0)-st(7) 30 * Callee save set: 31 * rbx, rbp, r12-r15 32 * Return regs: 33 * 32-bit in eax 34 * 64-bit in rax 35 * fp on xmm0 36 * 37 * First 8 fp parameters came in xmm0-xmm7. 38 * First 6 non-fp parameters came in rdi, rsi, rdx, rcx, r8, r9. 39 * Other parameters passed on stack, pushed right-to-left. On entry to target, first 40 * param is at 8(%esp). 41 * 42 * Stack must be 16-byte aligned to support SSE in native code. 43 */ 44 45#define IN_ARG3 %rcx 46#define IN_ARG2 %rdx 47#define IN_ARG1 %rsi 48#define IN_ARG0 %rdi 49/* Out Args */ 50#define OUT_ARG3 %rcx 51#define OUT_ARG2 %rdx 52#define OUT_ARG1 %rsi 53#define OUT_ARG0 %rdi 54#define OUT_32_ARG3 %ecx 55#define OUT_32_ARG2 %edx 56#define OUT_32_ARG1 %esi 57#define OUT_32_ARG0 %edi 58#define OUT_FP_ARG1 %xmm1 59#define OUT_FP_ARG0 %xmm0 60 61/* 62 * single-purpose registers, given names for clarity 63 */ 64#define rSELF %gs 65#define rPC %r12 66#define CFI_DEX 12 // DWARF register number of the register holding dex-pc (rPC). 67#define CFI_TMP 5 // DWARF register number of the first argument register (rdi). 68#define rFP %r13 69#define rINST %ebx 70#define rINSTq %rbx 71#define rINSTw %bx 72#define rINSTbh %bh 73#define rINSTbl %bl 74#define rIBASE %r14 75#define rREFS %r15 76#define CFI_REFS 15 // DWARF register number of the reference array (r15). 77 78// Temporary registers while setting up a frame. 79#define rNEW_FP %r8 80#define rNEW_REFS %r9 81#define CFI_NEW_REFS 9 82 83/* 84 * Get/set the 32-bit value from a Dalvik register. 85 */ 86#define VREG_ADDRESS(_vreg) (rFP,_vreg,4) 87#define VREG_HIGH_ADDRESS(_vreg) 4(rFP,_vreg,4) 88#define VREG_REF_ADDRESS(_vreg) (rREFS,_vreg,4) 89#define VREG_REF_HIGH_ADDRESS(_vreg) 4(rREFS,_vreg,4) 90 91// Includes the return address implictly pushed on stack by 'call'. 92#define CALLEE_SAVES_SIZE (6 * 8 + 4 * 8 + 1 * 8) 93 94// +8 for the ArtMethod of the caller. 95#define OFFSET_TO_FIRST_ARGUMENT_IN_STACK (CALLEE_SAVES_SIZE + 8) 96 97/* 98 * Refresh rINST. 99 * At enter to handler rINST does not contain the opcode number. 100 * However some utilities require the full value, so this macro 101 * restores the opcode number. 102 */ 103.macro REFRESH_INST _opnum 104 movb rINSTbl, rINSTbh 105 movb $$\_opnum, rINSTbl 106.endm 107 108/* 109 * Fetch the next instruction from rPC into rINSTw. Does not advance rPC. 110 */ 111.macro FETCH_INST 112 movzwq (rPC), rINSTq 113.endm 114 115/* 116 * Remove opcode from rINST, compute the address of handler and jump to it. 117 */ 118.macro GOTO_NEXT 119 movzx rINSTbl,%ecx 120 movzbl rINSTbh,rINST 121 shll MACRO_LITERAL(${handler_size_bits}), %ecx 122 addq rIBASE, %rcx 123 jmp *%rcx 124.endm 125 126/* 127 * Advance rPC by instruction count. 128 */ 129.macro ADVANCE_PC _count 130 leaq 2*\_count(rPC), rPC 131.endm 132 133/* 134 * Advance rPC by instruction count, fetch instruction and jump to handler. 135 */ 136.macro ADVANCE_PC_FETCH_AND_GOTO_NEXT _count 137 ADVANCE_PC \_count 138 FETCH_INST 139 GOTO_NEXT 140.endm 141 142.macro GET_VREG _reg _vreg 143 movl VREG_ADDRESS(\_vreg), \_reg 144.endm 145 146.macro GET_VREG_OBJECT _reg _vreg 147 movl VREG_REF_ADDRESS(\_vreg), \_reg 148.endm 149 150/* Read wide value. */ 151.macro GET_WIDE_VREG _reg _vreg 152 movq VREG_ADDRESS(\_vreg), \_reg 153.endm 154 155.macro SET_VREG _reg _vreg 156 movl \_reg, VREG_ADDRESS(\_vreg) 157 movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) 158.endm 159 160/* Write wide value. reg is clobbered. */ 161.macro SET_WIDE_VREG _reg _vreg 162 movq \_reg, VREG_ADDRESS(\_vreg) 163 xorq \_reg, \_reg 164 movq \_reg, VREG_REF_ADDRESS(\_vreg) 165.endm 166 167.macro SET_VREG_OBJECT _reg _vreg 168 movl \_reg, VREG_ADDRESS(\_vreg) 169 movl \_reg, VREG_REF_ADDRESS(\_vreg) 170.endm 171 172.macro GET_VREG_HIGH _reg _vreg 173 movl VREG_HIGH_ADDRESS(\_vreg), \_reg 174.endm 175 176.macro SET_VREG_HIGH _reg _vreg 177 movl \_reg, VREG_HIGH_ADDRESS(\_vreg) 178 movl MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg) 179.endm 180 181.macro CLEAR_REF _vreg 182 movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) 183.endm 184 185.macro CLEAR_WIDE_REF _vreg 186 movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) 187 movl MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg) 188.endm 189 190.macro GET_VREG_XMMs _xmmreg _vreg 191 movss VREG_ADDRESS(\_vreg), \_xmmreg 192.endm 193.macro GET_VREG_XMMd _xmmreg _vreg 194 movsd VREG_ADDRESS(\_vreg), \_xmmreg 195.endm 196.macro SET_VREG_XMMs _xmmreg _vreg 197 movss \_xmmreg, VREG_ADDRESS(\_vreg) 198.endm 199.macro SET_VREG_XMMd _xmmreg _vreg 200 movsd \_xmmreg, VREG_ADDRESS(\_vreg) 201.endm 202 203// An assembly entry that has a OatQuickMethodHeader prefix. 204.macro OAT_ENTRY name, end 205 FUNCTION_TYPE(\name) 206 ASM_HIDDEN SYMBOL(\name) 207 .global SYMBOL(\name) 208 .balign 16 209 .long 0 210 .long (SYMBOL(\end) - SYMBOL(\name)) 211SYMBOL(\name): 212.endm 213 214.macro ENTRY name 215 .text 216 ASM_HIDDEN SYMBOL(\name) 217 .global SYMBOL(\name) 218 FUNCTION_TYPE(\name) 219SYMBOL(\name): 220.endm 221 222.macro END name 223 SIZE(\name) 224.endm 225 226// Macro for defining entrypoints into runtime. We don't need to save registers 227// (we're not holding references there), but there is no 228// kDontSave runtime method. So just use the kSaveRefsOnly runtime method. 229.macro NTERP_TRAMPOLINE name, helper 230DEFINE_FUNCTION \name 231 SETUP_SAVE_REFS_ONLY_FRAME 232 call \helper 233 RESTORE_SAVE_REFS_ONLY_FRAME 234 RETURN_OR_DELIVER_PENDING_EXCEPTION 235END_FUNCTION \name 236.endm 237 238.macro CLEAR_VOLATILE_MARKER reg 239 andq MACRO_LITERAL(-2), \reg 240.endm 241 242.macro EXPORT_PC 243 movq rPC, -16(rREFS) 244.endm 245 246 247.macro BRANCH 248 // Update method counter and do a suspend check if the branch is negative. 249 testq rINSTq, rINSTq 250 js 3f 2512: 252 leaq (rPC, rINSTq, 2), rPC 253 FETCH_INST 254 GOTO_NEXT 2553: 256 movq (%rsp), %rdi 257#if (NTERP_HOTNESS_MASK != 0xffff) 258#error "Nterp x86_64 expects Nterp hotness mask to be 0xffff" 259#endif 260 addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) 261 // If the counter overflows, handle this in the runtime. 262 jo NterpHandleHotnessOverflow 263 // Otherwise, do a suspend check. 264 testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET 265 jz 2b 266 EXPORT_PC 267 call SYMBOL(art_quick_test_suspend) 268 jmp 2b 269.endm 270 271// Setup the stack to start executing the method. Expects: 272// - rdi to contain the ArtMethod 273// - rbx, r10, r11 to be available. 274// 275// Outputs 276// - rbx contains the dex registers size 277// - r11 contains the old stack pointer. 278.macro SETUP_STACK_FRAME code_item, refs, fp, cfi_refs 279 // Fetch dex register size. 280 movzwl CODE_ITEM_REGISTERS_SIZE_OFFSET(\code_item), %ebx 281 // Fetch outs size. 282 movzwq CODE_ITEM_OUTS_SIZE_OFFSET(\code_item), \refs 283 284 // Compute required frame size for dex registers: ((2 * ebx) + refs) 285 leaq (\refs, %rbx, 2), %r11 286 salq $$2, %r11 287 288 // Compute new stack pointer in r10: add 24 for saving the previous frame, 289 // pc, and method being executed. 290 leaq -24(%rsp), %r10 291 subq %r11, %r10 292 // Alignment 293 // Note: There may be two pieces of alignment but there is no need to align 294 // out args to `kPointerSize` separately before aligning to kStackAlignment. 295 andq $$-16, %r10 296 297 // Set reference and dex registers, align to pointer size for previous frame and dex pc. 298 leaq 24 + 4(%r10, \refs, 4), \refs 299 andq LITERAL(-__SIZEOF_POINTER__), \refs 300 leaq (\refs, %rbx, 4), \fp 301 302 // Now setup the stack pointer. 303 movq %rsp, %r11 304 CFI_DEF_CFA_REGISTER(r11) 305 movq %r10, %rsp 306 movq %r11, -8(\refs) 307 CFI_DEF_CFA_BREG_PLUS_UCONST \cfi_refs, -8, ((6 + 4 + 1) * 8) 308 309 // Put nulls in reference frame. 310 testl %ebx, %ebx 311 je 2f 312 movq \refs, %r10 3131: 314 movl $$0, (%r10) 315 addq $$4, %r10 316 cmpq %r10, \fp 317 jne 1b 3182: 319 // Save the ArtMethod. 320 movq %rdi, (%rsp) 321.endm 322 323// Puts the next floating point argument into the expected register, 324// fetching values based on a non-range invoke. 325// Uses rax as temporary. 326// 327// TODO: We could simplify a lot of code by loading the G argument into 328// the "inst" register. Given that we enter the handler with "1(rPC)" in 329// the rINST, we can just add rINST<<16 to the args and we don't even 330// need to pass "arg_index" around. 331.macro LOOP_OVER_SHORTY_LOADING_XMMS xmm_reg, inst, shorty, arg_index, finished 3321: // LOOP 333 movb (REG_VAR(shorty)), %al // bl := *shorty 334 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 335 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 336 je VAR(finished) 337 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 338 je 2f 339 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 340 je 3f 341 shrq MACRO_LITERAL(4), REG_VAR(inst) 342 addq MACRO_LITERAL(1), REG_VAR(arg_index) 343 // Handle extra argument in arg array taken by a long. 344 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 345 jne 1b 346 shrq MACRO_LITERAL(4), REG_VAR(inst) 347 addq MACRO_LITERAL(1), REG_VAR(arg_index) 348 jmp 1b // goto LOOP 3492: // FOUND_DOUBLE 350 subq MACRO_LITERAL(8), %rsp 351 movq REG_VAR(inst), %rax 352 andq MACRO_LITERAL(0xf), %rax 353 GET_VREG %eax, %rax 354 movl %eax, (%rsp) 355 shrq MACRO_LITERAL(4), REG_VAR(inst) 356 addq MACRO_LITERAL(1), REG_VAR(arg_index) 357 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 358 je 5f 359 movq REG_VAR(inst), %rax 360 andq MACRO_LITERAL(0xf), %rax 361 shrq MACRO_LITERAL(4), REG_VAR(inst) 362 addq MACRO_LITERAL(1), REG_VAR(arg_index) 363 jmp 6f 3645: 365 movzbl 1(rPC), %eax 366 andq MACRO_LITERAL(0xf), %rax 3676: 368 GET_VREG %eax, %rax 369 movl %eax, 4(%rsp) 370 movsd (%rsp), REG_VAR(xmm_reg) 371 addq MACRO_LITERAL(8), %rsp 372 jmp 4f 3733: // FOUND_FLOAT 374 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 375 je 7f 376 movq REG_VAR(inst), %rax 377 andq MACRO_LITERAL(0xf), %rax 378 shrq MACRO_LITERAL(4), REG_VAR(inst) 379 addq MACRO_LITERAL(1), REG_VAR(arg_index) 380 jmp 8f 3817: 382 movzbl 1(rPC), %eax 383 andq MACRO_LITERAL(0xf), %rax 3848: 385 GET_VREG_XMMs REG_VAR(xmm_reg), %rax 3864: 387.endm 388 389// Puts the next int/long/object argument in the expected register, 390// fetching values based on a non-range invoke. 391// Uses rax as temporary. 392.macro LOOP_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, inst, shorty, arg_index, finished 3931: // LOOP 394 movb (REG_VAR(shorty)), %al // bl := *shorty 395 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 396 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 397 je VAR(finished) 398 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 399 je 2f 400 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 401 je 3f 402 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 403 je 4f 404 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 405 je 7f 406 movq REG_VAR(inst), %rax 407 andq MACRO_LITERAL(0xf), %rax 408 shrq MACRO_LITERAL(4), REG_VAR(inst) 409 addq MACRO_LITERAL(1), REG_VAR(arg_index) 410 jmp 8f 4117: 412 movzbl 1(rPC), %eax 413 andq MACRO_LITERAL(0xf), %rax 4148: 415 GET_VREG REG_VAR(gpr_reg32), %rax 416 jmp 5f 4172: // FOUND_LONG 418 subq MACRO_LITERAL(8), %rsp 419 movq REG_VAR(inst), %rax 420 andq MACRO_LITERAL(0xf), %rax 421 GET_VREG %eax, %rax 422 movl %eax, (%rsp) 423 shrq MACRO_LITERAL(4), REG_VAR(inst) 424 addq MACRO_LITERAL(1), REG_VAR(arg_index) 425 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 426 je 9f 427 movq REG_VAR(inst), %rax 428 andq MACRO_LITERAL(0xf), %rax 429 shrq MACRO_LITERAL(4), REG_VAR(inst) 430 addq MACRO_LITERAL(1), REG_VAR(arg_index) 431 jmp 10f 4329: 433 movzbl 1(rPC), %eax 434 andq MACRO_LITERAL(0xf), %rax 43510: 436 GET_VREG %eax, %rax 437 movl %eax, 4(%rsp) 438 movq (%rsp), REG_VAR(gpr_reg64) 439 addq MACRO_LITERAL(8), %rsp 440 jmp 5f 4413: // SKIP_FLOAT 442 shrq MACRO_LITERAL(4), REG_VAR(inst) 443 addq MACRO_LITERAL(1), REG_VAR(arg_index) 444 jmp 1b 4454: // SKIP_DOUBLE 446 shrq MACRO_LITERAL(4), REG_VAR(inst) 447 addq MACRO_LITERAL(1), REG_VAR(arg_index) 448 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 449 je 1b 450 shrq MACRO_LITERAL(4), REG_VAR(inst) 451 addq MACRO_LITERAL(1), REG_VAR(arg_index) 452 jmp 1b 4535: 454.endm 455 456// Puts the next floating point argument into the expected register, 457// fetching values based on a range invoke. 458// Uses rax as temporary. 459.macro LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm_reg, shorty, arg_index, stack_index, finished 4601: // LOOP 461 movb (REG_VAR(shorty)), %al // bl := *shorty 462 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 463 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 464 je VAR(finished) 465 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 466 je 2f 467 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 468 je 3f 469 addq MACRO_LITERAL(1), REG_VAR(arg_index) 470 addq MACRO_LITERAL(1), REG_VAR(stack_index) 471 // Handle extra argument in arg array taken by a long. 472 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 473 jne 1b 474 addq MACRO_LITERAL(1), REG_VAR(arg_index) 475 addq MACRO_LITERAL(1), REG_VAR(stack_index) 476 jmp 1b // goto LOOP 4772: // FOUND_DOUBLE 478 GET_VREG_XMMd REG_VAR(xmm_reg), REG_VAR(arg_index) 479 addq MACRO_LITERAL(2), REG_VAR(arg_index) 480 addq MACRO_LITERAL(2), REG_VAR(stack_index) 481 jmp 4f 4823: // FOUND_FLOAT 483 GET_VREG_XMMs REG_VAR(xmm_reg), REG_VAR(arg_index) 484 addq MACRO_LITERAL(1), REG_VAR(arg_index) 485 addq MACRO_LITERAL(1), REG_VAR(stack_index) 4864: 487.endm 488 489// Puts the next floating point argument into the expected stack slot, 490// fetching values based on a range invoke. 491// Uses rax as temporary. 492// 493// TODO: We could just copy all the vregs to the stack slots in a simple loop 494// (or REP MOVSD) without looking at the shorty at all. (We could also drop 495// the "stack_index" from the macros for loading registers.) We could also do 496// that conditionally if argument word count > 6; otherwise we know that all 497// args fit into registers. 498.macro LOOP_RANGE_OVER_FPs shorty, arg_index, stack_index, finished 4991: // LOOP 500 movb (REG_VAR(shorty)), %al // bl := *shorty 501 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 502 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 503 je VAR(finished) 504 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 505 je 2f 506 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 507 je 3f 508 addq MACRO_LITERAL(1), REG_VAR(arg_index) 509 addq MACRO_LITERAL(1), REG_VAR(stack_index) 510 // Handle extra argument in arg array taken by a long. 511 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 512 jne 1b 513 addq MACRO_LITERAL(1), REG_VAR(arg_index) 514 addq MACRO_LITERAL(1), REG_VAR(stack_index) 515 jmp 1b // goto LOOP 5162: // FOUND_DOUBLE 517 movq (rFP, REG_VAR(arg_index), 4), %rax 518 movq %rax, 8(%rsp, REG_VAR(stack_index), 4) 519 addq MACRO_LITERAL(2), REG_VAR(arg_index) 520 addq MACRO_LITERAL(2), REG_VAR(stack_index) 521 jmp 1b 5223: // FOUND_FLOAT 523 movl (rFP, REG_VAR(arg_index), 4), %eax 524 movl %eax, 8(%rsp, REG_VAR(stack_index), 4) 525 addq MACRO_LITERAL(1), REG_VAR(arg_index) 526 addq MACRO_LITERAL(1), REG_VAR(stack_index) 527 jmp 1b 528.endm 529 530// Puts the next int/long/object argument in the expected register, 531// fetching values based on a range invoke. 532// Uses rax as temporary. 533.macro LOOP_RANGE_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, shorty, arg_index, stack_index, finished 5341: // LOOP 535 movb (REG_VAR(shorty)), %al // bl := *shorty 536 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 537 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 538 je VAR(finished) 539 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 540 je 2f 541 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 542 je 3f 543 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 544 je 4f 545 movl (rFP, REG_VAR(arg_index), 4), REG_VAR(gpr_reg32) 546 addq MACRO_LITERAL(1), REG_VAR(arg_index) 547 addq MACRO_LITERAL(1), REG_VAR(stack_index) 548 jmp 5f 5492: // FOUND_LONG 550 movq (rFP, REG_VAR(arg_index), 4), REG_VAR(gpr_reg64) 551 addq MACRO_LITERAL(2), REG_VAR(arg_index) 552 addq MACRO_LITERAL(2), REG_VAR(stack_index) 553 jmp 5f 5543: // SKIP_FLOAT 555 addq MACRO_LITERAL(1), REG_VAR(arg_index) 556 addq MACRO_LITERAL(1), REG_VAR(stack_index) 557 jmp 1b 5584: // SKIP_DOUBLE 559 addq MACRO_LITERAL(2), REG_VAR(arg_index) 560 addq MACRO_LITERAL(2), REG_VAR(stack_index) 561 jmp 1b 5625: 563.endm 564 565// Puts the next int/long/object argument in the expected stack slot, 566// fetching values based on a range invoke. 567// Uses rax as temporary. 568.macro LOOP_RANGE_OVER_INTs shorty, arg_index, stack_index, finished 5691: // LOOP 570 movb (REG_VAR(shorty)), %al // al := *shorty 571 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 572 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 573 je VAR(finished) 574 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 575 je 2f 576 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 577 je 3f 578 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 579 je 4f 580 movl (rFP, REG_VAR(arg_index), 4), %eax 581 movl %eax, 8(%rsp, REG_VAR(stack_index), 4) 582 addq MACRO_LITERAL(1), REG_VAR(arg_index) 583 addq MACRO_LITERAL(1), REG_VAR(stack_index) 584 jmp 1b 5852: // FOUND_LONG 586 movq (rFP, REG_VAR(arg_index), 4), %rax 587 movq %rax, 8(%rsp, REG_VAR(stack_index), 4) 588 addq MACRO_LITERAL(2), REG_VAR(arg_index) 589 addq MACRO_LITERAL(2), REG_VAR(stack_index) 590 jmp 1b 5913: // SKIP_FLOAT 592 addq MACRO_LITERAL(1), REG_VAR(arg_index) 593 addq MACRO_LITERAL(1), REG_VAR(stack_index) 594 jmp 1b 5954: // SKIP_DOUBLE 596 addq MACRO_LITERAL(2), REG_VAR(arg_index) 597 addq MACRO_LITERAL(2), REG_VAR(stack_index) 598 jmp 1b 599.endm 600 601// Puts the next floating point parameter passed in physical register 602// in the expected dex register array entry. 603// Uses rax as temporary. 604.macro LOOP_OVER_SHORTY_STORING_XMMS xmm_reg, shorty, arg_index, fp, finished 6051: // LOOP 606 movb (REG_VAR(shorty)), %al // al := *shorty 607 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 608 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 609 je VAR(finished) 610 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 611 je 2f 612 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 613 je 3f 614 addq MACRO_LITERAL(1), REG_VAR(arg_index) 615 // Handle extra argument in arg array taken by a long. 616 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 617 jne 1b 618 addq MACRO_LITERAL(1), REG_VAR(arg_index) 619 jmp 1b // goto LOOP 6202: // FOUND_DOUBLE 621 movsd REG_VAR(xmm_reg),(REG_VAR(fp), REG_VAR(arg_index), 4) 622 addq MACRO_LITERAL(2), REG_VAR(arg_index) 623 jmp 4f 6243: // FOUND_FLOAT 625 movss REG_VAR(xmm_reg), (REG_VAR(fp), REG_VAR(arg_index), 4) 626 addq MACRO_LITERAL(1), REG_VAR(arg_index) 6274: 628.endm 629 630// Puts the next int/long/object parameter passed in physical register 631// in the expected dex register array entry, and in case of object in the 632// expected reference array entry. 633// Uses rax as temporary. 634.macro LOOP_OVER_SHORTY_STORING_GPRS gpr_reg64, gpr_reg32, shorty, arg_index, regs, refs, finished 6351: // LOOP 636 movb (REG_VAR(shorty)), %al // bl := *shorty 637 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 638 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 639 je VAR(finished) 640 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 641 je 2f 642 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 643 je 3f 644 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 645 je 4f 646 movl REG_VAR(gpr_reg32), (REG_VAR(regs), REG_VAR(arg_index), 4) 647 cmpb MACRO_LITERAL(76), %al // if (al != 'L') goto NOT_REFERENCE 648 jne 6f 649 movl REG_VAR(gpr_reg32), (REG_VAR(refs), REG_VAR(arg_index), 4) 6506: // NOT_REFERENCE 651 addq MACRO_LITERAL(1), REG_VAR(arg_index) 652 jmp 5f 6532: // FOUND_LONG 654 movq REG_VAR(gpr_reg64), (REG_VAR(regs), REG_VAR(arg_index), 4) 655 addq MACRO_LITERAL(2), REG_VAR(arg_index) 656 jmp 5f 6573: // SKIP_FLOAT 658 addq MACRO_LITERAL(1), REG_VAR(arg_index) 659 jmp 1b 6604: // SKIP_DOUBLE 661 addq MACRO_LITERAL(2), REG_VAR(arg_index) 662 jmp 1b 6635: 664.endm 665 666// Puts the next floating point parameter passed in stack 667// in the expected dex register array entry. 668// Uses rax as temporary. 669// 670// TODO: Or we could just spill regs to the reserved slots in the caller's 671// frame and copy all regs in a simple loop. This time, however, we would 672// need to look at the shorty anyway to look for the references. 673// (The trade-off is different for passing arguments and receiving them.) 674.macro LOOP_OVER_FPs shorty, arg_index, regs, stack_ptr, finished 6751: // LOOP 676 movb (REG_VAR(shorty)), %al // bl := *shorty 677 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 678 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 679 je VAR(finished) 680 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 681 je 2f 682 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 683 je 3f 684 addq MACRO_LITERAL(1), REG_VAR(arg_index) 685 // Handle extra argument in arg array taken by a long. 686 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 687 jne 1b 688 addq MACRO_LITERAL(1), REG_VAR(arg_index) 689 jmp 1b // goto LOOP 6902: // FOUND_DOUBLE 691 movq OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %rax 692 movq %rax, (REG_VAR(regs), REG_VAR(arg_index), 4) 693 addq MACRO_LITERAL(2), REG_VAR(arg_index) 694 jmp 1b 6953: // FOUND_FLOAT 696 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax 697 movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) 698 addq MACRO_LITERAL(1), REG_VAR(arg_index) 699 jmp 1b 700.endm 701 702// Puts the next int/long/object parameter passed in stack 703// in the expected dex register array entry, and in case of object in the 704// expected reference array entry. 705// Uses rax as temporary. 706.macro LOOP_OVER_INTs shorty, arg_index, regs, refs, stack_ptr, finished 7071: // LOOP 708 movb (REG_VAR(shorty)), %al // bl := *shorty 709 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 710 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 711 je VAR(finished) 712 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 713 je 2f 714 cmpb MACRO_LITERAL(76), %al // if (al == 'L') goto FOUND_REFERENCE 715 je 6f 716 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 717 je 3f 718 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 719 je 4f 720 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax 721 movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) 722 addq MACRO_LITERAL(1), REG_VAR(arg_index) 723 jmp 1b 7246: // FOUND_REFERENCE 725 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax 726 movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) 727 movl %eax, (REG_VAR(refs), REG_VAR(arg_index), 4) 728 addq MACRO_LITERAL(1), REG_VAR(arg_index) 729 jmp 1b 7302: // FOUND_LONG 731 movq OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %rax 732 movq %rax, (REG_VAR(regs), REG_VAR(arg_index), 4) 733 addq MACRO_LITERAL(2), REG_VAR(arg_index) 734 jmp 1b 7353: // SKIP_FLOAT 736 addq MACRO_LITERAL(1), REG_VAR(arg_index) 737 jmp 1b 7384: // SKIP_DOUBLE 739 addq MACRO_LITERAL(2), REG_VAR(arg_index) 740 jmp 1b 741.endm 742 743// Increase method hotness and do suspend check before starting executing the method. 744.macro START_EXECUTING_INSTRUCTIONS 745 movq (%rsp), %rdi 746#if (NTERP_HOTNESS_MASK != 0xffff) 747#error "Nterp x86_64 expects Nterp hotness mask to be 0xffff" 748#endif 749 addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) 750 jo 2f 751 testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET 752 jz 1f 753 EXPORT_PC 754 call SYMBOL(art_quick_test_suspend) 7551: 756 FETCH_INST 757 GOTO_NEXT 7582: 759 movq $$0, %rsi 760 movq rFP, %rdx 761 call nterp_hot_method 762 jmp 1b 763.endm 764 765.macro SPILL_ALL_CALLEE_SAVES 766 PUSH r15 767 PUSH r14 768 PUSH r13 769 PUSH r12 770 PUSH rbp 771 PUSH rbx 772 SETUP_FP_CALLEE_SAVE_FRAME 773.endm 774 775.macro RESTORE_ALL_CALLEE_SAVES 776 RESTORE_FP_CALLEE_SAVE_FRAME 777 POP rbx 778 POP rbp 779 POP r12 780 POP r13 781 POP r14 782 POP r15 783.endm 784 785// Helper to setup the stack after doing a nterp to nterp call. This will setup: 786// - rNEW_FP: the new pointer to dex registers 787// - rNEW_REFS: the new pointer to references 788// - rPC: the new PC pointer to execute 789// - edi: number of arguments 790// - ecx: first dex register 791.macro SETUP_STACK_FOR_INVOKE 792 // We do the same stack overflow check as the compiler. See CanMethodUseNterp 793 // in how we limit the maximum nterp frame size. 794 testq %rax, -STACK_OVERFLOW_RESERVED_BYTES(%rsp) 795 796 // Spill all callee saves to have a consistent stack frame whether we 797 // are called by compiled code or nterp. 798 SPILL_ALL_CALLEE_SAVES 799 800 // Setup the frame. 801 SETUP_STACK_FRAME %rax, rNEW_REFS, rNEW_FP, CFI_NEW_REFS 802 // Make r11 point to the top of the dex register array. 803 leaq (rNEW_FP, %rbx, 4), %r11 804 805 // Fetch instruction information before replacing rPC. 806 movzbl 1(rPC), %edi 807 movzwl 4(rPC), %ecx 808 809 // Set the dex pc pointer. 810 leaq CODE_ITEM_INSNS_OFFSET(%rax), rPC 811 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 812.endm 813 814// Setup arguments based on a non-range nterp to nterp call, and start executing 815// the method. We expect: 816// - rNEW_FP: the new pointer to dex registers 817// - rNEW_REFS: the new pointer to references 818// - rPC: the new PC pointer to execute 819// - edi: number of arguments 820// - ecx: first dex register 821// - r11: top of dex register array 822// - esi: receiver if non-static. 823.macro SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 824 // Now all temporary registers (except r11 containing top of registers array) 825 // are available, copy the parameters. 826 // /* op vA, vB, {vC...vG} */ 827 movl %edi, %eax 828 shrl $$4, %eax # Number of arguments 829 jz 6f # shl sets the Z flag 830 movq MACRO_LITERAL(-1), %r10 831 cmpl MACRO_LITERAL(2), %eax 832 jl 1f 833 je 2f 834 cmpl MACRO_LITERAL(4), %eax 835 jl 3f 836 je 4f 837 838 // We use a decrementing r10 to store references relative 839 // to rNEW_FP and dex registers relative to r11. 840 // 841 // TODO: We could set up r10 as the number of registers (this can be an additional output from 842 // SETUP_STACK_FOR_INVOKE) and then just decrement it by one before copying each arg to 843 // (rNEW_FP, r10, 4) and (rNEW_REFS, r10, 4). 844 // Maybe even introduce macros NEW_VREG_ADDRESS/NEW_VREG_REF_ADDRESS. 8455: 846 andq MACRO_LITERAL(15), %rdi 847 GET_VREG_OBJECT %edx, %rdi 848 movl %edx, (rNEW_FP, %r10, 4) 849 GET_VREG %edx, %rdi 850 movl %edx, (%r11, %r10, 4) 851 subq MACRO_LITERAL(1), %r10 8524: 853 movl %ecx, %eax 854 shrl MACRO_LITERAL(12), %eax 855 GET_VREG_OBJECT %edx, %rax 856 movl %edx, (rNEW_FP, %r10, 4) 857 GET_VREG %edx, %rax 858 movl %edx, (%r11, %r10, 4) 859 subq MACRO_LITERAL(1), %r10 8603: 861 movl %ecx, %eax 862 shrl MACRO_LITERAL(8), %eax 863 andl MACRO_LITERAL(0xf), %eax 864 GET_VREG_OBJECT %edx, %rax 865 movl %edx, (rNEW_FP, %r10, 4) 866 GET_VREG %edx, %rax 867 movl %edx, (%r11, %r10, 4) 868 subq MACRO_LITERAL(1), %r10 8692: 870 movl %ecx, %eax 871 shrl MACRO_LITERAL(4), %eax 872 andl MACRO_LITERAL(0xf), %eax 873 GET_VREG_OBJECT %edx, %rax 874 movl %edx, (rNEW_FP, %r10, 4) 875 GET_VREG %edx, %rax 876 movl %edx, (%r11, %r10, 4) 877 subq MACRO_LITERAL(1), %r10 8781: 879 .if \is_string_init 880 // Ignore the first argument 881 .elseif \is_static 882 movl %ecx, %eax 883 andq MACRO_LITERAL(0x000f), %rax 884 GET_VREG_OBJECT %edx, %rax 885 movl %edx, (rNEW_FP, %r10, 4) 886 GET_VREG %edx, %rax 887 movl %edx, (%r11, %r10, 4) 888 .else 889 movl %esi, (rNEW_FP, %r10, 4) 890 movl %esi, (%r11, %r10, 4) 891 .endif 892 8936: 894 // Start executing the method. 895 movq rNEW_FP, rFP 896 movq rNEW_REFS, rREFS 897 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, ((6 + 4 + 1) * 8) 898 START_EXECUTING_INSTRUCTIONS 899.endm 900 901// Setup arguments based on a range nterp to nterp call, and start executing 902// the method. 903.macro SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 904 // edi is number of arguments 905 // ecx is first register 906 movq MACRO_LITERAL(-4), %r10 907 .if \is_string_init 908 // Ignore the first argument 909 subl $$1, %edi 910 addl $$1, %ecx 911 .elseif !\is_static 912 subl $$1, %edi 913 addl $$1, %ecx 914 .endif 915 916 testl %edi, %edi 917 je 2f 918 leaq (rREFS, %rcx, 4), %rax # pointer to first argument in reference array 919 leaq (%rax, %rdi, 4), %rax # pointer to last argument in reference array 920 leaq (rFP, %rcx, 4), %rcx # pointer to first argument in register array 921 leaq (%rcx, %rdi, 4), %rdi # pointer to last argument in register array 922 // TODO: Same comment for copying arguments as in SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE. 9231: 924 movl -4(%rax), %edx 925 movl %edx, (rNEW_FP, %r10, 1) 926 movl -4(%rdi), %edx 927 movl %edx, (%r11, %r10, 1) 928 subq MACRO_LITERAL(4), %r10 929 subq MACRO_LITERAL(4), %rax 930 subq MACRO_LITERAL(4), %rdi 931 cmpq %rcx, %rdi 932 jne 1b 933 9342: 935 .if \is_string_init 936 // Ignore first argument 937 .elseif !\is_static 938 movl %esi, (rNEW_FP, %r10, 1) 939 movl %esi, (%r11, %r10, 1) 940 .endif 941 movq rNEW_FP, rFP 942 movq rNEW_REFS, rREFS 943 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, ((6 + 4 + 1) * 8) 944 START_EXECUTING_INSTRUCTIONS 945.endm 946 947.macro GET_SHORTY dest, is_interface, is_polymorphic, is_custom 948 push %rdi 949 push %rsi 950 .if \is_polymorphic 951 movq 16(%rsp), %rdi 952 movq rPC, %rsi 953 call SYMBOL(NterpGetShortyFromInvokePolymorphic) 954 .elseif \is_custom 955 movq 16(%rsp), %rdi 956 movq rPC, %rsi 957 call SYMBOL(NterpGetShortyFromInvokeCustom) 958 .elseif \is_interface 959 movq 16(%rsp), %rdi 960 movzwl 2(rPC), %esi 961 call SYMBOL(NterpGetShortyFromMethodId) 962 .else 963 call SYMBOL(NterpGetShorty) 964 .endif 965 pop %rsi 966 pop %rdi 967 movq %rax, \dest 968.endm 969 970.macro DO_ENTRY_POINT_CHECK call_compiled_code 971 // On entry, the method is %rdi, the instance is %rsi 972 leaq ExecuteNterpImpl(%rip), %rax 973 cmpq %rax, ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) 974 jne VAR(call_compiled_code) 975 976 // TODO: Get code item in a better way and remove below 977 push %rdi 978 push %rsi 979 call SYMBOL(NterpGetCodeItem) 980 pop %rsi 981 pop %rdi 982 // TODO: Get code item in a better way and remove above 983.endm 984 985// Uses r9 and r10 as temporary 986.macro UPDATE_REGISTERS_FOR_STRING_INIT old_value, new_value 987 movq rREFS, %r9 988 movq rFP, %r10 9891: 990 cmpl (%r9), \old_value 991 jne 2f 992 movl \new_value, (%r9) 993 movl \new_value, (%r10) 9942: 995 addq $$4, %r9 996 addq $$4, %r10 997 cmpq %r9, rFP 998 jne 1b 999.endm 1000 1001.macro COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 1002 .if \is_polymorphic 1003 // We always go to compiled code for polymorphic calls. 1004 .elseif \is_custom 1005 // We always go to compiled code for custom calls. 1006 .else 1007 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_\suffix 1008 .if \is_string_init 1009 call nterp_to_nterp_string_init_non_range 1010 .elseif \is_static 1011 call nterp_to_nterp_static_non_range 1012 .else 1013 call nterp_to_nterp_instance_non_range 1014 .endif 1015 jmp .Ldone_return_\suffix 1016 .endif 1017 1018.Lcall_compiled_code_\suffix: 1019 GET_SHORTY rINSTq, \is_interface, \is_polymorphic, \is_custom 1020 // From this point: 1021 // - rISNTq contains shorty (in callee-save to switch over return value after call). 1022 // - rdi contains method 1023 // - rsi contains 'this' pointer for instance method. 1024 leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character 1025 movzwl 4(rPC), %r11d // arguments 1026 .if \is_string_init 1027 shrq MACRO_LITERAL(4), %r11 1028 movq $$1, %r10 // ignore first argument 1029 .elseif \is_static 1030 movq $$0, %r10 // arg_index 1031 .else 1032 shrq MACRO_LITERAL(4), %r11 1033 movq $$1, %r10 // arg_index 1034 .endif 1035 LOOP_OVER_SHORTY_LOADING_XMMS xmm0, r11, r9, r10, .Lxmm_setup_finished_\suffix 1036 LOOP_OVER_SHORTY_LOADING_XMMS xmm1, r11, r9, r10, .Lxmm_setup_finished_\suffix 1037 LOOP_OVER_SHORTY_LOADING_XMMS xmm2, r11, r9, r10, .Lxmm_setup_finished_\suffix 1038 LOOP_OVER_SHORTY_LOADING_XMMS xmm3, r11, r9, r10, .Lxmm_setup_finished_\suffix 1039 LOOP_OVER_SHORTY_LOADING_XMMS xmm4, r11, r9, r10, .Lxmm_setup_finished_\suffix 1040.Lxmm_setup_finished_\suffix: 1041 leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character 1042 movzwl 4(rPC), %r11d // arguments 1043 .if \is_string_init 1044 movq $$1, %r10 // ignore first argument 1045 shrq MACRO_LITERAL(4), %r11 1046 LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r9, r10, .Lgpr_setup_finished_\suffix 1047 .elseif \is_static 1048 movq $$0, %r10 // arg_index 1049 LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r9, r10, .Lgpr_setup_finished_\suffix 1050 .else 1051 shrq MACRO_LITERAL(4), %r11 1052 movq $$1, %r10 // arg_index 1053 .endif 1054 LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, r11, r9, r10, .Lgpr_setup_finished_\suffix 1055 LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, r11, r9, r10, .Lgpr_setup_finished_\suffix 1056 LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, r11, r9, r10, .Lgpr_setup_finished_\suffix 1057 LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, r11, r9, r10, .Lgpr_setup_finished_\suffix 1058.Lgpr_setup_finished_\suffix: 1059 .if \is_polymorphic 1060 call SYMBOL(art_quick_invoke_polymorphic) 1061 .elseif \is_custom 1062 call SYMBOL(art_quick_invoke_custom) 1063 .else 1064 .if \is_interface 1065 movzwl 2(rPC), %eax 1066 .endif 1067 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1068 .endif 1069 cmpb LITERAL(68), (rINSTq) // Test if result type char == 'D'. 1070 je .Lreturn_double_\suffix 1071 cmpb LITERAL(70), (rINSTq) // Test if result type char == 'F'. 1072 jne .Ldone_return_\suffix 1073.Lreturn_float_\suffix: 1074 movd %xmm0, %eax 1075 jmp .Ldone_return_\suffix 1076.Lreturn_double_\suffix: 1077 movq %xmm0, %rax 1078.Ldone_return_\suffix: 1079 /* resume execution of caller */ 1080 .if \is_string_init 1081 movzwl 4(rPC), %r11d // arguments 1082 andq $$0xf, %r11 1083 GET_VREG %esi, %r11 1084 UPDATE_REGISTERS_FOR_STRING_INIT %esi, %eax 1085 .endif 1086 1087 .if \is_polymorphic 1088 ADVANCE_PC_FETCH_AND_GOTO_NEXT 4 1089 .else 1090 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1091 .endif 1092.endm 1093 1094.macro COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 1095 .if \is_polymorphic 1096 // We always go to compiled code for polymorphic calls. 1097 .elseif \is_custom 1098 // We always go to compiled code for custom calls. 1099 .else 1100 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_range_\suffix 1101 .if \is_string_init 1102 call nterp_to_nterp_string_init_range 1103 .elseif \is_static 1104 call nterp_to_nterp_static_range 1105 .else 1106 call nterp_to_nterp_instance_range 1107 .endif 1108 jmp .Ldone_return_range_\suffix 1109 .endif 1110 1111.Lcall_compiled_code_range_\suffix: 1112 GET_SHORTY rINSTq, \is_interface, \is_polymorphic, \is_custom 1113 // From this point: 1114 // - rINSTq contains shorty (in callee-save to switch over return value after call). 1115 // - rdi contains method 1116 // - rsi contains 'this' pointer for instance method. 1117 leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character 1118 movzwl 4(rPC), %r10d // arg start index 1119 .if \is_string_init 1120 addq $$1, %r10 // arg start index 1121 movq $$1, %rbp // index in stack 1122 .elseif \is_static 1123 movq $$0, %rbp // index in stack 1124 .else 1125 addq $$1, %r10 // arg start index 1126 movq $$1, %rbp // index in stack 1127 .endif 1128 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm0, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1129 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm1, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1130 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm2, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1131 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm3, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1132 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm4, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1133 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm5, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1134 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm6, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1135 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm7, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1136 LOOP_RANGE_OVER_FPs r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1137.Lxmm_setup_finished_range_\suffix: 1138 leaq 1(%rbx), %r11 // shorty + 1 ; ie skip return arg character 1139 movzwl 4(rPC), %r10d // arg start index 1140 .if \is_string_init 1141 addq $$1, %r10 // arg start index 1142 movq $$1, %rbp // index in stack 1143 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r10, rbp, .Lgpr_setup_finished_\suffix 1144 .elseif \is_static 1145 movq $$0, %rbp // index in stack 1146 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r10, rbp, .Lgpr_setup_finished_\suffix 1147 .else 1148 addq $$1, %r10 // arg start index 1149 movq $$1, %rbp // index in stack 1150 .endif 1151 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rdx, edx, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1152 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rcx, ecx, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1153 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS r8, r8d, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1154 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS r9, r9d, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1155 LOOP_RANGE_OVER_INTs r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1156 1157.Lgpr_setup_finished_range_\suffix: 1158 .if \is_polymorphic 1159 call SYMBOL(art_quick_invoke_polymorphic) 1160 .elseif \is_custom 1161 call SYMBOL(art_quick_invoke_custom) 1162 .else 1163 .if \is_interface 1164 movzwl 2(rPC), %eax 1165 .endif 1166 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1167 .endif 1168 cmpb LITERAL(68), (%rbx) // Test if result type char == 'D'. 1169 je .Lreturn_range_double_\suffix 1170 cmpb LITERAL(70), (%rbx) // Test if result type char == 'F'. 1171 je .Lreturn_range_float_\suffix 1172 /* resume execution of caller */ 1173.Ldone_return_range_\suffix: 1174 .if \is_string_init 1175 movzwl 4(rPC), %r11d // arguments 1176 GET_VREG %esi, %r11 1177 UPDATE_REGISTERS_FOR_STRING_INIT %esi, %eax 1178 .endif 1179 1180 .if \is_polymorphic 1181 ADVANCE_PC_FETCH_AND_GOTO_NEXT 4 1182 .else 1183 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1184 .endif 1185.Lreturn_range_double_\suffix: 1186 movq %xmm0, %rax 1187 jmp .Ldone_return_range_\suffix 1188.Lreturn_range_float_\suffix: 1189 movd %xmm0, %eax 1190 jmp .Ldone_return_range_\suffix 1191.endm 1192 1193// Fetch some information from the thread cache. 1194// Uses rax, rdx, rcx as temporaries. 1195.macro FETCH_FROM_THREAD_CACHE dest_reg, slow_path 1196 movq rSELF:THREAD_SELF_OFFSET, %rax 1197 movq rPC, %rdx 1198 salq MACRO_LITERAL(THREAD_INTERPRETER_CACHE_SIZE_SHIFT), %rdx 1199 andq MACRO_LITERAL(THREAD_INTERPRETER_CACHE_SIZE_MASK), %rdx 1200 cmpq THREAD_INTERPRETER_CACHE_OFFSET(%rax, %rdx, 1), rPC 1201 jne \slow_path 1202 movq __SIZEOF_POINTER__+THREAD_INTERPRETER_CACHE_OFFSET(%rax, %rdx, 1), \dest_reg 1203.endm 1204 1205// Helper for static field get. 1206.macro OP_SGET load="movl", wide="0" 1207 // Fast-path which gets the field from thread-local cache. 1208 FETCH_FROM_THREAD_CACHE %rax, 2f 12091: 1210 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1211 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1212 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1213 jne 3f 12144: 1215 .if \wide 1216 movq (%eax,%edx,1), %rax 1217 SET_WIDE_VREG %rax, rINSTq # fp[A] <- value 1218 .else 1219 \load (%eax, %edx, 1), %eax 1220 SET_VREG %eax, rINSTq # fp[A] <- value 1221 .endif 1222 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 12232: 1224 movq rSELF:THREAD_SELF_OFFSET, %rdi 1225 movq 0(%rsp), %rsi 1226 movq rPC, %rdx 1227 EXPORT_PC 1228 call nterp_get_static_field 1229 // Clear the marker that we put for volatile fields. The x86 memory 1230 // model doesn't require a barrier. 1231 andq $$-2, %rax 1232 jmp 1b 12333: 1234 call art_quick_read_barrier_mark_reg00 1235 jmp 4b 1236.endm 1237 1238// Helper for static field put. 1239.macro OP_SPUT rINST_reg="rINST", store="movl", wide="0": 1240 // Fast-path which gets the field from thread-local cache. 1241 FETCH_FROM_THREAD_CACHE %rax, 2f 12421: 1243 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1244 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1245 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1246 jne 3f 12474: 1248 .if \wide 1249 GET_WIDE_VREG rINSTq, rINSTq # rINST <- v[A] 1250 .else 1251 GET_VREG rINST, rINSTq # rINST <- v[A] 1252 .endif 1253 \store \rINST_reg, (%rax,%rdx,1) 1254 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 12552: 1256 movq rSELF:THREAD_SELF_OFFSET, %rdi 1257 movq 0(%rsp), %rsi 1258 movq rPC, %rdx 1259 EXPORT_PC 1260 call nterp_get_static_field 1261 testq MACRO_LITERAL(1), %rax 1262 je 1b 1263 // Clear the marker that we put for volatile fields. The x86 memory 1264 // model doesn't require a barrier. 1265 CLEAR_VOLATILE_MARKER %rax 1266 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1267 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1268 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1269 jne 6f 12705: 1271 .if \wide 1272 GET_WIDE_VREG rINSTq, rINSTq # rINST <- v[A] 1273 .else 1274 GET_VREG rINST, rINSTq # rINST <- v[A] 1275 .endif 1276 \store \rINST_reg, (%rax,%rdx,1) 1277 lock addl $$0, (%rsp) 1278 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 12793: 1280 call art_quick_read_barrier_mark_reg00 1281 jmp 4b 12826: 1283 call art_quick_read_barrier_mark_reg00 1284 jmp 5b 1285.endm 1286 1287 1288.macro OP_IPUT_INTERNAL rINST_reg="rINST", store="movl", wide="0": 1289 movzbq rINSTbl, %rcx # rcx <- BA 1290 sarl $$4, %ecx # ecx <- B 1291 GET_VREG %ecx, %rcx # vB (object we're operating on) 1292 testl %ecx, %ecx # is object null? 1293 je common_errNullObject 1294 andb $$0xf, rINSTbl # rINST <- A 1295 .if \wide 1296 GET_WIDE_VREG rINSTq, rINSTq # rax<- fp[A]/fp[A+1] 1297 .else 1298 GET_VREG rINST, rINSTq # rINST <- v[A] 1299 .endif 1300 \store \rINST_reg, (%rcx,%rax,1) 1301.endm 1302 1303// Helper for instance field put. 1304.macro OP_IPUT rINST_reg="rINST", store="movl", wide="0": 1305 // Fast-path which gets the field from thread-local cache. 1306 FETCH_FROM_THREAD_CACHE %rax, 2f 13071: 1308 OP_IPUT_INTERNAL \rINST_reg, \store, \wide 1309 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 13102: 1311 movq rSELF:THREAD_SELF_OFFSET, %rdi 1312 movq 0(%rsp), %rsi 1313 movq rPC, %rdx 1314 EXPORT_PC 1315 call nterp_get_instance_field_offset 1316 testl %eax, %eax 1317 jns 1b 1318 negl %eax 1319 OP_IPUT_INTERNAL \rINST_reg, \store, \wide 1320 lock addl $$0, (%rsp) 1321 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 1322.endm 1323 1324// Helper for instance field get. 1325.macro OP_IGET load="movl", wide="0" 1326 // Fast-path which gets the field from thread-local cache. 1327 FETCH_FROM_THREAD_CACHE %rax, 2f 13281: 1329 movl rINST, %ecx # rcx <- BA 1330 sarl $$4, %ecx # ecx <- B 1331 GET_VREG %ecx, %rcx # vB (object we're operating on) 1332 testl %ecx, %ecx # is object null? 1333 je common_errNullObject 1334 andb $$0xf,rINSTbl # rINST <- A 1335 .if \wide 1336 movq (%rcx,%rax,1), %rax 1337 SET_WIDE_VREG %rax, rINSTq # fp[A] <- value 1338 .else 1339 \load (%rcx,%rax,1), %eax 1340 SET_VREG %eax, rINSTq # fp[A] <- value 1341 .endif 1342 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 13432: 1344 movq rSELF:THREAD_SELF_OFFSET, %rdi 1345 movq 0(%rsp), %rsi 1346 movq rPC, %rdx 1347 EXPORT_PC 1348 call nterp_get_instance_field_offset 1349 testl %eax, %eax 1350 jns 1b 1351 negl %eax 1352 jmp 1b 1353.endm 1354 1355%def entry(): 1356/* 1357 * ArtMethod entry point. 1358 * 1359 * On entry: 1360 * rdi ArtMethod* callee 1361 * rest method parameters 1362 */ 1363 1364OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl 1365 .cfi_startproc 1366 .cfi_def_cfa rsp, 8 1367 testq %rax, -STACK_OVERFLOW_RESERVED_BYTES(%rsp) 1368 /* Spill callee save regs */ 1369 SPILL_ALL_CALLEE_SAVES 1370 1371 // TODO: Get shorty in a better way and remove below 1372 PUSH rdi 1373 PUSH rsi 1374 PUSH rdx 1375 PUSH rcx 1376 PUSH r8 1377 PUSH r9 1378 1379 // Save xmm registers + alignment. 1380 subq MACRO_LITERAL(8 * 8 + 8), %rsp 1381 CFI_ADJUST_CFA_OFFSET(8 * 8 + 8) 1382 movq %xmm0, 0(%rsp) 1383 movq %xmm1, 8(%rsp) 1384 movq %xmm2, 16(%rsp) 1385 movq %xmm3, 24(%rsp) 1386 movq %xmm4, 32(%rsp) 1387 movq %xmm5, 40(%rsp) 1388 movq %xmm6, 48(%rsp) 1389 movq %xmm7, 56(%rsp) 1390 1391 // Save method in callee-save rbx. 1392 movq %rdi, %rbx 1393 call SYMBOL(NterpGetShorty) 1394 // Save shorty in callee-save rbp. 1395 movq %rax, %rbp 1396 movq %rbx, %rdi 1397 call SYMBOL(NterpGetCodeItem) 1398 movq %rax, rPC 1399 1400 // Restore xmm registers + alignment. 1401 movq 0(%rsp), %xmm0 1402 movq 8(%rsp), %xmm1 1403 movq 16(%rsp), %xmm2 1404 movq 24(%rsp), %xmm3 1405 movq 32(%rsp), %xmm4 1406 movq 40(%rsp), %xmm5 1407 movq 48(%rsp), %xmm6 1408 movq 56(%rsp), %xmm7 1409 addq MACRO_LITERAL(8 * 8 + 8), %rsp 1410 CFI_ADJUST_CFA_OFFSET(-8 * 8 - 8) 1411 1412 POP r9 1413 POP r8 1414 POP rcx 1415 POP rdx 1416 POP rsi 1417 POP rdi 1418 // TODO: Get shorty in a better way and remove above 1419 1420 // Setup the stack for executing the method. 1421 SETUP_STACK_FRAME rPC, rREFS, rFP, CFI_REFS 1422 1423 // Setup the parameters 1424 movzwl CODE_ITEM_INS_SIZE_OFFSET(rPC), %r14d 1425 testl %r14d, %r14d 1426 je .Lxmm_setup_finished 1427 1428 subq %r14, %rbx 1429 salq $$2, %rbx // rbx is now the offset for inputs into the registers array. 1430 1431 testl $$ART_METHOD_IS_STATIC_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1432 1433 // Available: rdi, r10, r14 1434 // Note the leaq below don't change the flags. 1435 leaq 1(%rbp), %r10 // shorty + 1 ; ie skip return arg character 1436 leaq (rFP, %rbx, 1), %rdi 1437 leaq (rREFS, %rbx, 1), %rbx 1438 jne .Lhandle_static_method 1439 movl %esi, (%rdi) 1440 movl %esi, (%rbx) 1441 addq $$4, %rdi 1442 addq $$4, %rbx 1443 addq $$4, %r11 1444 movq $$0, %r14 1445 jmp .Lcontinue_setup_gprs 1446.Lhandle_static_method: 1447 movq $$0, %r14 1448 LOOP_OVER_SHORTY_STORING_GPRS rsi, esi, r10, r14, rdi, rbx, .Lgpr_setup_finished 1449.Lcontinue_setup_gprs: 1450 LOOP_OVER_SHORTY_STORING_GPRS rdx, edx, r10, r14, rdi, rbx, .Lgpr_setup_finished 1451 LOOP_OVER_SHORTY_STORING_GPRS rcx, ecx, r10, r14, rdi, rbx, .Lgpr_setup_finished 1452 LOOP_OVER_SHORTY_STORING_GPRS r8, r8d, r10, r14, rdi, rbx, .Lgpr_setup_finished 1453 LOOP_OVER_SHORTY_STORING_GPRS r9, r9d, r10, r14, rdi, rbx, .Lgpr_setup_finished 1454 LOOP_OVER_INTs r10, r14, rdi, rbx, r11, .Lgpr_setup_finished 1455.Lgpr_setup_finished: 1456 leaq 1(%rbp), %r10 // shorty + 1 ; ie skip return arg character 1457 movq $$0, %r14 // reset counter 1458 LOOP_OVER_SHORTY_STORING_XMMS xmm0, r10, r14, rdi, .Lxmm_setup_finished 1459 LOOP_OVER_SHORTY_STORING_XMMS xmm1, r10, r14, rdi, .Lxmm_setup_finished 1460 LOOP_OVER_SHORTY_STORING_XMMS xmm2, r10, r14, rdi, .Lxmm_setup_finished 1461 LOOP_OVER_SHORTY_STORING_XMMS xmm3, r10, r14, rdi, .Lxmm_setup_finished 1462 LOOP_OVER_SHORTY_STORING_XMMS xmm4, r10, r14, rdi, .Lxmm_setup_finished 1463 LOOP_OVER_SHORTY_STORING_XMMS xmm5, r10, r14, rdi, .Lxmm_setup_finished 1464 LOOP_OVER_SHORTY_STORING_XMMS xmm6, r10, r14, rdi, .Lxmm_setup_finished 1465 LOOP_OVER_SHORTY_STORING_XMMS xmm7, r10, r14, rdi, .Lxmm_setup_finished 1466 LOOP_OVER_FPs r10, r14, rdi, r11, .Lxmm_setup_finished 1467.Lxmm_setup_finished: 1468 // Set the dex pc pointer. 1469 addq $$CODE_ITEM_INSNS_OFFSET, rPC 1470 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 1471 1472 // Set rIBASE 1473 leaq artNterpAsmInstructionStart(%rip), rIBASE 1474 /* start executing the instruction at rPC */ 1475 START_EXECUTING_INSTRUCTIONS 1476 /* NOTE: no fallthrough */ 1477 // cfi info continues, and covers the whole nterp implementation. 1478 END ExecuteNterpImpl 1479 1480%def opcode_pre(): 1481 1482%def helpers(): 1483 1484%def footer(): 1485/* 1486 * =========================================================================== 1487 * Common subroutines and data 1488 * =========================================================================== 1489 */ 1490 1491 .text 1492 .align 2 1493 1494// Note: mterp also uses the common_* names below for helpers, but that's OK 1495// as the C compiler compiled each interpreter separately. 1496common_errDivideByZero: 1497 EXPORT_PC 1498 call art_quick_throw_div_zero 1499 1500common_errArrayIndex: 1501 EXPORT_PC 1502 movl MIRROR_ARRAY_LENGTH_OFFSET(%edi), %eax 1503 movl %esi, %edi 1504 movl %eax, %esi 1505 call art_quick_throw_array_bounds 1506 1507common_errNullObject: 1508 EXPORT_PC 1509 call art_quick_throw_null_pointer_exception 1510 1511NterpCommonInvokeStatic: 1512 COMMON_INVOKE_NON_RANGE is_static=1, is_interface=0, suffix="invokeStatic" 1513 1514NterpCommonInvokeStaticRange: 1515 COMMON_INVOKE_RANGE is_static=1, is_interface=0, suffix="invokeStatic" 1516 1517NterpCommonInvokeInstance: 1518 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="invokeInstance" 1519 1520NterpCommonInvokeInstanceRange: 1521 COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="invokeInstance" 1522 1523NterpCommonInvokeInterface: 1524 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=1, suffix="invokeInterface" 1525 1526NterpCommonInvokeInterfaceRange: 1527 COMMON_INVOKE_RANGE is_static=0, is_interface=1, suffix="invokeInterface" 1528 1529NterpCommonInvokePolymorphic: 1530 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, is_string_init=0, is_polymorphic=1, suffix="invokePolymorphic" 1531 1532NterpCommonInvokePolymorphicRange: 1533 COMMON_INVOKE_RANGE is_static=0, is_interface=0, is_polymorphic=1, suffix="invokePolymorphic" 1534 1535NterpCommonInvokeCustom: 1536 COMMON_INVOKE_NON_RANGE is_static=1, is_interface=0, is_string_init=0, is_polymorphic=0, is_custom=1, suffix="invokeCustom" 1537 1538NterpCommonInvokeCustomRange: 1539 COMMON_INVOKE_RANGE is_static=1, is_interface=0, is_polymorphic=0, is_custom=1, suffix="invokeCustom" 1540 1541NterpHandleStringInit: 1542 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, is_string_init=1, suffix="stringInit" 1543 1544NterpHandleStringInitRange: 1545 COMMON_INVOKE_RANGE is_static=0, is_interface=0, is_string_init=1, suffix="stringInit" 1546 1547NterpNewInstance: 1548 EXPORT_PC 1549 // Fast-path which gets the class from thread-local cache. 1550 FETCH_FROM_THREAD_CACHE %rdi, 2f 1551 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1552 jne 3f 15534: 1554 callq *rSELF:THREAD_ALLOC_OBJECT_ENTRYPOINT_OFFSET 15551: 1556 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1557 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 15582: 1559 movq rSELF:THREAD_SELF_OFFSET, %rdi 1560 movq 0(%rsp), %rsi 1561 movq rPC, %rdx 1562 call nterp_get_class_or_allocate_object 1563 jmp 1b 15643: 1565 // 07 is %rdi 1566 call art_quick_read_barrier_mark_reg07 1567 jmp 4b 1568 1569NterpNewArray: 1570 /* new-array vA, vB, class@CCCC */ 1571 EXPORT_PC 1572 // Fast-path which gets the class from thread-local cache. 1573 FETCH_FROM_THREAD_CACHE %rdi, 2f 1574 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1575 jne 3f 15761: 1577 movzbl rINSTbl,%esi 1578 sarl $$4,%esi # esi<- B 1579 GET_VREG %esi %rsi # esi<- vB (array length) 1580 andb $$0xf,rINSTbl # rINST<- A 1581 callq *rSELF:THREAD_ALLOC_ARRAY_ENTRYPOINT_OFFSET 1582 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1583 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 15842: 1585 movq rSELF:THREAD_SELF_OFFSET, %rdi 1586 movq 0(%rsp), %rsi 1587 movq rPC, %rdx 1588 call nterp_get_class_or_allocate_object 1589 movq %rax, %rdi 1590 jmp 1b 15913: 1592 // 07 is %rdi 1593 call art_quick_read_barrier_mark_reg07 1594 jmp 1b 1595 1596NterpPutObjectInstanceField: 1597 // Fast-path which gets the field from thread-local cache. 1598 FETCH_FROM_THREAD_CACHE %rax, 2f 15991: 1600 movzbq rINSTbl, %rcx # rcx <- BA 1601 sarl $$4, %ecx # ecx <- B 1602 GET_VREG %ecx, %rcx # vB (object we're operating on) 1603 testl %ecx, %ecx # is object null? 1604 je common_errNullObject 1605 andb $$0xf, rINSTbl # rINST <- A 1606 GET_VREG rINST, rINSTq # rINST <- v[A] 1607 movl rINST, (%rcx,%rax,1) 1608 testl rINST, rINST 1609 je 4f 1610 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rax 1611 shrq $$CARD_TABLE_CARD_SHIFT, %rcx 1612 movb %al, (%rax, %rcx, 1) 16134: 1614 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 16152: 1616 EXPORT_PC 1617 movq rSELF:THREAD_SELF_OFFSET, %rdi 1618 movq 0(%rsp), %rsi 1619 movq rPC, %rdx 1620 EXPORT_PC 1621 call nterp_get_instance_field_offset 1622 testl %eax, %eax 1623 jns 1b 1624 negl %eax 1625 movzbq rINSTbl, %rcx # rcx <- BA 1626 sarl $$4, %ecx # ecx <- B 1627 GET_VREG %ecx, %rcx # vB (object we're operating on) 1628 testl %ecx, %ecx # is object null? 1629 je common_errNullObject 1630 andb $$0xf, rINSTbl # rINST <- A 1631 GET_VREG rINST, rINSTq # rINST <- v[A] 1632 movl rINST, (%rcx,%rax,1) 1633 testl rINST, rINST 1634 je 5f 1635 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rax 1636 shrq $$CARD_TABLE_CARD_SHIFT, %rcx 1637 movb %al, (%rcx, %rax, 1) 16385: 1639 lock addl $$0, (%rsp) 1640 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 1641 1642NterpGetObjectInstanceField: 1643 // Fast-path which gets the field from thread-local cache. 1644 FETCH_FROM_THREAD_CACHE %rax, 2f 16451: 1646 movl rINST, %ecx # rcx <- BA 1647 sarl $$4, %ecx # ecx <- B 1648 GET_VREG %ecx, %rcx # vB (object we're operating on) 1649 testl %ecx, %ecx # is object null? 1650 je common_errNullObject 1651 testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%ecx) 1652 movl (%rcx,%rax,1), %eax 1653 jnz 3f 16544: 1655 andb $$0xf,rINSTbl # rINST <- A 1656 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1657 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 16582: 1659 EXPORT_PC 1660 movq rSELF:THREAD_SELF_OFFSET, %rdi 1661 movq 0(%rsp), %rsi 1662 movq rPC, %rdx 1663 EXPORT_PC 1664 call nterp_get_instance_field_offset 1665 testl %eax, %eax 1666 jns 1b 1667 // For volatile fields, we return a negative offset. Remove the sign 1668 // and no need for any barrier thanks to the memory model. 1669 negl %eax 1670 jmp 1b 16713: 1672 // reg00 is eax 1673 call art_quick_read_barrier_mark_reg00 1674 jmp 4b 1675 1676NterpPutObjectStaticField: 1677 // Fast-path which gets the field from thread-local cache. 1678 FETCH_FROM_THREAD_CACHE %rax, 2f 16791: 1680 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1681 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1682 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1683 jne 3f 16845: 1685 GET_VREG %ecx, rINSTq 1686 movl %ecx, (%eax, %edx, 1) 1687 testl %ecx, %ecx 1688 je 4f 1689 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rcx 1690 shrq $$CARD_TABLE_CARD_SHIFT, %rax 1691 movb %cl, (%rax, %rcx, 1) 16924: 1693 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 16942: 1695 movq rSELF:THREAD_SELF_OFFSET, %rdi 1696 movq 0(%rsp), %rsi 1697 movq rPC, %rdx 1698 EXPORT_PC 1699 call nterp_get_static_field 1700 testq MACRO_LITERAL(1), %rax 1701 je 1b 1702 CLEAR_VOLATILE_MARKER %rax 1703 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1704 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1705 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1706 jne 7f 17076: 1708 movzbl rINSTbl, %ecx 1709 GET_VREG %ecx, %rcx 1710 movl %ecx, (%eax, %edx, 1) 1711 testl %ecx, %ecx 1712 je 8f 1713 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rcx 1714 shrq $$CARD_TABLE_CARD_SHIFT, %rax 1715 movb %cl, (%rax, %rcx, 1) 17168: 1717 lock addl $$0, (%rsp) 1718 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 17193: 1720 call art_quick_read_barrier_mark_reg00 1721 jmp 5b 17227: 1723 call art_quick_read_barrier_mark_reg00 1724 jmp 6b 1725 1726NterpGetObjectStaticField: 1727 // Fast-path which gets the field from thread-local cache. 1728 FETCH_FROM_THREAD_CACHE %rax, 2f 17291: 1730 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1731 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1732 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1733 jne 5f 17346: 1735 testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax) 1736 movl (%eax, %edx, 1), %eax 1737 jnz 3f 17384: 1739 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1740 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 17412: 1742 movq rSELF:THREAD_SELF_OFFSET, %rdi 1743 movq 0(%rsp), %rsi 1744 movq rPC, %rdx 1745 EXPORT_PC 1746 call nterp_get_static_field 1747 andq $$-2, %rax 1748 jmp 1b 17493: 1750 call art_quick_read_barrier_mark_reg00 1751 jmp 4b 17525: 1753 call art_quick_read_barrier_mark_reg00 1754 jmp 6b 1755 1756NterpGetBooleanStaticField: 1757 OP_SGET load="movsbl", wide=0 1758 1759NterpGetByteStaticField: 1760 OP_SGET load="movsbl", wide=0 1761 1762NterpGetCharStaticField: 1763 OP_SGET load="movzwl", wide=0 1764 1765NterpGetShortStaticField: 1766 OP_SGET load="movswl", wide=0 1767 1768NterpGetWideStaticField: 1769 OP_SGET load="movq", wide=1 1770 1771NterpGetIntStaticField: 1772 OP_SGET load="movl", wide=0 1773 1774NterpPutStaticField: 1775 OP_SPUT rINST_reg=rINST, store="movl", wide=0 1776 1777NterpPutBooleanStaticField: 1778NterpPutByteStaticField: 1779 OP_SPUT rINST_reg=rINSTbl, store="movb", wide=0 1780 1781NterpPutCharStaticField: 1782NterpPutShortStaticField: 1783 OP_SPUT rINST_reg=rINSTw, store="movw", wide=0 1784 1785NterpPutWideStaticField: 1786 OP_SPUT rINST_reg=rINSTq, store="movq", wide=1 1787 1788NterpPutInstanceField: 1789 OP_IPUT rINST_reg=rINST, store="movl", wide=0 1790 1791NterpPutBooleanInstanceField: 1792NterpPutByteInstanceField: 1793 OP_IPUT rINST_reg=rINSTbl, store="movb", wide=0 1794 1795NterpPutCharInstanceField: 1796NterpPutShortInstanceField: 1797 OP_IPUT rINST_reg=rINSTw, store="movw", wide=0 1798 1799NterpPutWideInstanceField: 1800 OP_IPUT rINST_reg=rINSTq, store="movq", wide=1 1801 1802NterpGetBooleanInstanceField: 1803 OP_IGET load="movzbl", wide=0 1804 1805NterpGetByteInstanceField: 1806 OP_IGET load="movsbl", wide=0 1807 1808NterpGetCharInstanceField: 1809 OP_IGET load="movzwl", wide=0 1810 1811NterpGetShortInstanceField: 1812 OP_IGET load="movswl", wide=0 1813 1814NterpGetWideInstanceField: 1815 OP_IGET load="movq", wide=1 1816 1817NterpGetInstanceField: 1818 OP_IGET load="movl", wide=0 1819 1820NterpInstanceOf: 1821 /* instance-of vA, vB, class@CCCC */ 1822 // Fast-path which gets the class from thread-local cache. 1823 EXPORT_PC 1824 FETCH_FROM_THREAD_CACHE %rsi, 2f 1825 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1826 jne 5f 18271: 1828 movzbl rINSTbl,%edi 1829 sarl $$4,%edi # edi<- B 1830 GET_VREG %edi %rdi # edi<- vB (object) 1831 andb $$0xf,rINSTbl # rINST<- A 1832 testl %edi, %edi 1833 je 3f 1834 call art_quick_instance_of 1835 SET_VREG %eax, rINSTq # fp[A] <- value 18364: 1837 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 18383: 1839 SET_VREG %edi, rINSTq # fp[A] <-0 1840 jmp 4b 18412: 1842 movq rSELF:THREAD_SELF_OFFSET, %rdi 1843 movq 0(%rsp), %rsi 1844 movq rPC, %rdx 1845 call nterp_get_class_or_allocate_object 1846 movq %rax, %rsi 1847 jmp 1b 18485: 1849 // 06 is %rsi 1850 call art_quick_read_barrier_mark_reg06 1851 jmp 1b 1852 1853NterpCheckCast: 1854 // Fast-path which gets the class from thread-local cache. 1855 EXPORT_PC 1856 FETCH_FROM_THREAD_CACHE %rsi, 3f 1857 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1858 jne 4f 18591: 1860 GET_VREG %edi, rINSTq 1861 testl %edi, %edi 1862 je 2f 1863 call art_quick_check_instance_of 18642: 1865 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 18663: 1867 movq rSELF:THREAD_SELF_OFFSET, %rdi 1868 movq 0(%rsp), %rsi 1869 movq rPC, %rdx 1870 call nterp_get_class_or_allocate_object 1871 movq %rax, %rsi 1872 jmp 1b 18734: 1874 // 06 is %rsi 1875 call art_quick_read_barrier_mark_reg06 1876 jmp 1b 1877 1878NterpHandleHotnessOverflow: 1879 leaq (rPC, rINSTq, 2), %rsi 1880 movq rFP, %rdx 1881 call nterp_hot_method 1882 testq %rax, %rax 1883 jne 1f 1884 leaq (rPC, rINSTq, 2), rPC 1885 FETCH_INST 1886 GOTO_NEXT 18871: 1888 // Drop the current frame. 1889 movq -8(rREFS), %rsp 1890 CFI_DEF_CFA(rsp, CALLEE_SAVES_SIZE) 1891 1892 // Setup the new frame 1893 movq OSR_DATA_FRAME_SIZE(%rax), %rcx 1894 // Given stack size contains all callee saved registers, remove them. 1895 subq $$CALLEE_SAVES_SIZE, %rcx 1896 1897 // Remember CFA. 1898 movq %rsp, %rbp 1899 CFI_DEF_CFA_REGISTER(rbp) 1900 1901 subq %rcx, %rsp 1902 movq %rsp, %rdi // rdi := beginning of stack 1903 leaq OSR_DATA_MEMORY(%rax), %rsi // rsi := memory to copy 1904 rep movsb // while (rcx--) { *rdi++ = *rsi++ } 1905 1906 // Fetch the native PC to jump to and save it in a callee-save register. 1907 movq OSR_DATA_NATIVE_PC(%rax), %rbx 1908 1909 // Free the memory holding OSR Data. 1910 movq %rax, %rdi 1911 call free 1912 1913 // Jump to the compiled code. 1914 jmp *%rbx 1915 1916NterpHandleInvokeInterfaceOnObjectMethodRange: 1917 // First argument is the 'this' pointer. 1918 movzwl 4(rPC), %r11d // arguments 1919 movl (rFP, %r11, 4), %esi 1920 // Note: if esi is null, this will be handled by our SIGSEGV handler. 1921 movl MIRROR_OBJECT_CLASS_OFFSET(%esi), %edx 1922 movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %eax, 8), %rdi 1923 jmp NterpCommonInvokeInstanceRange 1924 1925NterpHandleInvokeInterfaceOnObjectMethod: 1926 // First argument is the 'this' pointer. 1927 movzwl 4(rPC), %r11d // arguments 1928 andq MACRO_LITERAL(0xf), %r11 1929 movl (rFP, %r11, 4), %esi 1930 // Note: if esi is null, this will be handled by our SIGSEGV handler. 1931 movl MIRROR_OBJECT_CLASS_OFFSET(%esi), %edx 1932 movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %eax, 8), %rdi 1933 jmp NterpCommonInvokeInstance 1934 1935// This is the logical end of ExecuteNterpImpl, where the frame info applies. 1936// EndExecuteNterpImpl includes the methods below as we want the runtime to 1937// see them as part of the Nterp PCs. 1938.cfi_endproc 1939 1940nterp_to_nterp_static_non_range: 1941 .cfi_startproc 1942 .cfi_def_cfa rsp, 8 1943 SETUP_STACK_FOR_INVOKE 1944 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=1, is_string_init=0 1945 .cfi_endproc 1946 1947nterp_to_nterp_string_init_non_range: 1948 .cfi_startproc 1949 .cfi_def_cfa rsp, 8 1950 SETUP_STACK_FOR_INVOKE 1951 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 1952 .cfi_endproc 1953 1954nterp_to_nterp_instance_non_range: 1955 .cfi_startproc 1956 .cfi_def_cfa rsp, 8 1957 SETUP_STACK_FOR_INVOKE 1958 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 1959 .cfi_endproc 1960 1961nterp_to_nterp_static_range: 1962 .cfi_startproc 1963 .cfi_def_cfa rsp, 8 1964 SETUP_STACK_FOR_INVOKE 1965 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=1 1966 .cfi_endproc 1967 1968nterp_to_nterp_instance_range: 1969 .cfi_startproc 1970 .cfi_def_cfa rsp, 8 1971 SETUP_STACK_FOR_INVOKE 1972 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0 1973 .cfi_endproc 1974 1975nterp_to_nterp_string_init_range: 1976 .cfi_startproc 1977 .cfi_def_cfa rsp, 8 1978 SETUP_STACK_FOR_INVOKE 1979 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 1980 .cfi_endproc 1981 1982// This is the end of PCs contained by the OatQuickMethodHeader created for the interpreter 1983// entry point. 1984 FUNCTION_TYPE(EndExecuteNterpImpl) 1985 ASM_HIDDEN SYMBOL(EndExecuteNterpImpl) 1986 .global SYMBOL(EndExecuteNterpImpl) 1987SYMBOL(EndExecuteNterpImpl): 1988 1989// Entrypoints into runtime. 1990NTERP_TRAMPOLINE nterp_get_static_field, NterpGetStaticField 1991NTERP_TRAMPOLINE nterp_get_instance_field_offset, NterpGetInstanceFieldOffset 1992NTERP_TRAMPOLINE nterp_filled_new_array, NterpFilledNewArray 1993NTERP_TRAMPOLINE nterp_filled_new_array_range, NterpFilledNewArrayRange 1994NTERP_TRAMPOLINE nterp_get_class_or_allocate_object, NterpGetClassOrAllocateObject 1995NTERP_TRAMPOLINE nterp_get_method, NterpGetMethod 1996NTERP_TRAMPOLINE nterp_hot_method, NterpHotMethod 1997NTERP_TRAMPOLINE nterp_load_object, NterpLoadObject 1998 1999// gen_mterp.py will inline the following definitions 2000// within [ExecuteNterpImpl, EndExecuteNterpImpl). 2001%def instruction_end(): 2002 2003 FUNCTION_TYPE(artNterpAsmInstructionEnd) 2004 ASM_HIDDEN SYMBOL(artNterpAsmInstructionEnd) 2005 .global SYMBOL(artNterpAsmInstructionEnd) 2006SYMBOL(artNterpAsmInstructionEnd): 2007 // artNterpAsmInstructionEnd is used as landing pad for exception handling. 2008 FETCH_INST 2009 GOTO_NEXT 2010 2011%def instruction_start(): 2012 2013 FUNCTION_TYPE(artNterpAsmInstructionStart) 2014 ASM_HIDDEN SYMBOL(artNterpAsmInstructionStart) 2015 .global SYMBOL(artNterpAsmInstructionStart) 2016SYMBOL(artNterpAsmInstructionStart) = .L_op_nop 2017 .text 2018 2019%def opcode_start(): 2020 ENTRY nterp_${opcode} 2021%def opcode_end(): 2022 END nterp_${opcode} 2023%def helper_start(name): 2024 ENTRY ${name} 2025%def helper_end(name): 2026 END ${name} 2027