1%def header():
2/*
3 * Copyright (C) 2016 The Android Open Source Project
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 *      http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18/*
19  Art assembly interpreter notes:
20
21  First validate assembly code by implementing ExecuteXXXImpl() style body (doesn't
22  handle invoke, allows higher-level code to create frame & shadow frame.
23
24  Once that's working, support direct entry code & eliminate shadow frame (and
25  excess locals allocation.
26
27  Some (hopefully) temporary ugliness.  We'll treat rFP as pointing to the
28  base of the vreg array within the shadow frame.  Access the other fields,
29  dex_pc_, method_ and number_of_vregs_ via negative offsets.  For now, we'll continue
30  the shadow frame mechanism of double-storing object references - via rFP &
31  number_of_vregs_.
32
33 */
34
35/*
36x86_64 ABI general notes:
37
38Caller save set:
39   rax, rdx, rcx, rsi, rdi, r8-r11, st(0)-st(7)
40Callee save set:
41   rbx, rbp, r12-r15
42Return regs:
43   32-bit in eax
44   64-bit in rax
45   fp on xmm0
46
47First 8 fp parameters came in xmm0-xmm7.
48First 6 non-fp parameters came in rdi, rsi, rdx, rcx, r8, r9.
49Other parameters passed on stack, pushed right-to-left.  On entry to target, first
50param is at 8(%esp).  Traditional entry code is:
51
52Stack must be 16-byte aligned to support SSE in native code.
53
54If we're not doing variable stack allocation (alloca), the frame pointer can be
55eliminated and all arg references adjusted to be esp relative.
56*/
57
58/*
59Mterp and x86_64 notes:
60
61Some key interpreter variables will be assigned to registers.
62
63  nick     reg   purpose
64  rPROFILE rbp   countdown register for jit profiling
65  rPC      r12   interpreted program counter, used for fetching instructions
66  rFP      r13   interpreted frame pointer, used for accessing locals and args
67  rINSTw   bx    first 16-bit code of current instruction
68  rINSTbl  bl    opcode portion of instruction word
69  rINSTbh  bh    high byte of inst word, usually contains src/tgt reg names
70  rIBASE   r14   base of instruction handler table
71  rREFS    r15   base of object references in shadow frame.
72
73Notes:
74   o High order 16 bits of ebx must be zero on entry to handler
75   o rPC, rFP, rINSTw/rINSTbl valid on handler entry and exit
76   o eax and ecx are scratch, rINSTw/ebx sometimes scratch
77
78Macros are provided for common operations.  Each macro MUST emit only
79one instruction to make instruction-counting easier.  They MUST NOT alter
80unspecified registers or condition codes.
81*/
82
83/*
84 * This is a #include, not a %include, because we want the C pre-processor
85 * to expand the macros into assembler assignment statements.
86 */
87#include "asm_support.h"
88#include "interpreter/cfi_asm_support.h"
89
90#define LITERAL(value) $$(value)
91
92/*
93 * Handle mac compiler specific
94 */
95#if defined(__APPLE__)
96    #define MACRO_LITERAL(value) $$(value)
97    #define FUNCTION_TYPE(name)
98    #define OBJECT_TYPE(name)
99    #define SIZE(start,end)
100    // Mac OS' symbols have an _ prefix.
101    #define SYMBOL(name) _ ## name
102    #define ASM_HIDDEN .private_extern
103#else
104    #define MACRO_LITERAL(value) $$value
105    #define FUNCTION_TYPE(name) .type name, @function
106    #define OBJECT_TYPE(name) .type name, @object
107    #define SIZE(start,end) .size start, .-end
108    #define SYMBOL(name) name
109    #define ASM_HIDDEN .hidden
110#endif
111
112.macro PUSH _reg
113    pushq \_reg
114    .cfi_adjust_cfa_offset 8
115    .cfi_rel_offset \_reg, 0
116.endm
117
118.macro POP _reg
119    popq \_reg
120    .cfi_adjust_cfa_offset -8
121    .cfi_restore \_reg
122.endm
123
124/*
125 * Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs.  So,
126 * to access other shadow frame fields, we need to use a backwards offset.  Define those here.
127 */
128#define OFF_FP(a) (a - SHADOWFRAME_VREGS_OFFSET)
129#define OFF_FP_NUMBER_OF_VREGS OFF_FP(SHADOWFRAME_NUMBER_OF_VREGS_OFFSET)
130#define OFF_FP_DEX_PC OFF_FP(SHADOWFRAME_DEX_PC_OFFSET)
131#define OFF_FP_LINK OFF_FP(SHADOWFRAME_LINK_OFFSET)
132#define OFF_FP_METHOD OFF_FP(SHADOWFRAME_METHOD_OFFSET)
133#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
134#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
135#define OFF_FP_DEX_INSTRUCTIONS OFF_FP(SHADOWFRAME_DEX_INSTRUCTIONS_OFFSET)
136#define OFF_FP_COUNTDOWN_OFFSET OFF_FP(SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET)
137#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
138
139/* Frame size must be 16-byte aligned.
140 * Remember about 8 bytes for return address + 6 * 8 for spills.
141 */
142#define FRAME_SIZE     8
143
144/* Frame diagram while executing ExecuteMterpImpl, high to low addresses */
145#define IN_ARG3        %rcx
146#define IN_ARG2        %rdx
147#define IN_ARG1        %rsi
148#define IN_ARG0        %rdi
149/* Spill offsets relative to %esp */
150#define SELF_SPILL     (FRAME_SIZE -  8)
151/* Out Args  */
152#define OUT_ARG3       %rcx
153#define OUT_ARG2       %rdx
154#define OUT_ARG1       %rsi
155#define OUT_ARG0       %rdi
156#define OUT_32_ARG3    %ecx
157#define OUT_32_ARG2    %edx
158#define OUT_32_ARG1    %esi
159#define OUT_32_ARG0    %edi
160#define OUT_FP_ARG1    %xmm1
161#define OUT_FP_ARG0    %xmm0
162
163/* During bringup, we'll use the shadow frame model instead of rFP */
164/* single-purpose registers, given names for clarity */
165#define rSELF    SELF_SPILL(%rsp)
166#define rPC      %r12
167#define CFI_DEX  12 // DWARF register number of the register holding dex-pc (rPC).
168#define CFI_TMP  5  // DWARF register number of the first argument register (rdi).
169#define rFP      %r13
170#define rINST    %ebx
171#define rINSTq   %rbx
172#define rINSTw   %bx
173#define rINSTbh  %bh
174#define rINSTbl  %bl
175#define rIBASE   %r14
176#define rREFS    %r15
177#define rPROFILE %ebp
178
179#define MTERP_LOGGING 0
180
181/*
182 * "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects.  Must
183 * be done *before* something throws.
184 *
185 * It's okay to do this more than once.
186 *
187 * NOTE: the fast interpreter keeps track of dex pc as a direct pointer to the mapped
188 * dex byte codes.  However, the rest of the runtime expects dex pc to be an instruction
189 * offset into the code_items_[] array.  For effiency, we will "export" the
190 * current dex pc as a direct pointer using the EXPORT_PC macro, and rely on GetDexPC
191 * to convert to a dex pc when needed.
192 */
193.macro EXPORT_PC
194    movq    rPC, OFF_FP_DEX_PC_PTR(rFP)
195.endm
196
197/*
198 * Refresh handler table.
199 * IBase handles uses the caller save register so we must restore it after each call.
200 * Also it is used as a result of some 64-bit operations (like imul) and we should
201 * restore it in such cases also.
202 *
203 */
204.macro REFRESH_IBASE_REG self_reg
205    movq    THREAD_CURRENT_IBASE_OFFSET(\self_reg), rIBASE
206.endm
207.macro REFRESH_IBASE
208    movq    rSELF, rIBASE
209    REFRESH_IBASE_REG rIBASE
210.endm
211
212/*
213 * Refresh rINST.
214 * At enter to handler rINST does not contain the opcode number.
215 * However some utilities require the full value, so this macro
216 * restores the opcode number.
217 */
218.macro REFRESH_INST _opnum
219    movb    rINSTbl, rINSTbh
220    movb    $$\_opnum, rINSTbl
221.endm
222
223/*
224 * Fetch the next instruction from rPC into rINSTw.  Does not advance rPC.
225 */
226.macro FETCH_INST
227    movzwq  (rPC), rINSTq
228.endm
229
230/*
231 * Remove opcode from rINST, compute the address of handler and jump to it.
232 */
233.macro GOTO_NEXT
234    movzx   rINSTbl,%eax
235    movzbl  rINSTbh,rINST
236    shll    MACRO_LITERAL(${handler_size_bits}), %eax
237    addq    rIBASE, %rax
238    jmp     *%rax
239.endm
240
241/*
242 * Advance rPC by instruction count.
243 */
244.macro ADVANCE_PC _count
245    leaq    2*\_count(rPC), rPC
246.endm
247
248/*
249 * Advance rPC by instruction count, fetch instruction and jump to handler.
250 */
251.macro ADVANCE_PC_FETCH_AND_GOTO_NEXT _count
252    ADVANCE_PC \_count
253    FETCH_INST
254    GOTO_NEXT
255.endm
256
257/*
258 * Get/set the 32-bit value from a Dalvik register.
259 */
260#define VREG_ADDRESS(_vreg) (rFP,_vreg,4)
261#define VREG_HIGH_ADDRESS(_vreg) 4(rFP,_vreg,4)
262#define VREG_REF_ADDRESS(_vreg) (rREFS,_vreg,4)
263#define VREG_REF_HIGH_ADDRESS(_vreg) 4(rREFS,_vreg,4)
264
265.macro GET_VREG _reg _vreg
266    movl    VREG_ADDRESS(\_vreg), \_reg
267.endm
268
269/* Read wide value. */
270.macro GET_WIDE_VREG _reg _vreg
271    movq    VREG_ADDRESS(\_vreg), \_reg
272.endm
273
274.macro SET_VREG _reg _vreg
275    movl    \_reg, VREG_ADDRESS(\_vreg)
276    movl    MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg)
277.endm
278
279/* Write wide value. reg is clobbered. */
280.macro SET_WIDE_VREG _reg _vreg
281    movq    \_reg, VREG_ADDRESS(\_vreg)
282    xorq    \_reg, \_reg
283    movq    \_reg, VREG_REF_ADDRESS(\_vreg)
284.endm
285
286.macro SET_VREG_OBJECT _reg _vreg
287    movl    \_reg, VREG_ADDRESS(\_vreg)
288    movl    \_reg, VREG_REF_ADDRESS(\_vreg)
289.endm
290
291.macro GET_VREG_HIGH _reg _vreg
292    movl    VREG_HIGH_ADDRESS(\_vreg), \_reg
293.endm
294
295.macro SET_VREG_HIGH _reg _vreg
296    movl    \_reg, VREG_HIGH_ADDRESS(\_vreg)
297    movl    MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg)
298.endm
299
300.macro CLEAR_REF _vreg
301    movl    MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg)
302.endm
303
304.macro CLEAR_WIDE_REF _vreg
305    movl    MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg)
306    movl    MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg)
307.endm
308
309.macro GET_VREG_XMMs _xmmreg _vreg
310    movss VREG_ADDRESS(\_vreg), \_xmmreg
311.endm
312.macro GET_VREG_XMMd _xmmreg _vreg
313    movsd VREG_ADDRESS(\_vreg), \_xmmreg
314.endm
315.macro SET_VREG_XMMs _xmmreg _vreg
316    movss \_xmmreg, VREG_ADDRESS(\_vreg)
317.endm
318.macro SET_VREG_XMMd _xmmreg _vreg
319    movsd \_xmmreg, VREG_ADDRESS(\_vreg)
320.endm
321
322/*
323 * function support macros.
324 */
325.macro ENTRY name
326    .text
327    ASM_HIDDEN SYMBOL(\name)
328    .global SYMBOL(\name)
329    FUNCTION_TYPE(\name)
330SYMBOL(\name):
331.endm
332
333.macro END name
334    SIZE(\name,\name)
335.endm
336
337%def entry():
338/*
339 * Copyright (C) 2016 The Android Open Source Project
340 *
341 * Licensed under the Apache License, Version 2.0 (the "License");
342 * you may not use this file except in compliance with the License.
343 * You may obtain a copy of the License at
344 *
345 *      http://www.apache.org/licenses/LICENSE-2.0
346 *
347 * Unless required by applicable law or agreed to in writing, software
348 * distributed under the License is distributed on an "AS IS" BASIS,
349 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
350 * See the License for the specific language governing permissions and
351 * limitations under the License.
352 */
353/*
354 * Interpreter entry point.
355 *
356 * On entry:
357 *  0  Thread* self
358 *  1  insns_
359 *  2  ShadowFrame
360 *  3  JValue* result_register
361 *
362 */
363
364ENTRY ExecuteMterpImpl
365    .cfi_startproc
366    .cfi_def_cfa rsp, 8
367
368    /* Spill callee save regs */
369    PUSH %rbx
370    PUSH %rbp
371    PUSH %r12
372    PUSH %r13
373    PUSH %r14
374    PUSH %r15
375
376    /* Allocate frame */
377    subq    $$FRAME_SIZE, %rsp
378    .cfi_adjust_cfa_offset FRAME_SIZE
379
380    /* Remember the return register */
381    movq    IN_ARG3, SHADOWFRAME_RESULT_REGISTER_OFFSET(IN_ARG2)
382
383    /* Remember the code_item */
384    movq    IN_ARG1, SHADOWFRAME_DEX_INSTRUCTIONS_OFFSET(IN_ARG2)
385
386    /* set up "named" registers */
387    movl    SHADOWFRAME_NUMBER_OF_VREGS_OFFSET(IN_ARG2), %eax
388    leaq    SHADOWFRAME_VREGS_OFFSET(IN_ARG2), rFP
389    leaq    (rFP, %rax, 4), rREFS
390    movl    SHADOWFRAME_DEX_PC_OFFSET(IN_ARG2), %eax
391    leaq    (IN_ARG1, %rax, 2), rPC
392    CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0)
393    EXPORT_PC
394
395    /* Starting ibase */
396    movq    IN_ARG0, rSELF
397    REFRESH_IBASE_REG IN_ARG0
398
399    /* Set up for backwards branches & osr profiling */
400    movq    IN_ARG0, OUT_ARG2  /* Set up OUT_ARG2 before clobbering IN_ARG0 */
401    movq    OFF_FP_METHOD(rFP), OUT_ARG0
402    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
403    call    SYMBOL(MterpSetUpHotnessCountdown)
404    movswl  %ax, rPROFILE
405
406    /* start executing the instruction at rPC */
407    FETCH_INST
408    GOTO_NEXT
409    /* NOTE: no fallthrough */
410    // cfi info continues, and covers the whole mterp implementation.
411    END ExecuteMterpImpl
412
413%def dchecks_before_helper():
414    // Call C++ to do debug checks and return to the handler using tail call.
415    .extern MterpCheckBefore
416    popq    %rax                     # Return address (the instuction handler).
417    REFRESH_IBASE
418    movq    rSELF, OUT_ARG0
419    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
420    movq    rPC, OUT_ARG2
421    pushq   %rax                     # Return address for the tail call.
422    jmp     SYMBOL(MterpCheckBefore) # (self, shadow_frame, dex_pc_ptr)
423
424%def opcode_pre():
425%  add_helper(dchecks_before_helper, "mterp_dchecks_before_helper")
426    #if !defined(NDEBUG)
427    call    SYMBOL(mterp_dchecks_before_helper)
428    #endif
429
430%def fallback():
431/* Transfer stub to alternate interpreter */
432    jmp     MterpFallback
433
434
435%def helpers():
436    ENTRY MterpHelpers
437
438%def footer():
439/*
440 * ===========================================================================
441 *  Common subroutines and data
442 * ===========================================================================
443 */
444
445    .text
446    .align  2
447
448/*
449 * We've detected a condition that will result in an exception, but the exception
450 * has not yet been thrown.  Just bail out to the reference interpreter to deal with it.
451 * TUNING: for consistency, we may want to just go ahead and handle these here.
452 */
453common_errDivideByZero:
454    EXPORT_PC
455#if MTERP_LOGGING
456    movq    rSELF, OUT_ARG0
457    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
458    call    SYMBOL(MterpLogDivideByZeroException)
459#endif
460    jmp     MterpCommonFallback
461
462common_errArrayIndex:
463    EXPORT_PC
464#if MTERP_LOGGING
465    movq    rSELF, OUT_ARG0
466    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
467    call    SYMBOL(MterpLogArrayIndexException)
468#endif
469    jmp     MterpCommonFallback
470
471common_errNegativeArraySize:
472    EXPORT_PC
473#if MTERP_LOGGING
474    movq    rSELF, OUT_ARG0
475    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
476    call    SYMBOL(MterpLogNegativeArraySizeException)
477#endif
478    jmp     MterpCommonFallback
479
480common_errNoSuchMethod:
481    EXPORT_PC
482#if MTERP_LOGGING
483    movq    rSELF, OUT_ARG0
484    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
485    call    SYMBOL(MterpLogNoSuchMethodException)
486#endif
487    jmp     MterpCommonFallback
488
489common_errNullObject:
490    EXPORT_PC
491#if MTERP_LOGGING
492    movq    rSELF, OUT_ARG0
493    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
494    call    SYMBOL(MterpLogNullObjectException)
495#endif
496    jmp     MterpCommonFallback
497
498common_exceptionThrown:
499    EXPORT_PC
500#if MTERP_LOGGING
501    movq    rSELF, OUT_ARG0
502    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
503    call    SYMBOL(MterpLogExceptionThrownException)
504#endif
505    jmp     MterpCommonFallback
506
507MterpSuspendFallback:
508    EXPORT_PC
509#if MTERP_LOGGING
510    movq    rSELF, OUT_ARG0
511    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
512    movl    THREAD_FLAGS_OFFSET(OUT_ARG0), OUT_32_ARG2
513    call    SYMBOL(MterpLogSuspendFallback)
514#endif
515    jmp     MterpCommonFallback
516
517/*
518 * If we're here, something is out of the ordinary.  If there is a pending
519 * exception, handle it.  Otherwise, roll back and retry with the reference
520 * interpreter.
521 */
522MterpPossibleException:
523    movq    rSELF, %rcx
524    cmpq    $$0, THREAD_EXCEPTION_OFFSET(%rcx)
525    jz      MterpFallback
526    /* intentional fallthrough - handle pending exception. */
527
528/*
529 * On return from a runtime helper routine, we've found a pending exception.
530 * Can we handle it here - or need to bail out to caller?
531 *
532 */
533MterpException:
534    movq    rSELF, OUT_ARG0
535    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
536    call    SYMBOL(MterpHandleException)
537    testb   %al, %al
538    jz      MterpExceptionReturn
539    movq    OFF_FP_DEX_INSTRUCTIONS(rFP), %rax
540    mov     OFF_FP_DEX_PC(rFP), %ecx
541    leaq    (%rax, %rcx, 2), rPC
542    movq    rPC, OFF_FP_DEX_PC_PTR(rFP)
543    /* Do we need to switch interpreters? */
544    movq    rSELF, %rax
545    cmpb    LITERAL(0), THREAD_USE_MTERP_OFFSET(%rax)
546    jz      MterpFallback
547    /* resume execution at catch block */
548    REFRESH_IBASE
549    FETCH_INST
550    GOTO_NEXT
551    /* NOTE: no fallthrough */
552
553/*
554 * Common handling for branches with support for Jit profiling.
555 * On entry:
556 *    rINST          <= signed offset
557 *    rPROFILE       <= signed hotness countdown (expanded to 32 bits)
558 *    condition bits <= set to establish sign of offset (use "NoFlags" entry if not)
559 *
560 * We have quite a few different cases for branch profiling, OSR detection and
561 * suspend check support here.
562 *
563 * Taken backward branches:
564 *    If profiling active, do hotness countdown and report if we hit zero.
565 *    If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
566 *    Is there a pending suspend request?  If so, suspend.
567 *
568 * Taken forward branches and not-taken backward branches:
569 *    If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
570 *
571 * Our most common case is expected to be a taken backward branch with active jit profiling,
572 * but no full OSR check and no pending suspend request.
573 * Next most common case is not-taken branch with no full OSR check.
574 *
575 */
576MterpCommonTakenBranch:
577    jg      .L_forward_branch               # don't add forward branches to hotness
578/*
579 * We need to subtract 1 from positive values and we should not see 0 here,
580 * so we may use the result of the comparison with -1.
581 */
582#if JIT_CHECK_OSR != -1
583#  error "JIT_CHECK_OSR must be -1."
584#endif
585    cmpl    $$JIT_CHECK_OSR, rPROFILE
586    je      .L_osr_check
587    decl    rPROFILE
588    je      .L_add_batch                    # counted down to zero - report
589.L_resume_backward_branch:
590    movq    rSELF, %rax
591    testl   $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(%rax)
592    REFRESH_IBASE_REG %rax
593    leaq    (rPC, rINSTq, 2), rPC
594    FETCH_INST
595    jnz     .L_suspend_request_pending
596    GOTO_NEXT
597
598.L_suspend_request_pending:
599    EXPORT_PC
600    movq    rSELF, OUT_ARG0
601    call    SYMBOL(MterpSuspendCheck)       # (self)
602    testb   %al, %al
603    jnz     MterpFallback
604    REFRESH_IBASE                           # might have changed during suspend
605    GOTO_NEXT
606
607.L_no_count_backwards:
608    cmpl    $$JIT_CHECK_OSR, rPROFILE         # possible OSR re-entry?
609    jne     .L_resume_backward_branch
610.L_osr_check:
611    EXPORT_PC
612    movq    rSELF, OUT_ARG0
613    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
614    movq    rINSTq, OUT_ARG2
615    call    SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
616    testb   %al, %al
617    jz      .L_resume_backward_branch
618    jmp     MterpOnStackReplacement
619
620.L_forward_branch:
621    cmpl    $$JIT_CHECK_OSR, rPROFILE         # possible OSR re-entry?
622    je      .L_check_osr_forward
623.L_resume_forward_branch:
624    leaq    (rPC, rINSTq, 2), rPC
625    FETCH_INST
626    GOTO_NEXT
627
628.L_check_osr_forward:
629    EXPORT_PC
630    movq    rSELF, OUT_ARG0
631    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
632    movq    rINSTq, OUT_ARG2
633    call    SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
634    testb   %al, %al
635    jz      .L_resume_forward_branch
636    jmp     MterpOnStackReplacement
637
638.L_add_batch:
639    movl    rPROFILE, %eax
640    movq    OFF_FP_METHOD(rFP), OUT_ARG0
641    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
642    movw    %ax, OFF_FP_COUNTDOWN_OFFSET(rFP)
643    movq    rSELF, OUT_ARG2
644    call    SYMBOL(MterpAddHotnessBatch)    # (method, shadow_frame, self)
645    movswl  %ax, rPROFILE
646    jmp     .L_no_count_backwards
647
648/*
649 * Entered from the conditional branch handlers when OSR check request active on
650 * not-taken path.  All Dalvik not-taken conditional branch offsets are 2.
651 */
652.L_check_not_taken_osr:
653    EXPORT_PC
654    movq    rSELF, OUT_ARG0
655    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
656    movl    $$2, OUT_32_ARG2
657    call    SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
658    testb   %al, %al
659    jnz     MterpOnStackReplacement
660    ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
661
662/*
663 * On-stack replacement has happened, and now we've returned from the compiled method.
664 */
665MterpOnStackReplacement:
666#if MTERP_LOGGING
667    movq    rSELF, OUT_ARG0
668    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
669    movl    rINST, OUT_32_ARG2
670    call    SYMBOL(MterpLogOSR)
671#endif
672    movl    $$1, %eax
673    jmp     MterpDone
674
675/*
676 * Bail out to reference interpreter.
677 */
678MterpFallback:
679    EXPORT_PC
680#if MTERP_LOGGING
681    movq    rSELF, OUT_ARG0
682    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
683    call    SYMBOL(MterpLogFallback)
684#endif
685MterpCommonFallback:
686    xorl    %eax, %eax
687    jmp     MterpDone
688
689/*
690 * On entry:
691 *  uint32_t* rFP  (should still be live, pointer to base of vregs)
692 */
693MterpExceptionReturn:
694    movl    $$1, %eax
695    jmp     MterpDone
696MterpReturn:
697    movq    OFF_FP_RESULT_REGISTER(rFP), %rdx
698    movq    %rax, (%rdx)
699    movl    $$1, %eax
700MterpDone:
701/*
702 * At this point, we expect rPROFILE to be non-zero.  If negative, hotness is disabled or we're
703 * checking for OSR.  If greater than zero, we might have unreported hotness to register
704 * (the difference between the ending rPROFILE and the cached hotness counter).  rPROFILE
705 * should only reach zero immediately after a hotness decrement, and is then reset to either
706 * a negative special state or the new non-zero countdown value.
707 */
708    testl   rPROFILE, rPROFILE
709    jle     MRestoreFrame                   # if > 0, we may have some counts to report.
710
711    movl    %eax, rINST                     # stash return value
712    /* Report cached hotness counts */
713    movl    rPROFILE, %eax
714    movq    OFF_FP_METHOD(rFP), OUT_ARG0
715    leaq    OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
716    movw    %ax, OFF_FP_COUNTDOWN_OFFSET(rFP)
717    movq    rSELF, OUT_ARG2
718    call    SYMBOL(MterpAddHotnessBatch)    # (method, shadow_frame, self)
719    movl    rINST, %eax                     # restore return value
720
721    /* pop up frame */
722MRestoreFrame:
723    addq    $$FRAME_SIZE, %rsp
724    .cfi_adjust_cfa_offset -FRAME_SIZE
725
726    /* Restore callee save register */
727    POP %r15
728    POP %r14
729    POP %r13
730    POP %r12
731    POP %rbp
732    POP %rbx
733    ret
734    .cfi_endproc
735    END MterpHelpers
736
737%def instruction_end():
738
739    OBJECT_TYPE(artMterpAsmInstructionEnd)
740    ASM_HIDDEN SYMBOL(artMterpAsmInstructionEnd)
741    .global SYMBOL(artMterpAsmInstructionEnd)
742SYMBOL(artMterpAsmInstructionEnd):
743
744%def instruction_start():
745
746    OBJECT_TYPE(artMterpAsmInstructionStart)
747    ASM_HIDDEN SYMBOL(artMterpAsmInstructionStart)
748    .global SYMBOL(artMterpAsmInstructionStart)
749SYMBOL(artMterpAsmInstructionStart) = .L_op_nop
750    .text
751
752%def opcode_start():
753    ENTRY mterp_${opcode}
754%def opcode_end():
755    END mterp_${opcode}
756%def helper_start(name):
757    ENTRY ${name}
758%def helper_end(name):
759    END ${name}
760