Lines Matching refs:__
67 #define __ down_cast<X86_64Assembler*>(codegen->GetAssembler())-> // NOLINT macro
94 __ Bind(GetEntryLabel()); in EmitNativeCode()
96 __ Bind(&loop); in EmitNativeCode()
97 __ movl(CpuRegister(TMP), Address(src_curr_addr, 0)); in EmitNativeCode()
98 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in EmitNativeCode()
107 __ MaybePoisonHeapReference(CpuRegister(TMP)); in EmitNativeCode()
108 __ movl(Address(dst_curr_addr, 0), CpuRegister(TMP)); in EmitNativeCode()
109 __ addl(src_curr_addr, Immediate(element_size)); in EmitNativeCode()
110 __ addl(dst_curr_addr, Immediate(element_size)); in EmitNativeCode()
111 __ cmpl(src_curr_addr, src_stop_addr); in EmitNativeCode()
112 __ j(kNotEqual, &loop); in EmitNativeCode()
113 __ jmp(GetExitLabel()); in EmitNativeCode()
122 #undef __
124 #define __ assembler-> macro
143 __ movd(output.AsRegister<CpuRegister>(), input.AsFpuRegister<XmmRegister>(), is64bit); in MoveFPToInt()
149 __ movd(output.AsFpuRegister<XmmRegister>(), input.AsRegister<CpuRegister>(), is64bit); in MoveIntToFP()
195 __ bswapl(out); in GenReverseBytes()
196 __ sarl(out, Immediate(16)); in GenReverseBytes()
199 __ bswapl(out); in GenReverseBytes()
202 __ bswapq(out); in GenReverseBytes()
269 __ roundsd(out, in, Immediate(round_mode)); in GenSSE41FPToFPIntrinsic()
332 __ movss(t2, in); in VisitMathRoundFloat()
333 __ roundss(t1, in, Immediate(1)); in VisitMathRoundFloat()
334 __ subss(t2, t1); in VisitMathRoundFloat()
335 __ comiss(t2, codegen_->LiteralFloatAddress(0.5f)); in VisitMathRoundFloat()
336 __ j(kBelow, &skip_incr); in VisitMathRoundFloat()
337 __ addss(t1, codegen_->LiteralFloatAddress(1.0f)); in VisitMathRoundFloat()
338 __ Bind(&skip_incr); in VisitMathRoundFloat()
344 __ cvtsi2ss(t2, out); in VisitMathRoundFloat()
345 __ comiss(t1, t2); in VisitMathRoundFloat()
346 __ j(kAboveEqual, &done); // clipped to max (already in out), does not jump on unordered in VisitMathRoundFloat()
347 __ movl(out, Immediate(0)); // does not change flags in VisitMathRoundFloat()
348 __ j(kUnordered, &done); // NaN mapped to 0 (just moved in out) in VisitMathRoundFloat()
349 __ cvttss2si(out, t1); in VisitMathRoundFloat()
350 __ Bind(&done); in VisitMathRoundFloat()
373 __ movsd(t2, in); in VisitMathRoundDouble()
374 __ roundsd(t1, in, Immediate(1)); in VisitMathRoundDouble()
375 __ subsd(t2, t1); in VisitMathRoundDouble()
376 __ comisd(t2, codegen_->LiteralDoubleAddress(0.5)); in VisitMathRoundDouble()
377 __ j(kBelow, &skip_incr); in VisitMathRoundDouble()
378 __ addsd(t1, codegen_->LiteralDoubleAddress(1.0f)); in VisitMathRoundDouble()
379 __ Bind(&skip_incr); in VisitMathRoundDouble()
385 __ cvtsi2sd(t2, out, /* is64bit= */ true); in VisitMathRoundDouble()
386 __ comisd(t1, t2); in VisitMathRoundDouble()
387 __ j(kAboveEqual, &done); // clipped to max (already in out), does not jump on unordered in VisitMathRoundDouble()
388 __ movl(out, Immediate(0)); // does not change flags, implicit zero extension to 64-bit in VisitMathRoundDouble()
389 __ j(kUnordered, &done); // NaN mapped to 0 (just moved in out) in VisitMathRoundDouble()
390 __ cvttsd2si(out, t1, /* is64bit= */ true); in VisitMathRoundDouble()
391 __ Bind(&done); in VisitMathRoundDouble()
622 __ cmpl(Address(input, length_offset), in CheckPosition()
625 __ cmpl(Address(input, length_offset), length.AsRegister<CpuRegister>()); in CheckPosition()
627 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
631 __ movl(temp, Address(input, length_offset)); in CheckPosition()
632 __ subl(temp, Immediate(pos_const)); in CheckPosition()
633 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
637 __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in CheckPosition()
639 __ cmpl(temp, length.AsRegister<CpuRegister>()); in CheckPosition()
641 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
646 __ testl(pos_reg, pos_reg); in CheckPosition()
647 __ j(kNotEqual, slow_path->GetEntryLabel()); in CheckPosition()
651 __ testl(pos_reg, pos_reg); in CheckPosition()
652 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
655 __ cmpl(Address(input, length_offset), pos_reg); in CheckPosition()
656 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
659 __ movl(temp, Address(input, length_offset)); in CheckPosition()
660 __ subl(temp, pos_reg); in CheckPosition()
662 __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in CheckPosition()
664 __ cmpl(temp, length.AsRegister<CpuRegister>()); in CheckPosition()
666 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
692 __ cmpl(src, dest); in VisitSystemArrayCopyChar()
693 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
696 __ testl(src, src); in VisitSystemArrayCopyChar()
697 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
700 __ testl(dest, dest); in VisitSystemArrayCopyChar()
701 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
706 __ testl(length.AsRegister<CpuRegister>(), length.AsRegister<CpuRegister>()); in VisitSystemArrayCopyChar()
707 __ j(kLess, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
718 __ movl(count, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in VisitSystemArrayCopyChar()
720 __ movl(count, length.AsRegister<CpuRegister>()); in VisitSystemArrayCopyChar()
732 __ leal(src_base, Address(src, char_size * src_pos_const + data_offset)); in VisitSystemArrayCopyChar()
734 __ leal(src_base, Address(src, src_pos.AsRegister<CpuRegister>(), in VisitSystemArrayCopyChar()
739 __ leal(dest_base, Address(dest, char_size * dest_pos_const + data_offset)); in VisitSystemArrayCopyChar()
741 __ leal(dest_base, Address(dest, dest_pos.AsRegister<CpuRegister>(), in VisitSystemArrayCopyChar()
746 __ rep_movsw(); in VisitSystemArrayCopyChar()
748 __ Bind(slow_path->GetExitLabel()); in VisitSystemArrayCopyChar()
783 __ leal(src_base, Address(src, element_size * constant + data_offset)); in GenSystemArrayCopyAddresses()
785 __ leal(src_base, Address(src, src_pos.AsRegister<CpuRegister>(), scale_factor, data_offset)); in GenSystemArrayCopyAddresses()
790 __ leal(dst_base, Address(dst, element_size * constant + data_offset)); in GenSystemArrayCopyAddresses()
792 __ leal(dst_base, Address(dst, dst_pos.AsRegister<CpuRegister>(), scale_factor, data_offset)); in GenSystemArrayCopyAddresses()
797 __ leal(src_end, Address(src_base, element_size * constant)); in GenSystemArrayCopyAddresses()
799 __ leal(src_end, Address(src_base, copy_length.AsRegister<CpuRegister>(), scale_factor, 0)); in GenSystemArrayCopyAddresses()
847 __ cmpl(src, dest); in VisitSystemArrayCopy()
848 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
852 __ cmpl(src, dest); in VisitSystemArrayCopy()
853 __ j(kNotEqual, &conditions_on_positions_validated); in VisitSystemArrayCopy()
855 __ cmpl(dest_pos.AsRegister<CpuRegister>(), Immediate(src_pos_constant)); in VisitSystemArrayCopy()
856 __ j(kGreater, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
860 __ cmpl(src, dest); in VisitSystemArrayCopy()
861 __ j(kNotEqual, &conditions_on_positions_validated); in VisitSystemArrayCopy()
865 __ cmpl(src_pos.AsRegister<CpuRegister>(), Immediate(dest_pos_constant)); in VisitSystemArrayCopy()
866 __ j(kLess, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
868 __ cmpl(src_pos.AsRegister<CpuRegister>(), dest_pos.AsRegister<CpuRegister>()); in VisitSystemArrayCopy()
869 __ j(kLess, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
873 __ Bind(&conditions_on_positions_validated); in VisitSystemArrayCopy()
877 __ testl(src, src); in VisitSystemArrayCopy()
878 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
883 __ testl(dest, dest); in VisitSystemArrayCopy()
884 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
892 __ testl(length.AsRegister<CpuRegister>(), length.AsRegister<CpuRegister>()); in VisitSystemArrayCopy()
893 __ j(kLess, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
938 __ movl(temp1, Address(dest, class_offset)); in VisitSystemArrayCopy()
940 __ movl(temp2, Address(src, class_offset)); in VisitSystemArrayCopy()
945 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
946 __ MaybeUnpoisonHeapReference(temp2); in VisitSystemArrayCopy()
957 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
958 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
963 __ movl(CpuRegister(TMP), Address(temp1, component_offset)); in VisitSystemArrayCopy()
964 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
965 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
966 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in VisitSystemArrayCopy()
968 __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
969 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
980 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
981 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
986 __ movl(CpuRegister(TMP), Address(temp2, component_offset)); in VisitSystemArrayCopy()
987 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
988 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
989 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in VisitSystemArrayCopy()
991 __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
992 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
995 __ cmpl(temp1, temp2); in VisitSystemArrayCopy()
999 __ j(kEqual, &do_copy); in VisitSystemArrayCopy()
1008 __ cmpl(Address(temp1, super_offset), Immediate(0)); in VisitSystemArrayCopy()
1011 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1014 __ movl(temp1, Address(temp1, component_offset)); in VisitSystemArrayCopy()
1015 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1018 __ cmpl(Address(temp1, super_offset), Immediate(0)); in VisitSystemArrayCopy()
1020 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1021 __ Bind(&do_copy); in VisitSystemArrayCopy()
1023 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1035 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1036 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1039 __ movl(temp1, Address(src, class_offset)); in VisitSystemArrayCopy()
1040 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1042 __ movl(CpuRegister(TMP), Address(temp1, component_offset)); in VisitSystemArrayCopy()
1044 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1045 __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1046 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in VisitSystemArrayCopy()
1048 __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
1049 __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1084 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1085 __ j(kEqual, &done); in VisitSystemArrayCopy()
1097 __ testb(Address(src, monitor_offset + gray_byte_position), Immediate(test_value)); in VisitSystemArrayCopy()
1109 __ j(kNotZero, read_barrier_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1114 __ Bind(&loop); in VisitSystemArrayCopy()
1115 __ movl(CpuRegister(TMP), Address(temp1, 0)); in VisitSystemArrayCopy()
1116 __ movl(Address(temp2, 0), CpuRegister(TMP)); in VisitSystemArrayCopy()
1117 __ addl(temp1, Immediate(element_size)); in VisitSystemArrayCopy()
1118 __ addl(temp2, Immediate(element_size)); in VisitSystemArrayCopy()
1119 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1120 __ j(kNotEqual, &loop); in VisitSystemArrayCopy()
1122 __ Bind(read_barrier_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
1123 __ Bind(&done); in VisitSystemArrayCopy()
1130 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1131 __ j(kEqual, &done); in VisitSystemArrayCopy()
1132 __ Bind(&loop); in VisitSystemArrayCopy()
1133 __ movl(CpuRegister(TMP), Address(temp1, 0)); in VisitSystemArrayCopy()
1134 __ movl(Address(temp2, 0), CpuRegister(TMP)); in VisitSystemArrayCopy()
1135 __ addl(temp1, Immediate(element_size)); in VisitSystemArrayCopy()
1136 __ addl(temp2, Immediate(element_size)); in VisitSystemArrayCopy()
1137 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1138 __ j(kNotEqual, &loop); in VisitSystemArrayCopy()
1139 __ Bind(&done); in VisitSystemArrayCopy()
1145 __ Bind(intrinsic_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
1165 __ testl(argument, argument); in VisitStringCompareTo()
1168 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringCompareTo()
1171 __ Bind(slow_path->GetExitLabel()); in VisitStringCompareTo()
1211 __ testl(arg, arg); in VisitStringEquals()
1212 __ j(kEqual, &return_false); in VisitStringEquals()
1227 __ movl(rcx, Address(str, class_offset)); in VisitStringEquals()
1229 __ cmpl(rcx, Address(arg, class_offset)); in VisitStringEquals()
1230 __ j(kNotEqual, &return_false); in VisitStringEquals()
1234 __ cmpl(str, arg); in VisitStringEquals()
1235 __ j(kEqual, &return_true); in VisitStringEquals()
1238 __ movl(rcx, Address(str, count_offset)); in VisitStringEquals()
1242 __ cmpl(rcx, Address(arg, count_offset)); in VisitStringEquals()
1243 __ j(kNotEqual, &return_false); in VisitStringEquals()
1247 __ jrcxz(&return_true); in VisitStringEquals()
1253 __ shrl(rcx, Immediate(1)); in VisitStringEquals()
1254 __ j(kCarrySet, &string_uncompressed); in VisitStringEquals()
1257 __ addl(rcx, Immediate(1)); in VisitStringEquals()
1258 __ shrl(rcx, Immediate(1)); in VisitStringEquals()
1259 __ Bind(&string_uncompressed); in VisitStringEquals()
1262 __ leal(rsi, Address(str, value_offset)); in VisitStringEquals()
1263 __ leal(rdi, Address(arg, value_offset)); in VisitStringEquals()
1266 __ addl(rcx, Immediate(3)); in VisitStringEquals()
1267 __ shrl(rcx, Immediate(2)); in VisitStringEquals()
1275 __ repe_cmpsq(); in VisitStringEquals()
1277 __ j(kNotEqual, &return_false); in VisitStringEquals()
1281 __ Bind(&return_true); in VisitStringEquals()
1282 __ movl(rsi, Immediate(1)); in VisitStringEquals()
1283 __ jmp(&end); in VisitStringEquals()
1286 __ Bind(&return_false); in VisitStringEquals()
1287 __ xorl(rsi, rsi); in VisitStringEquals()
1288 __ Bind(&end); in VisitStringEquals()
1348 __ jmp(slow_path->GetEntryLabel()); in GenerateStringIndexOf()
1349 __ Bind(slow_path->GetExitLabel()); in GenerateStringIndexOf()
1353 __ cmpl(search_value, Immediate(std::numeric_limits<uint16_t>::max())); in GenerateStringIndexOf()
1356 __ j(kAbove, slow_path->GetEntryLabel()); in GenerateStringIndexOf()
1367 __ movl(string_length, Address(string_obj, count_offset)); in GenerateStringIndexOf()
1372 __ testl(string_length, string_length); in GenerateStringIndexOf()
1373 __ j(kEqual, ¬_found_label); in GenerateStringIndexOf()
1377 __ movl(CpuRegister(TMP), string_length); in GenerateStringIndexOf()
1379 __ shrl(string_length, Immediate(1)); in GenerateStringIndexOf()
1384 __ movl(counter, string_length); in GenerateStringIndexOf()
1386 __ addq(string_obj, Immediate(value_offset)); in GenerateStringIndexOf()
1391 __ cmpl(start_index, string_length); in GenerateStringIndexOf()
1392 __ j(kGreaterEqual, ¬_found_label); in GenerateStringIndexOf()
1395 __ xorl(counter, counter); in GenerateStringIndexOf()
1396 __ cmpl(start_index, Immediate(0)); in GenerateStringIndexOf()
1397 __ cmov(kGreater, counter, start_index, /* is64bit= */ false); // 32-bit copy is enough. in GenerateStringIndexOf()
1401 __ testl(CpuRegister(TMP), Immediate(1)); in GenerateStringIndexOf()
1402 __ j(kNotZero, &offset_uncompressed_label); in GenerateStringIndexOf()
1403 __ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_1, value_offset)); in GenerateStringIndexOf()
1404 __ jmp(&modify_counter); in GenerateStringIndexOf()
1406 __ Bind(&offset_uncompressed_label); in GenerateStringIndexOf()
1407 __ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); in GenerateStringIndexOf()
1408 __ Bind(&modify_counter); in GenerateStringIndexOf()
1410 __ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); in GenerateStringIndexOf()
1413 __ negq(counter); // Needs to be 64-bit negation, as the address computation is 64-bit. in GenerateStringIndexOf()
1414 __ leaq(counter, Address(string_length, counter, ScaleFactor::TIMES_1, 0)); in GenerateStringIndexOf()
1420 __ testl(CpuRegister(TMP), Immediate(1)); in GenerateStringIndexOf()
1421 __ j(kNotZero, &uncompressed_string_comparison); in GenerateStringIndexOf()
1423 __ cmpl(search_value, Immediate(127)); in GenerateStringIndexOf()
1424 __ j(kGreater, ¬_found_label); in GenerateStringIndexOf()
1426 __ repne_scasb(); in GenerateStringIndexOf()
1427 __ jmp(&comparison_done); in GenerateStringIndexOf()
1431 __ Bind(&uncompressed_string_comparison); in GenerateStringIndexOf()
1432 __ repne_scasw(); in GenerateStringIndexOf()
1433 __ Bind(&comparison_done); in GenerateStringIndexOf()
1435 __ repne_scasw(); in GenerateStringIndexOf()
1438 __ j(kNotEqual, ¬_found_label); in GenerateStringIndexOf()
1441 __ subl(string_length, counter); in GenerateStringIndexOf()
1442 __ leal(out, Address(string_length, -1)); in GenerateStringIndexOf()
1445 __ jmp(&done); in GenerateStringIndexOf()
1448 __ Bind(¬_found_label); in GenerateStringIndexOf()
1449 __ movl(out, Immediate(-1)); in GenerateStringIndexOf()
1452 __ Bind(&done); in GenerateStringIndexOf()
1454 __ Bind(slow_path->GetExitLabel()); in GenerateStringIndexOf()
1490 __ testl(byte_array, byte_array); in VisitStringNewStringFromBytes()
1493 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringNewStringFromBytes()
1497 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromBytes()
1534 __ testl(string_to_copy, string_to_copy); in VisitStringNewStringFromString()
1537 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringNewStringFromString()
1541 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromString()
1585 __ movl(CpuRegister(RCX), srcEnd); in VisitStringGetCharsNoCheck()
1587 __ subl(CpuRegister(RCX), Immediate(srcBegin_value)); in VisitStringGetCharsNoCheck()
1590 __ subl(CpuRegister(RCX), srcBegin.AsRegister<CpuRegister>()); in VisitStringGetCharsNoCheck()
1599 __ testl(Address(obj, count_offset), Immediate(1)); in VisitStringGetCharsNoCheck()
1602 __ j(kNotZero, ©_uncompressed); in VisitStringGetCharsNoCheck()
1605 __ leaq(CpuRegister(RSI), in VisitStringGetCharsNoCheck()
1608 __ leaq(CpuRegister(RDI), Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); in VisitStringGetCharsNoCheck()
1610 __ Bind(©_loop); in VisitStringGetCharsNoCheck()
1611 __ jrcxz(&done); in VisitStringGetCharsNoCheck()
1614 __ movzxb(CpuRegister(TMP), Address(CpuRegister(RSI), 0)); in VisitStringGetCharsNoCheck()
1615 __ movw(Address(CpuRegister(RDI), 0), CpuRegister(TMP)); in VisitStringGetCharsNoCheck()
1616 __ leaq(CpuRegister(RDI), Address(CpuRegister(RDI), char_size)); in VisitStringGetCharsNoCheck()
1617 __ leaq(CpuRegister(RSI), Address(CpuRegister(RSI), c_char_size)); in VisitStringGetCharsNoCheck()
1619 __ subl(CpuRegister(RCX), Immediate(1)); in VisitStringGetCharsNoCheck()
1620 __ jmp(©_loop); in VisitStringGetCharsNoCheck()
1622 __ Bind(©_uncompressed); in VisitStringGetCharsNoCheck()
1625 __ leaq(CpuRegister(RSI), in VisitStringGetCharsNoCheck()
1628 __ leaq(CpuRegister(RDI), Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); in VisitStringGetCharsNoCheck()
1630 __ rep_movsw(); in VisitStringGetCharsNoCheck()
1632 __ Bind(&done); in VisitStringGetCharsNoCheck()
1642 __ movsxb(out, Address(address, 0)); in GenPeek()
1645 __ movsxw(out, Address(address, 0)); in GenPeek()
1648 __ movl(out, Address(address, 0)); in GenPeek()
1651 __ movq(out, Address(address, 0)); in GenPeek()
1706 __ movb(Address(address, 0), in GenPoke()
1709 __ movb(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
1714 __ movw(Address(address, 0), in GenPoke()
1717 __ movw(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
1722 __ movl(Address(address, 0), in GenPoke()
1725 __ movl(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
1733 __ movq(Address(address, 0), Immediate(v_32)); in GenPoke()
1735 __ movq(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
1803 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
1813 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
1818 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
1819 __ MaybeUnpoisonHeapReference(output); in GenUnsafeGet()
1825 __ movq(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
1948 __ movq(Address(base, offset, ScaleFactor::TIMES_1, 0), value); in GenUnsafePut()
1951 __ movl(temp, value); in GenUnsafePut()
1952 __ PoisonHeapReference(temp); in GenUnsafePut()
1953 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), temp); in GenUnsafePut()
1955 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), value); in GenUnsafePut()
2099 __ movl(CpuRegister(value_reg), base); in GenCAS()
2113 __ PoisonHeapReference(expected); in GenCAS()
2114 __ PoisonHeapReference(CpuRegister(value_reg)); in GenCAS()
2117 __ LockCmpxchgl(field_addr, CpuRegister(value_reg)); in GenCAS()
2123 __ setcc(kZero, out); in GenCAS()
2124 __ movzxb(out, out); in GenCAS()
2136 __ UnpoisonHeapReference(CpuRegister(value_reg)); in GenCAS()
2141 __ UnpoisonHeapReference(expected); in GenCAS()
2145 __ LockCmpxchgl(Address(base, offset, TIMES_1, 0), value); in GenCAS()
2147 __ LockCmpxchgq(Address(base, offset, TIMES_1, 0), value); in GenCAS()
2156 __ setcc(kZero, out); in GenCAS()
2157 __ movzxb(out, out); in GenCAS()
2189 __ movl(temp, reg); in SwapBits()
2190 __ shrl(reg, imm_shift); in SwapBits()
2191 __ andl(temp, imm_mask); in SwapBits()
2192 __ andl(reg, imm_mask); in SwapBits()
2193 __ shll(temp, imm_shift); in SwapBits()
2194 __ orl(reg, temp); in SwapBits()
2213 __ bswapl(reg); in VisitIntegerReverse()
2231 __ movq(temp_mask, Immediate(mask)); in SwapBits64()
2232 __ movq(temp, reg); in SwapBits64()
2233 __ shrq(reg, imm_shift); in SwapBits64()
2234 __ andq(temp, temp_mask); in SwapBits64()
2235 __ andq(reg, temp_mask); in SwapBits64()
2236 __ shlq(temp, imm_shift); in SwapBits64()
2237 __ orq(reg, temp); in SwapBits64()
2257 __ bswapq(reg); in VisitLongReverse()
2296 __ popcntq(out, src.AsRegister<CpuRegister>()); in GenBitCount()
2298 __ popcntl(out, src.AsRegister<CpuRegister>()); in GenBitCount()
2302 __ popcntq(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenBitCount()
2305 __ popcntl(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenBitCount()
2346 __ xorl(out, out); // Clears upper bits too. in GenOneBit()
2368 __ blsi(out, src.AsRegister<CpuRegister>()); in GenOneBit()
2375 __ bsrq(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2377 __ bsrl(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2381 __ bsrq(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2384 __ bsrl(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2388 __ j(kEqual, &is_zero); in GenOneBit()
2389 __ movl(out, Immediate(1)); // Clears upper bits too. in GenOneBit()
2391 __ shlq(out, tmp); in GenOneBit()
2393 __ shll(out, tmp); in GenOneBit()
2395 __ jmp(&done); in GenOneBit()
2396 __ Bind(&is_zero); in GenOneBit()
2397 __ xorl(out, out); // Clears upper bits too. in GenOneBit()
2398 __ Bind(&done); in GenOneBit()
2403 __ movq(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2405 __ movl(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2409 __ movq(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2412 __ movl(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2416 __ movq(out, tmp); in GenOneBit()
2417 __ negq(tmp); in GenOneBit()
2418 __ andq(out, tmp); in GenOneBit()
2420 __ movl(out, tmp); in GenOneBit()
2421 __ negl(tmp); in GenOneBit()
2422 __ andl(out, tmp); in GenOneBit()
2490 __ bsrq(out, src.AsRegister<CpuRegister>()); in GenLeadingZeros()
2492 __ bsrl(out, src.AsRegister<CpuRegister>()); in GenLeadingZeros()
2496 __ bsrq(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenLeadingZeros()
2499 __ bsrl(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenLeadingZeros()
2504 __ j(kEqual, &is_zero); in GenLeadingZeros()
2507 __ xorl(out, Immediate(zero_value_result - 1)); in GenLeadingZeros()
2508 __ jmp(&done); in GenLeadingZeros()
2511 __ Bind(&is_zero); in GenLeadingZeros()
2512 __ movl(out, Immediate(zero_value_result)); in GenLeadingZeros()
2514 __ Bind(&done); in GenLeadingZeros()
2563 __ bsfq(out, src.AsRegister<CpuRegister>()); in GenTrailingZeros()
2565 __ bsfl(out, src.AsRegister<CpuRegister>()); in GenTrailingZeros()
2569 __ bsfq(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenTrailingZeros()
2572 __ bsfl(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenTrailingZeros()
2577 __ j(kNotEqual, &done); in GenTrailingZeros()
2580 __ movl(out, Immediate(zero_value_result)); in GenTrailingZeros()
2582 __ Bind(&done); in GenTrailingZeros()
2632 __ movl(Address(out, info.value_offset), Immediate(value)); in VisitIntegerValueOf()
2638 __ leal(out, Address(in, -info.low)); in VisitIntegerValueOf()
2639 __ cmpl(out, Immediate(info.length)); in VisitIntegerValueOf()
2641 __ j(kAboveEqual, &allocate); in VisitIntegerValueOf()
2647 __ movl(out, Address(argument, out, TIMES_4, 0)); in VisitIntegerValueOf()
2648 __ MaybeUnpoisonHeapReference(out); in VisitIntegerValueOf()
2649 __ jmp(&done); in VisitIntegerValueOf()
2650 __ Bind(&allocate); in VisitIntegerValueOf()
2654 __ movl(Address(out, info.value_offset), in); in VisitIntegerValueOf()
2655 __ Bind(&done); in VisitIntegerValueOf()
2671 __ gs()->movl(out, address); in VisitThreadInterrupted()
2672 __ testl(out, out); in VisitThreadInterrupted()
2673 __ j(kEqual, &done); in VisitThreadInterrupted()
2674 __ gs()->movl(address, Immediate(0)); in VisitThreadInterrupted()
2676 __ Bind(&done); in VisitThreadInterrupted()
2711 __ testl(second_reg, second_reg); in VisitIntegerDivideUnsigned()
2714 __ j(kEqual, slow_path->GetEntryLabel()); in VisitIntegerDivideUnsigned()
2716 __ xorl(rdx, rdx); in VisitIntegerDivideUnsigned()
2717 __ divl(second_reg); in VisitIntegerDivideUnsigned()
2719 __ Bind(slow_path->GetExitLabel()); in VisitIntegerDivideUnsigned()
2805 #undef __