Lines Matching refs:__
39 #define __ assembler->GetVIXLAssembler()-> macro
89 __ Add(base, array, element_size * constant + data_offset); in GenSystemArrayCopyBaseAddress()
91 __ Add(base, array, Operand(RegisterFrom(pos), vixl32::LSL, element_size_shift)); in GenSystemArrayCopyBaseAddress()
92 __ Add(base, base, data_offset); in GenSystemArrayCopyBaseAddress()
111 __ Add(end, base, element_size * constant); in GenSystemArrayCopyEndAddress()
113 __ Add(end, base, Operand(RegisterFrom(copy_length), vixl32::LSL, element_size_shift)); in GenSystemArrayCopyEndAddress()
147 __ Bind(GetEntryLabel()); in EmitNativeCode()
152 __ Bind(&loop); in EmitNativeCode()
153 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex)); in EmitNativeCode()
180 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex)); in EmitNativeCode()
181 __ Cmp(src_curr_addr, src_stop_addr); in EmitNativeCode()
182 __ B(ne, &loop, /* is_far_target= */ false); in EmitNativeCode()
183 __ B(GetExitLabel()); in EmitNativeCode()
227 __ Vmov(LowRegisterFrom(output), HighRegisterFrom(output), DRegisterFrom(input)); in MoveFPToInt()
229 __ Vmov(RegisterFrom(output), SRegisterFrom(input)); in MoveFPToInt()
237 __ Vmov(DRegisterFrom(output), LowRegisterFrom(input), HighRegisterFrom(input)); in MoveIntToFP()
239 __ Vmov(SRegisterFrom(output), RegisterFrom(input)); in MoveIntToFP()
307 __ Clz(out, in_reg_hi); in GenNumberOfLeadingZeros()
308 __ CompareAndBranchIfNonZero(in_reg_hi, final_label, /* is_far_target= */ false); in GenNumberOfLeadingZeros()
309 __ Clz(out, in_reg_lo); in GenNumberOfLeadingZeros()
310 __ Add(out, out, 32); in GenNumberOfLeadingZeros()
312 __ Bind(&end); in GenNumberOfLeadingZeros()
315 __ Clz(out, RegisterFrom(in)); in GenNumberOfLeadingZeros()
349 __ Rbit(out, in_reg_lo); in GenNumberOfTrailingZeros()
350 __ Clz(out, out); in GenNumberOfTrailingZeros()
351 __ CompareAndBranchIfNonZero(in_reg_lo, final_label, /* is_far_target= */ false); in GenNumberOfTrailingZeros()
352 __ Rbit(out, in_reg_hi); in GenNumberOfTrailingZeros()
353 __ Clz(out, out); in GenNumberOfTrailingZeros()
354 __ Add(out, out, 32); in GenNumberOfTrailingZeros()
356 __ Bind(&end); in GenNumberOfTrailingZeros()
360 __ Rbit(out, in); in GenNumberOfTrailingZeros()
361 __ Clz(out, out); in GenNumberOfTrailingZeros()
387 __ Vsqrt(OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); in VisitMathSqrt()
399 __ Vrintn(F64, OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); in VisitMathRint()
424 __ Vcvta(S32, F32, temp1, in_reg); in VisitMathRoundFloat()
425 __ Vmov(out_reg, temp1); in VisitMathRoundFloat()
428 __ Cmp(out_reg, 0); in VisitMathRoundFloat()
429 __ B(ge, final_label, /* is_far_target= */ false); in VisitMathRoundFloat()
434 __ Vrinta(F32, temp1, in_reg); in VisitMathRoundFloat()
435 __ Vmov(temp2, 0.5); in VisitMathRoundFloat()
436 __ Vsub(F32, temp1, in_reg, temp1); in VisitMathRoundFloat()
437 __ Vcmp(F32, temp1, temp2); in VisitMathRoundFloat()
438 __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); in VisitMathRoundFloat()
444 __ it(eq); in VisitMathRoundFloat()
445 __ add(eq, out_reg, out_reg, 1); in VisitMathRoundFloat()
449 __ Bind(&done); in VisitMathRoundFloat()
460 __ Ldrsb(OutputRegister(invoke), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPeekByte()
470 __ Ldr(OutputRegister(invoke), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPeekIntNative()
486 __ Ldr(hi, MemOperand(addr, 4)); in VisitMemoryPeekLongNative()
487 __ Ldr(lo, MemOperand(addr)); in VisitMemoryPeekLongNative()
489 __ Ldr(lo, MemOperand(addr)); in VisitMemoryPeekLongNative()
490 __ Ldr(hi, MemOperand(addr, 4)); in VisitMemoryPeekLongNative()
501 __ Ldrsh(OutputRegister(invoke), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPeekShortNative()
517 __ Strb(InputRegisterAt(invoke, 1), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPokeByte()
526 __ Str(InputRegisterAt(invoke, 1), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPokeIntNative()
539 __ Str(LowRegisterFrom(invoke->GetLocations()->InAt(1)), MemOperand(addr)); in VisitMemoryPokeLongNative()
540 __ Str(HighRegisterFrom(invoke->GetLocations()->InAt(1)), MemOperand(addr, 4)); in VisitMemoryPokeLongNative()
549 __ Strh(InputRegisterAt(invoke, 1), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPokeShortNative()
560 __ Ldr(OutputRegister(invoke), in VisitThreadCurrentThread()
579 __ Ldr(trg, MemOperand(base, offset)); in GenUnsafeGet()
581 __ Dmb(vixl32::ISH); in GenUnsafeGet()
592 __ Add(RegisterFrom(temp), base, Operand(offset)); in GenUnsafeGet()
597 __ Dmb(vixl32::ISH); in GenUnsafeGet()
600 __ Ldr(trg, MemOperand(base, offset)); in GenUnsafeGet()
602 __ Dmb(vixl32::ISH); in GenUnsafeGet()
607 __ Ldr(trg, MemOperand(base, offset)); in GenUnsafeGet()
609 __ Dmb(vixl32::ISH); in GenUnsafeGet()
622 __ Add(temp_reg, base, offset); in GenUnsafeGet()
623 __ Ldrexd(trg_lo, trg_hi, MemOperand(temp_reg)); in GenUnsafeGet()
625 __ Ldrd(trg_lo, trg_hi, MemOperand(base, offset)); in GenUnsafeGet()
628 __ Dmb(vixl32::ISH); in GenUnsafeGet()
778 __ Dmb(vixl32::ISH); in GenUnsafePut()
791 __ Add(temp_reg, base, offset); in GenUnsafePut()
793 __ Bind(&loop_head); in GenUnsafePut()
794 __ Ldrexd(temp_lo, temp_hi, MemOperand(temp_reg)); in GenUnsafePut()
795 __ Strexd(temp_lo, value_lo, value_hi, MemOperand(temp_reg)); in GenUnsafePut()
796 __ Cmp(temp_lo, 0); in GenUnsafePut()
797 __ B(ne, &loop_head, /* is_far_target= */ false); in GenUnsafePut()
799 __ Strd(value_lo, value_hi, MemOperand(base, offset)); in GenUnsafePut()
806 __ Mov(temp, value); in GenUnsafePut()
810 __ Str(source, MemOperand(base, offset)); in GenUnsafePut()
814 __ Dmb(vixl32::ISH); in GenUnsafePut()
927 __ Bind(GetEntryLabel()); in EmitNativeCode()
942 __ Cmp(tmp_ptr, expected); in EmitNativeCode()
943 __ B(ne, GetExitLabel()); in EmitNativeCode()
955 __ Add(tmp_ptr, base, offset); in EmitNativeCode()
956 __ Mov(adjusted_old_value, tmp); in EmitNativeCode()
964 __ Bind(&loop_head); in EmitNativeCode()
965 __ Ldrex(tmp, MemOperand(tmp_ptr)); // This can now load null stored by another thread. in EmitNativeCode()
967 __ Subs(tmp, tmp, expected); // Use SUBS to get non-zero value if both compares fail. in EmitNativeCode()
971 __ it(ne); in EmitNativeCode()
972 __ cmp(ne, tmp, adjusted_old_value); in EmitNativeCode()
974 __ B(ne, GetExitLabel()); in EmitNativeCode()
976 __ Strex(tmp, value, MemOperand(tmp_ptr)); in EmitNativeCode()
978 __ Cmp(tmp, 0); in EmitNativeCode()
979 __ B(ne, &loop_head, /* is_far_target= */ false); in EmitNativeCode()
980 __ B(GetExitLabel()); in EmitNativeCode()
1029 __ Dmb(vixl32::ISH); in GenCas()
1031 __ Add(tmp_ptr, base, offset); in GenCas()
1039 __ Bind(&loop_head); in GenCas()
1040 __ Ldrex(tmp, MemOperand(tmp_ptr)); in GenCas()
1044 __ Subs(tmp, tmp, expected); in GenCas()
1050 __ Strex(tmp, value, MemOperand(tmp_ptr)); in GenCas()
1054 __ Cmp(tmp, 0); in GenCas()
1055 __ B(ne, &loop_head, /* is_far_target= */ false); in GenCas()
1057 __ Bind(loop_exit); in GenCas()
1059 __ Dmb(vixl32::ISH); in GenCas()
1062 __ Clz(out, tmp); in GenCas()
1063 __ Lsr(out, out, WhichPowerOf2(out.GetSizeInBits())); in GenCas()
1155 __ CompareAndBranchIfZero(arg, slow_path->GetEntryLabel()); in VisitStringCompareTo()
1159 __ Subs(out, str, arg); in VisitStringCompareTo()
1160 __ B(eq, &end); in VisitStringCompareTo()
1164 __ Ldr(temp3, MemOperand(str, count_offset)); in VisitStringCompareTo()
1165 __ Ldr(temp2, MemOperand(arg, count_offset)); in VisitStringCompareTo()
1167 __ Lsr(temp0, temp3, 1u); in VisitStringCompareTo()
1168 __ Lsr(temp1, temp2, 1u); in VisitStringCompareTo()
1171 __ Ldr(temp0, MemOperand(str, count_offset)); in VisitStringCompareTo()
1172 __ Ldr(temp1, MemOperand(arg, count_offset)); in VisitStringCompareTo()
1175 __ Subs(out, temp0, temp1); in VisitStringCompareTo()
1183 __ it(gt); in VisitStringCompareTo()
1184 __ mov(gt, temp0, temp1); in VisitStringCompareTo()
1190 __ CompareAndBranchIfZero(temp0, &end, mirror::kUseStringCompression); in VisitStringCompareTo()
1194 __ Eors(temp2, temp2, temp3); in VisitStringCompareTo()
1195 __ Lsrs(temp2, temp2, 1u); in VisitStringCompareTo()
1196 __ B(cs, &different_compression); in VisitStringCompareTo()
1199 __ Lsls(temp3, temp3, 31u); // Extract purely the compression flag. in VisitStringCompareTo()
1205 __ it(ne); in VisitStringCompareTo()
1206 __ add(ne, temp0, temp0, temp0); in VisitStringCompareTo()
1212 __ Bind(&end); in VisitStringCompareTo()
1215 __ Bind(slow_path->GetExitLabel()); in VisitStringCompareTo()
1242 __ Mov(temp1, value_offset); in GenerateStringCompareToLoop()
1256 __ Bind(&loop); in GenerateStringCompareToLoop()
1258 __ Ldr(temp_reg, MemOperand(str, temp1)); in GenerateStringCompareToLoop()
1259 __ Ldr(temp2, MemOperand(arg, temp1)); in GenerateStringCompareToLoop()
1260 __ Cmp(temp_reg, temp2); in GenerateStringCompareToLoop()
1261 __ B(ne, &find_char_diff, /* is_far_target= */ false); in GenerateStringCompareToLoop()
1262 __ Add(temp1, temp1, char_size * 2); in GenerateStringCompareToLoop()
1264 __ Ldr(temp_reg, MemOperand(str, temp1)); in GenerateStringCompareToLoop()
1265 __ Ldr(temp2, MemOperand(arg, temp1)); in GenerateStringCompareToLoop()
1266 __ Cmp(temp_reg, temp2); in GenerateStringCompareToLoop()
1267 __ B(ne, &find_char_diff_2nd_cmp, /* is_far_target= */ false); in GenerateStringCompareToLoop()
1268 __ Add(temp1, temp1, char_size * 2); in GenerateStringCompareToLoop()
1270 __ Subs(temp0, temp0, (mirror::kUseStringCompression ? 8 : 4)); in GenerateStringCompareToLoop()
1271 __ B(hi, &loop, /* is_far_target= */ false); in GenerateStringCompareToLoop()
1272 __ B(end); in GenerateStringCompareToLoop()
1274 __ Bind(&find_char_diff_2nd_cmp); in GenerateStringCompareToLoop()
1276 __ Subs(temp0, temp0, 4); // 4 bytes previously compared. in GenerateStringCompareToLoop()
1277 __ B(ls, end, /* is_far_target= */ false); // Was the second comparison fully beyond the end? in GenerateStringCompareToLoop()
1281 __ Sub(temp0, temp0, 2); in GenerateStringCompareToLoop()
1285 __ Bind(&find_char_diff); in GenerateStringCompareToLoop()
1287 __ Eor(temp1, temp2, temp_reg); in GenerateStringCompareToLoop()
1288 __ Rbit(temp1, temp1); in GenerateStringCompareToLoop()
1289 __ Clz(temp1, temp1); in GenerateStringCompareToLoop()
1304 __ Cmp(temp0, Operand(temp1, vixl32::LSR, (mirror::kUseStringCompression ? 3 : 4))); in GenerateStringCompareToLoop()
1305 __ B((mirror::kUseStringCompression ? ls : le), end, /* is_far_target= */ false); in GenerateStringCompareToLoop()
1312 __ Orr(temp3, temp3, 0xffu << 23); // uncompressed ? 0xff800000u : 0x7ff80000u in GenerateStringCompareToLoop()
1313 __ Bic(temp1, temp1, Operand(temp3, vixl32::LSR, 31 - 3)); // &= ~(uncompressed ? 0xfu : 0x7u) in GenerateStringCompareToLoop()
1314 __ Asr(temp3, temp3, 7u); // uncompressed ? 0xffff0000u : 0xff0000u. in GenerateStringCompareToLoop()
1315 __ Lsr(temp2, temp2, temp1); // Extract second character. in GenerateStringCompareToLoop()
1316 __ Lsr(temp3, temp3, 16u); // uncompressed ? 0xffffu : 0xffu in GenerateStringCompareToLoop()
1317 __ Lsr(out, temp_reg, temp1); // Extract first character. in GenerateStringCompareToLoop()
1318 __ And(temp2, temp2, temp3); in GenerateStringCompareToLoop()
1319 __ And(out, out, temp3); in GenerateStringCompareToLoop()
1321 __ Bic(temp1, temp1, 0xf); in GenerateStringCompareToLoop()
1322 __ Lsr(temp2, temp2, temp1); in GenerateStringCompareToLoop()
1323 __ Lsr(out, temp_reg, temp1); in GenerateStringCompareToLoop()
1324 __ Movt(temp2, 0); in GenerateStringCompareToLoop()
1325 __ Movt(out, 0); in GenerateStringCompareToLoop()
1328 __ Sub(out, out, temp2); in GenerateStringCompareToLoop()
1332 __ B(end); in GenerateStringCompareToLoop()
1333 __ Bind(different_compression); in GenerateStringCompareToLoop()
1344 __ Add(temp0, temp0, temp0); // Unlike LSL, this ADD is always 16-bit. in GenerateStringCompareToLoop()
1346 __ Mov(temp1, str); in GenerateStringCompareToLoop()
1347 __ Mov(temp2, arg); in GenerateStringCompareToLoop()
1348 __ Lsrs(temp3, temp3, 1u); // Continue the move of the compression flag. in GenerateStringCompareToLoop()
1353 __ itt(cs); // Interleave with selection of temp1 and temp2. in GenerateStringCompareToLoop()
1354 __ mov(cs, temp1, arg); // Preserves flags. in GenerateStringCompareToLoop()
1355 __ mov(cs, temp2, str); // Preserves flags. in GenerateStringCompareToLoop()
1357 __ Sbc(temp0, temp0, 0); // Complete the move of the compression flag. in GenerateStringCompareToLoop()
1360 __ Add(temp1, temp1, value_offset); in GenerateStringCompareToLoop()
1361 __ Add(temp2, temp2, value_offset); in GenerateStringCompareToLoop()
1368 __ Bind(&different_compression_loop); in GenerateStringCompareToLoop()
1369 __ Ldrb(temp_reg, MemOperand(temp1, c_char_size, PostIndex)); in GenerateStringCompareToLoop()
1370 __ Ldrh(temp3, MemOperand(temp2, char_size, PostIndex)); in GenerateStringCompareToLoop()
1371 __ Cmp(temp_reg, temp3); in GenerateStringCompareToLoop()
1372 __ B(ne, &different_compression_diff, /* is_far_target= */ false); in GenerateStringCompareToLoop()
1373 __ Subs(temp0, temp0, 2); in GenerateStringCompareToLoop()
1374 __ B(hi, &different_compression_loop, /* is_far_target= */ false); in GenerateStringCompareToLoop()
1375 __ B(end); in GenerateStringCompareToLoop()
1378 __ Bind(&different_compression_diff); in GenerateStringCompareToLoop()
1379 __ Sub(out, temp_reg, temp3); in GenerateStringCompareToLoop()
1383 __ Lsrs(temp0, temp0, 1u); in GenerateStringCompareToLoop()
1390 __ it(cc); in GenerateStringCompareToLoop()
1391 __ rsb(cc, out, out, 0); in GenerateStringCompareToLoop()
1470 __ CompareAndBranchIfZero(arg, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1474 __ Cmp(str, arg); in VisitStringEquals()
1475 __ B(eq, &return_true, /* is_far_target= */ false); in VisitStringEquals()
1487 __ Ldr(temp, MemOperand(str, class_offset)); in VisitStringEquals()
1489 __ Ldr(out, MemOperand(arg, class_offset)); in VisitStringEquals()
1492 __ Cmp(temp, out); in VisitStringEquals()
1493 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1514 __ Ldr(temp, MemOperand(arg, count_offset)); in VisitStringEquals()
1515 __ Cmp(temp, Operand(mirror::String::GetFlaggedCount(const_string_length, is_compressed))); in VisitStringEquals()
1516 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1519 __ Ldr(temp, MemOperand(str, count_offset)); in VisitStringEquals()
1520 __ Ldr(out, MemOperand(arg, count_offset)); in VisitStringEquals()
1523 __ Cmp(temp, out); in VisitStringEquals()
1524 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1544 __ Ldrd(temp, temp1, MemOperand(str, offset)); in VisitStringEquals()
1545 __ Ldrd(temp2, out, MemOperand(arg, offset)); in VisitStringEquals()
1546 __ Cmp(temp, temp2); in VisitStringEquals()
1547 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1548 __ Cmp(temp1, out); in VisitStringEquals()
1549 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1554 __ Ldr(temp, MemOperand(str, offset)); in VisitStringEquals()
1555 __ Ldr(out, MemOperand(arg, offset)); in VisitStringEquals()
1556 __ Cmp(temp, out); in VisitStringEquals()
1557 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1563 __ CompareAndBranchIfZero(temp, &return_true, /* is_far_target= */ false); in VisitStringEquals()
1568 __ Lsrs(temp, temp, 1u); // Extract length and check compression flag. in VisitStringEquals()
1572 __ it(cs); // If uncompressed, in VisitStringEquals()
1573 __ add(cs, temp, temp, temp); // double the byte count. in VisitStringEquals()
1581 __ Mov(temp1, value_offset); in VisitStringEquals()
1584 __ Bind(&loop); in VisitStringEquals()
1585 __ Ldr(out, MemOperand(str, temp1)); in VisitStringEquals()
1586 __ Ldr(temp2, MemOperand(arg, temp1)); in VisitStringEquals()
1587 __ Add(temp1, temp1, Operand::From(sizeof(uint32_t))); in VisitStringEquals()
1588 __ Cmp(out, temp2); in VisitStringEquals()
1589 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1591 __ Subs(temp, temp, mirror::kUseStringCompression ? 4 : 2); in VisitStringEquals()
1592 __ B(hi, &loop, /* is_far_target= */ false); in VisitStringEquals()
1597 __ Bind(&return_true); in VisitStringEquals()
1598 __ Mov(out, 1); in VisitStringEquals()
1599 __ B(final_label); in VisitStringEquals()
1602 __ Bind(&return_false); in VisitStringEquals()
1603 __ Mov(out, 0); in VisitStringEquals()
1606 __ Bind(&end); in VisitStringEquals()
1630 __ B(slow_path->GetEntryLabel()); in GenerateVisitStringIndexOf()
1631 __ Bind(slow_path->GetExitLabel()); in GenerateVisitStringIndexOf()
1637 __ Cmp(char_reg, static_cast<uint32_t>(std::numeric_limits<uint16_t>::max()) + 1); in GenerateVisitStringIndexOf()
1640 __ B(hs, slow_path->GetEntryLabel()); in GenerateVisitStringIndexOf()
1647 __ Mov(tmp_reg, 0); in GenerateVisitStringIndexOf()
1654 __ Bind(slow_path->GetExitLabel()); in GenerateVisitStringIndexOf()
1706 __ Cmp(byte_array, 0); in VisitStringNewStringFromBytes()
1710 __ B(eq, slow_path->GetEntryLabel()); in VisitStringNewStringFromBytes()
1714 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromBytes()
1749 __ Cmp(string_to_copy, 0); in VisitStringNewStringFromString()
1753 __ B(eq, slow_path->GetEntryLabel()); in VisitStringNewStringFromString()
1758 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromString()
1811 __ Ldr(temp, MemOperand(input, length_offset)); in CheckPosition()
1813 __ Cmp(temp, Int32ConstantFrom(length)); in CheckPosition()
1815 __ Cmp(temp, RegisterFrom(length)); in CheckPosition()
1817 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1821 __ Ldr(temp, MemOperand(input, length_offset)); in CheckPosition()
1822 __ Subs(temp, temp, pos_const); in CheckPosition()
1823 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1827 __ Cmp(temp, Int32ConstantFrom(length)); in CheckPosition()
1829 __ Cmp(temp, RegisterFrom(length)); in CheckPosition()
1831 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1836 __ CompareAndBranchIfNonZero(pos_reg, slow_path->GetEntryLabel()); in CheckPosition()
1840 __ Cmp(pos_reg, 0); in CheckPosition()
1841 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1844 __ Ldr(temp, MemOperand(input, length_offset)); in CheckPosition()
1845 __ Subs(temp, temp, pos_reg); in CheckPosition()
1846 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1850 __ Cmp(temp, Int32ConstantFrom(length)); in CheckPosition()
1852 __ Cmp(temp, RegisterFrom(length)); in CheckPosition()
1854 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1901 __ Cmp(src, dest); in VisitSystemArrayCopy()
1902 __ B(eq, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1910 __ Cmp(src, dest); in VisitSystemArrayCopy()
1911 __ B(ne, &conditions_on_positions_validated, /* is_far_target= */ false); in VisitSystemArrayCopy()
1913 __ Cmp(RegisterFrom(dest_pos), src_pos_constant); in VisitSystemArrayCopy()
1914 __ B(gt, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1918 __ Cmp(src, dest); in VisitSystemArrayCopy()
1919 __ B(ne, &conditions_on_positions_validated, /* is_far_target= */ false); in VisitSystemArrayCopy()
1923 __ Cmp(RegisterFrom(src_pos), dest_pos_constant); in VisitSystemArrayCopy()
1925 __ Cmp(RegisterFrom(src_pos), RegisterFrom(dest_pos)); in VisitSystemArrayCopy()
1927 __ B(lt, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1930 __ Bind(&conditions_on_positions_validated); in VisitSystemArrayCopy()
1934 __ CompareAndBranchIfZero(src, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1939 __ CompareAndBranchIfZero(dest, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1947 __ Cmp(RegisterFrom(length), 0); in VisitSystemArrayCopy()
1948 __ B(lt, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1984 __ CompareAndBranchIfZero(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1988 __ Ldrh(temp1, MemOperand(temp1, primitive_offset)); in VisitSystemArrayCopy()
1990 __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2008 __ CompareAndBranchIfZero(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2012 __ Ldrh(temp2, MemOperand(temp2, primitive_offset)); in VisitSystemArrayCopy()
2014 __ CompareAndBranchIfNonZero(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2023 __ Cmp(temp1, temp2); in VisitSystemArrayCopy()
2027 __ B(eq, &do_copy, /* is_far_target= */ false); in VisitSystemArrayCopy()
2036 __ Ldr(temp1, MemOperand(temp1, super_offset)); in VisitSystemArrayCopy()
2037 __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2038 __ Bind(&do_copy); in VisitSystemArrayCopy()
2040 __ B(ne, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2046 __ Ldr(temp1, MemOperand(dest, class_offset)); in VisitSystemArrayCopy()
2048 __ Ldr(temp2, MemOperand(src, class_offset)); in VisitSystemArrayCopy()
2062 __ Ldr(temp3, MemOperand(temp1, component_offset)); in VisitSystemArrayCopy()
2063 __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2066 __ Ldrh(temp3, MemOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
2068 __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2074 __ Ldr(temp3, MemOperand(temp2, component_offset)); in VisitSystemArrayCopy()
2075 __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2078 __ Ldrh(temp3, MemOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
2080 __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2083 __ Cmp(temp1, temp2); in VisitSystemArrayCopy()
2087 __ B(eq, &do_copy, /* is_far_target= */ false); in VisitSystemArrayCopy()
2092 __ Ldr(temp1, MemOperand(temp1, component_offset)); in VisitSystemArrayCopy()
2095 __ Ldr(temp1, MemOperand(temp1, super_offset)); in VisitSystemArrayCopy()
2097 __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2098 __ Bind(&do_copy); in VisitSystemArrayCopy()
2100 __ B(ne, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2113 __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2118 __ Ldr(temp1, MemOperand(src, class_offset)); in VisitSystemArrayCopy()
2121 __ Ldr(temp3, MemOperand(temp1, component_offset)); in VisitSystemArrayCopy()
2122 __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2126 __ Ldrh(temp3, MemOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
2128 __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2140 __ CompareAndBranchIfZero(RegisterFrom(length), &done, /* is_far_target= */ false); in VisitSystemArrayCopy()
2165 __ Ldr(temp2, MemOperand(src, monitor_offset)); in VisitSystemArrayCopy()
2175 __ Add(src, src, Operand(temp2, vixl32::LSR, 32)); in VisitSystemArrayCopy()
2199 __ Lsrs(temp2, temp2, LockWord::kReadBarrierStateShift + 1); in VisitSystemArrayCopy()
2201 __ B(cs, read_barrier_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2209 __ Bind(&loop); in VisitSystemArrayCopy()
2213 __ Ldr(temp_reg, MemOperand(temp1, element_size, PostIndex)); in VisitSystemArrayCopy()
2214 __ Str(temp_reg, MemOperand(temp2, element_size, PostIndex)); in VisitSystemArrayCopy()
2216 __ Cmp(temp1, temp3); in VisitSystemArrayCopy()
2217 __ B(ne, &loop, /* is_far_target= */ false); in VisitSystemArrayCopy()
2219 __ Bind(read_barrier_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
2231 __ Bind(&loop); in VisitSystemArrayCopy()
2235 __ Ldr(temp_reg, MemOperand(temp1, element_size, PostIndex)); in VisitSystemArrayCopy()
2236 __ Str(temp_reg, MemOperand(temp2, element_size, PostIndex)); in VisitSystemArrayCopy()
2238 __ Cmp(temp1, temp3); in VisitSystemArrayCopy()
2239 __ B(ne, &loop, /* is_far_target= */ false); in VisitSystemArrayCopy()
2241 __ Bind(&done); in VisitSystemArrayCopy()
2247 __ Bind(intrinsic_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
2314 __ Vmov(RegisterFrom(locations->GetTemp(0)), in GenFPToFPCall()
2318 __ Vmov(OutputDRegister(invoke), in GenFPToFPCall()
2333 __ Vmov(RegisterFrom(locations->GetTemp(0)), in GenFPFPToFPCall()
2336 __ Vmov(RegisterFrom(locations->GetTemp(2)), in GenFPFPToFPCall()
2340 __ Vmov(OutputDRegister(invoke), in GenFPFPToFPCall()
2495 __ Rbit(OutputRegister(invoke), InputRegisterAt(invoke, 0)); in VisitIntegerReverse()
2511 __ Rbit(out_reg_lo, in_reg_hi); in VisitLongReverse()
2512 __ Rbit(out_reg_hi, in_reg_lo); in VisitLongReverse()
2521 __ Rev(OutputRegister(invoke), InputRegisterAt(invoke, 0)); in VisitIntegerReverseBytes()
2537 __ Rev(out_reg_lo, in_reg_hi); in VisitLongReverseBytes()
2538 __ Rev(out_reg_hi, in_reg_lo); in VisitLongReverseBytes()
2547 __ Revsh(OutputRegister(invoke), InputRegisterAt(invoke, 0)); in VisitShortReverseBytes()
2568 __ Vmov(tmp_d, src_1, src_0); // Temp DReg |--src_1|--src_0| in GenBitCount()
2569 __ Vcnt(Untyped8, tmp_d, tmp_d); // Temp DReg |c|c|c|c|c|c|c|c| in GenBitCount()
2570 __ Vpaddl(U8, tmp_d, tmp_d); // Temp DReg |--c|--c|--c|--c| in GenBitCount()
2571 __ Vpaddl(U16, tmp_d, tmp_d); // Temp DReg |------c|------c| in GenBitCount()
2573 __ Vpaddl(U32, tmp_d, tmp_d); // Temp DReg |--------------c| in GenBitCount()
2575 __ Vmov(out_r, tmp_s); in GenBitCount()
2614 __ Mov(temp, 0x80000000); // Modified immediate. in GenHighestOneBit()
2615 __ Clz(out_reg_lo, in_reg_lo); in GenHighestOneBit()
2616 __ Clz(out_reg_hi, in_reg_hi); in GenHighestOneBit()
2617 __ Lsr(out_reg_lo, temp, out_reg_lo); in GenHighestOneBit()
2618 __ Lsrs(out_reg_hi, temp, out_reg_hi); in GenHighestOneBit()
2627 __ Mov(LeaveFlags, temp, 0); in GenHighestOneBit()
2633 __ it(ne); in GenHighestOneBit()
2634 __ mov(ne, out_reg_lo, mov_src); in GenHighestOneBit()
2639 __ Mov(temp, 0x80000000); // Modified immediate. in GenHighestOneBit()
2640 __ Clz(out, in); in GenHighestOneBit()
2641 __ Lsr(out, temp, out); in GenHighestOneBit()
2680 __ Rsb(out_reg_hi, in_reg_hi, 0); in GenLowestOneBit()
2681 __ Rsb(out_reg_lo, in_reg_lo, 0); in GenLowestOneBit()
2682 __ And(out_reg_hi, out_reg_hi, in_reg_hi); in GenLowestOneBit()
2684 __ Ands(out_reg_lo, out_reg_lo, in_reg_lo); in GenLowestOneBit()
2693 __ Mov(LeaveFlags, temp, 0); in GenLowestOneBit()
2699 __ it(ne); in GenLowestOneBit()
2700 __ mov(ne, out_reg_hi, mov_src); in GenLowestOneBit()
2705 __ Rsb(temp, in, 0); in GenLowestOneBit()
2706 __ And(out, temp, in); in GenLowestOneBit()
2770 __ Add(dst_ptr, dstObj, data_offset); in VisitStringGetCharsNoCheck()
2771 __ Add(dst_ptr, dst_ptr, Operand(dstBegin, vixl32::LSL, 1)); in VisitStringGetCharsNoCheck()
2773 __ Subs(num_chr, srcEnd, srcBegin); in VisitStringGetCharsNoCheck()
2775 __ B(eq, final_label, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2778 __ Add(src_ptr, srcObj, value_offset); in VisitStringGetCharsNoCheck()
2788 __ Ldr(temp, MemOperand(srcObj, count_offset)); in VisitStringGetCharsNoCheck()
2789 __ Tst(temp, 1); in VisitStringGetCharsNoCheck()
2791 __ B(eq, &compressed_string_preloop, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2793 __ Add(src_ptr, src_ptr, Operand(srcBegin, vixl32::LSL, 1)); in VisitStringGetCharsNoCheck()
2800 __ Subs(temp, num_chr, 4); in VisitStringGetCharsNoCheck()
2801 __ B(lt, &remainder, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2804 __ Mov(num_chr, temp); in VisitStringGetCharsNoCheck()
2809 __ Bind(&loop); in VisitStringGetCharsNoCheck()
2810 __ Ldr(temp, MemOperand(src_ptr, char_size * 2)); in VisitStringGetCharsNoCheck()
2811 __ Subs(num_chr, num_chr, 4); in VisitStringGetCharsNoCheck()
2812 __ Str(temp, MemOperand(dst_ptr, char_size * 2)); in VisitStringGetCharsNoCheck()
2813 __ Ldr(temp, MemOperand(src_ptr, char_size * 4, PostIndex)); in VisitStringGetCharsNoCheck()
2814 __ Str(temp, MemOperand(dst_ptr, char_size * 4, PostIndex)); in VisitStringGetCharsNoCheck()
2816 __ B(ge, &loop, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2818 __ Adds(num_chr, num_chr, 4); in VisitStringGetCharsNoCheck()
2819 __ B(eq, final_label, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2823 __ Bind(&remainder); in VisitStringGetCharsNoCheck()
2825 __ Ldrh(temp, MemOperand(src_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2826 __ Subs(num_chr, num_chr, 1); in VisitStringGetCharsNoCheck()
2827 __ Strh(temp, MemOperand(dst_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2829 __ B(gt, &remainder, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2832 __ B(final_label); in VisitStringGetCharsNoCheck()
2837 __ Bind(&compressed_string_preloop); in VisitStringGetCharsNoCheck()
2838 __ Add(src_ptr, src_ptr, srcBegin); in VisitStringGetCharsNoCheck()
2839 __ Bind(&compressed_string_loop); in VisitStringGetCharsNoCheck()
2841 __ Ldrb(temp, MemOperand(src_ptr, c_char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2842 __ Strh(temp, MemOperand(dst_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2844 __ Subs(num_chr, num_chr, 1); in VisitStringGetCharsNoCheck()
2845 __ B(gt, &compressed_string_loop, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2849 __ Bind(&done); in VisitStringGetCharsNoCheck()
2864 __ Vmov(out, InputSRegisterAt(invoke, 0)); in VisitFloatIsInfinite()
2866 __ Lsl(out, out, 1); in VisitFloatIsInfinite()
2867 __ Eor(out, out, infinity); in VisitFloatIsInfinite()
2889 __ Vmov(temp, out, InputDRegisterAt(invoke, 0)); in VisitDoubleIsInfinite()
2890 __ Eor(out, out, infinity_high); in VisitDoubleIsInfinite()
2891 __ Eor(out, out, infinity_high2); in VisitDoubleIsInfinite()
2893 __ Orr(out, temp, Operand(out, vixl32::LSL, 1)); in VisitDoubleIsInfinite()
2906 __ Vrintp(F64, OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); in VisitMathCeil()
2918 __ Vrintm(F64, OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); in VisitMathFloor()
2952 __ Mov(temp, value); in VisitIntegerValueOf()
2962 __ Add(out, in, -info.low); in VisitIntegerValueOf()
2963 __ Cmp(out, info.length); in VisitIntegerValueOf()
2965 __ B(hs, &allocate, /* is_far_target= */ false); in VisitIntegerValueOf()
2970 __ B(&done); in VisitIntegerValueOf()
2971 __ Bind(&allocate); in VisitIntegerValueOf()
2979 __ Bind(&done); in VisitIntegerValueOf()
2993 __ Ldr(out, MemOperand(tr, offset)); in VisitThreadInterrupted()
2998 __ CompareAndBranchIfZero(out, final_label, /* is_far_target= */ false); in VisitThreadInterrupted()
2999 __ Dmb(vixl32::ISH); in VisitThreadInterrupted()
3000 __ Mov(temp, 0); in VisitThreadInterrupted()
3002 __ Dmb(vixl32::ISH); in VisitThreadInterrupted()
3004 __ Bind(&done); in VisitThreadInterrupted()
3100 #undef __