1 /*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "linker/arm/relative_patcher_thumb2.h"
18
19 #include "arch/arm/instruction_set_features_arm.h"
20 #include "base/casts.h"
21 #include "driver/compiler_options.h"
22 #include "linker/relative_patcher_test.h"
23 #include "lock_word.h"
24 #include "mirror/array-inl.h"
25 #include "mirror/object.h"
26 #include "oat_quick_method_header.h"
27 #include "optimizing/code_generator_arm_vixl.h"
28 #include "optimizing/optimizing_unit_test.h"
29
30 namespace art {
31 namespace linker {
32
33 class Thumb2RelativePatcherTest : public RelativePatcherTest {
34 public:
Thumb2RelativePatcherTest()35 Thumb2RelativePatcherTest() : RelativePatcherTest(InstructionSet::kThumb2, "default") { }
36
37 protected:
38 static const uint8_t kCallRawCode[];
39 static const ArrayRef<const uint8_t> kCallCode;
40 static const uint8_t kNopRawCode[];
41 static const ArrayRef<const uint8_t> kNopCode;
42 static const uint8_t kUnpatchedPcRelativeRawCode[];
43 static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCode;
44 static const uint32_t kPcInsnOffset;
45
46 // The PC in Thumb mode is 4 bytes after the instruction location.
47 static constexpr uint32_t kPcAdjustment = 4u;
48
49 // Branches within range [-256, 256) can be created from these by adding the low 8 bits.
50 static constexpr uint32_t kBlPlus0 = 0xf000f800u;
51 static constexpr uint32_t kBlMinus256 = 0xf7ffff00u;
52
53 // Special BL values.
54 static constexpr uint32_t kBlPlusMax = 0xf3ffd7ffu;
55 static constexpr uint32_t kBlMinusMax = 0xf400d000u;
56
57 // BNE +0, 32-bit, encoding T3. Bits 0-10, 11, 13, 16-21, 26 are placeholder for target offset.
58 static constexpr uint32_t kBneWPlus0 = 0xf0408000u;
59
60 // LDR immediate, 16-bit, encoding T1. Bits 6-10 are imm5, 0-2 are Rt, 3-5 are Rn.
61 static constexpr uint32_t kLdrInsn = 0x6800u;
62
63 // LDR immediate, 32-bit, encoding T3. Bits 0-11 are offset, 12-15 are Rt, 16-20 are Rn.
64 static constexpr uint32_t kLdrWInsn = 0xf8d00000u;
65
66 // LDR immediate, negative offset, encoding T4. Bits 0-7 are the offset to subtract.
67 static constexpr uint32_t kLdrNegativeOffset = 0xf8500c00u;
68
69 // LDR register, lsl #2. Bits 4-5 are the imm2, i.e. the lsl shift.
70 static constexpr uint32_t kLdrRegLsl2 = 0xf8500020u;
71
72 // NOP instructions.
73 static constexpr uint32_t kNopInsn = 0xbf00u;
74 static constexpr uint32_t kNopWInsn = 0xf3af8000u;
75
InsertInsn(std::vector<uint8_t> * code,size_t pos,uint32_t insn)76 void InsertInsn(std::vector<uint8_t>* code, size_t pos, uint32_t insn) {
77 CHECK_LE(pos, code->size());
78 if (IsUint<16>(insn)) {
79 const uint8_t insn_code[] = {
80 static_cast<uint8_t>(insn),
81 static_cast<uint8_t>(insn >> 8),
82 };
83 static_assert(sizeof(insn_code) == 2u, "Invalid sizeof(insn_code).");
84 code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
85 } else {
86 const uint8_t insn_code[] = {
87 static_cast<uint8_t>(insn >> 16),
88 static_cast<uint8_t>(insn >> 24),
89 static_cast<uint8_t>(insn),
90 static_cast<uint8_t>(insn >> 8),
91 };
92 static_assert(sizeof(insn_code) == 4u, "Invalid sizeof(insn_code).");
93 code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
94 }
95 }
96
PushBackInsn(std::vector<uint8_t> * code,uint32_t insn)97 void PushBackInsn(std::vector<uint8_t>* code, uint32_t insn) {
98 InsertInsn(code, code->size(), insn);
99 }
100
GenNops(size_t num_nops)101 std::vector<uint8_t> GenNops(size_t num_nops) {
102 std::vector<uint8_t> result;
103 result.reserve(num_nops * 2u);
104 for (size_t i = 0; i != num_nops; ++i) {
105 PushBackInsn(&result, kNopInsn);
106 }
107 return result;
108 }
109
RawCode(std::initializer_list<uint32_t> insns)110 std::vector<uint8_t> RawCode(std::initializer_list<uint32_t> insns) {
111 std::vector<uint8_t> raw_code;
112 size_t number_of_16_bit_insns =
113 std::count_if(insns.begin(), insns.end(), [](uint32_t x) { return IsUint<16>(x); });
114 raw_code.reserve(insns.size() * 4u - number_of_16_bit_insns * 2u);
115 for (uint32_t insn : insns) {
116 PushBackInsn(&raw_code, insn);
117 }
118 return raw_code;
119 }
120
BneWWithOffset(uint32_t bne_offset,uint32_t target_offset)121 uint32_t BneWWithOffset(uint32_t bne_offset, uint32_t target_offset) {
122 if (!IsAligned<2u>(bne_offset)) {
123 LOG(ERROR) << "Unaligned bne_offset: " << bne_offset;
124 return 0xffffffffu; // Fails code diff later.
125 }
126 if (!IsAligned<2u>(target_offset)) {
127 LOG(ERROR) << "Unaligned target_offset: " << target_offset;
128 return 0xffffffffu; // Fails code diff later.
129 }
130 uint32_t diff = target_offset - bne_offset - kPcAdjustment;
131 DCHECK_ALIGNED(diff, 2u);
132 if ((diff >> 20) != 0 && (diff >> 20) != 0xfffu) {
133 LOG(ERROR) << "Target out of range: " << diff;
134 return 0xffffffffu; // Fails code diff later.
135 }
136 return kBneWPlus0 | ((diff >> 1) & 0x7ffu) // imm11
137 | (((diff >> 12) & 0x3fu) << 16) // imm6
138 | (((diff >> 18) & 1) << 13) // J1
139 | (((diff >> 19) & 1) << 11) // J2
140 | (((diff >> 20) & 1) << 26); // S
141 }
142
Create2MethodsWithGap(const ArrayRef<const uint8_t> & method1_code,const ArrayRef<const LinkerPatch> & method1_patches,const ArrayRef<const uint8_t> & last_method_code,const ArrayRef<const LinkerPatch> & last_method_patches,uint32_t distance_without_thunks)143 uint32_t Create2MethodsWithGap(const ArrayRef<const uint8_t>& method1_code,
144 const ArrayRef<const LinkerPatch>& method1_patches,
145 const ArrayRef<const uint8_t>& last_method_code,
146 const ArrayRef<const LinkerPatch>& last_method_patches,
147 uint32_t distance_without_thunks) {
148 CHECK_EQ(distance_without_thunks % kArmAlignment, 0u);
149 uint32_t method1_offset =
150 kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
151 AddCompiledMethod(MethodRef(1u), method1_code, method1_patches);
152 const uint32_t gap_start = method1_offset + method1_code.size();
153
154 // We want to put the last method at a very precise offset.
155 const uint32_t last_method_offset = method1_offset + distance_without_thunks;
156 CHECK_ALIGNED(last_method_offset, kArmAlignment);
157 const uint32_t gap_end = last_method_offset - sizeof(OatQuickMethodHeader);
158
159 // Fill the gap with intermediate methods in chunks of 2MiB and the first in [2MiB, 4MiB).
160 // (This allows deduplicating the small chunks to avoid using 32MiB of memory for +-16MiB
161 // offsets by this test. Making the first chunk bigger makes it easy to give all intermediate
162 // methods the same alignment of the end, so the thunk insertion adds a predictable size as
163 // long as it's after the first chunk.)
164 uint32_t method_idx = 2u;
165 constexpr uint32_t kSmallChunkSize = 2 * MB;
166 std::vector<uint8_t> gap_code;
167 uint32_t gap_size = gap_end - gap_start;
168 uint32_t num_small_chunks = std::max(gap_size / kSmallChunkSize, 1u) - 1u;
169 uint32_t chunk_start = gap_start;
170 uint32_t chunk_size = gap_size - num_small_chunks * kSmallChunkSize;
171 for (uint32_t i = 0; i <= num_small_chunks; ++i) { // num_small_chunks+1 iterations.
172 uint32_t chunk_code_size =
173 chunk_size - CodeAlignmentSize(chunk_start) - sizeof(OatQuickMethodHeader);
174 gap_code.resize(chunk_code_size, 0u);
175 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(gap_code));
176 method_idx += 1u;
177 chunk_start += chunk_size;
178 chunk_size = kSmallChunkSize; // For all but the first chunk.
179 DCHECK_EQ(CodeAlignmentSize(gap_end), CodeAlignmentSize(chunk_start));
180 }
181
182 // Add the last method and link
183 AddCompiledMethod(MethodRef(method_idx), last_method_code, last_method_patches);
184 Link();
185
186 // Check assumptions.
187 CHECK_EQ(GetMethodOffset(1), method1_offset);
188 auto last_result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
189 CHECK(last_result.first);
190 // There may be a thunk before method2.
191 if (last_result.second != last_method_offset + 1 /* thumb mode */) {
192 // Thunk present. Check that there's only one.
193 uint32_t thunk_end =
194 CompiledCode::AlignCode(gap_end, InstructionSet::kThumb2) + MethodCallThunkSize();
195 uint32_t header_offset = thunk_end + CodeAlignmentSize(thunk_end);
196 CHECK_EQ(last_result.second,
197 header_offset + sizeof(OatQuickMethodHeader) + 1 /* thumb mode */);
198 }
199 return method_idx;
200 }
201
GetMethodOffset(uint32_t method_idx)202 uint32_t GetMethodOffset(uint32_t method_idx) {
203 auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
204 CHECK(result.first);
205 CHECK_NE(result.second & 1u, 0u);
206 return result.second - 1 /* thumb mode */;
207 }
208
CompileThunk(const LinkerPatch & patch,std::string * debug_name=nullptr)209 std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
210 /*out*/ std::string* debug_name = nullptr) {
211 OptimizingUnitTestHelper helper;
212 HGraph* graph = helper.CreateGraph();
213 CompilerOptions compiler_options;
214 arm::CodeGeneratorARMVIXL codegen(graph, compiler_options);
215 ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
216 codegen.EmitThunkCode(patch, &code, debug_name);
217 return std::vector<uint8_t>(code.begin(), code.end());
218 }
219
AddCompiledMethod(MethodReference method_ref,const ArrayRef<const uint8_t> & code,const ArrayRef<const LinkerPatch> & patches=ArrayRef<const LinkerPatch> ())220 void AddCompiledMethod(
221 MethodReference method_ref,
222 const ArrayRef<const uint8_t>& code,
223 const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
224 RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
225
226 // Make sure the ThunkProvider has all the necessary thunks.
227 for (const LinkerPatch& patch : patches) {
228 if (patch.GetType() == LinkerPatch::Type::kCallEntrypoint ||
229 patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
230 patch.GetType() == LinkerPatch::Type::kCallRelative) {
231 std::string debug_name;
232 std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
233 thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
234 }
235 }
236 }
237
CompileMethodCallThunk()238 std::vector<uint8_t> CompileMethodCallThunk() {
239 LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
240 /* target_dex_file*/ nullptr,
241 /* target_method_idx */ 0u);
242 return CompileThunk(patch);
243 }
244
MethodCallThunkSize()245 uint32_t MethodCallThunkSize() {
246 return CompileMethodCallThunk().size();
247 }
248
CheckThunk(uint32_t thunk_offset)249 bool CheckThunk(uint32_t thunk_offset) {
250 const std::vector<uint8_t> expected_code = CompileMethodCallThunk();
251 if (output_.size() < thunk_offset + expected_code.size()) {
252 LOG(ERROR) << "output_.size() == " << output_.size() << " < "
253 << "thunk_offset + expected_code.size() == " << (thunk_offset + expected_code.size());
254 return false;
255 }
256 ArrayRef<const uint8_t> linked_code(&output_[thunk_offset], expected_code.size());
257 if (linked_code == ArrayRef<const uint8_t>(expected_code)) {
258 return true;
259 }
260 // Log failure info.
261 DumpDiff(ArrayRef<const uint8_t>(expected_code), linked_code);
262 return false;
263 }
264
GenNopsAndBl(size_t num_nops,uint32_t bl)265 std::vector<uint8_t> GenNopsAndBl(size_t num_nops, uint32_t bl) {
266 std::vector<uint8_t> result;
267 result.reserve(num_nops * 2u + 4u);
268 for (size_t i = 0; i != num_nops; ++i) {
269 PushBackInsn(&result, kNopInsn);
270 }
271 PushBackInsn(&result, bl);
272 return result;
273 }
274
275 void TestStringBssEntry(uint32_t bss_begin, uint32_t string_entry_offset);
276 void TestStringReference(uint32_t string_offset);
277 void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset);
278
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg,bool narrow)279 static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
280 uint32_t holder_reg,
281 bool narrow) {
282 return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow);
283 }
284
EncodeBakerReadBarrierArrayData(uint32_t base_reg)285 static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
286 return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg);
287 }
288
EncodeBakerReadBarrierGcRootData(uint32_t root_reg,bool narrow)289 static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
290 return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow);
291 }
292
CompileBakerOffsetThunk(uint32_t base_reg,uint32_t holder_reg,bool narrow)293 std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg,
294 uint32_t holder_reg,
295 bool narrow) {
296 const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
297 /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
298 return CompileThunk(patch);
299 }
300
CompileBakerArrayThunk(uint32_t base_reg)301 std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
302 LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
303 /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
304 return CompileThunk(patch);
305 }
306
CompileBakerGcRootThunk(uint32_t root_reg,bool narrow)307 std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) {
308 LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
309 /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow));
310 return CompileThunk(patch);
311 }
312
GetOutputInsn32(uint32_t offset)313 uint32_t GetOutputInsn32(uint32_t offset) {
314 CHECK_LE(offset, output_.size());
315 CHECK_GE(output_.size() - offset, 4u);
316 return (static_cast<uint32_t>(output_[offset]) << 16) |
317 (static_cast<uint32_t>(output_[offset + 1]) << 24) |
318 (static_cast<uint32_t>(output_[offset + 2]) << 0) |
319 (static_cast<uint32_t>(output_[offset + 3]) << 8);
320 }
321
GetOutputInsn16(uint32_t offset)322 uint16_t GetOutputInsn16(uint32_t offset) {
323 CHECK_LE(offset, output_.size());
324 CHECK_GE(output_.size() - offset, 2u);
325 return (static_cast<uint32_t>(output_[offset]) << 0) |
326 (static_cast<uint32_t>(output_[offset + 1]) << 8);
327 }
328
329 void TestBakerFieldWide(uint32_t offset, uint32_t ref_reg);
330 void TestBakerFieldNarrow(uint32_t offset, uint32_t ref_reg);
331 };
332
333 const uint8_t Thumb2RelativePatcherTest::kCallRawCode[] = {
334 0x00, 0xf0, 0x00, 0xf8
335 };
336
337 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kCallCode(kCallRawCode);
338
339 const uint8_t Thumb2RelativePatcherTest::kNopRawCode[] = {
340 0x00, 0xbf
341 };
342
343 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kNopCode(kNopRawCode);
344
345 const uint8_t Thumb2RelativePatcherTest::kUnpatchedPcRelativeRawCode[] = {
346 0x40, 0xf2, 0x00, 0x00, // MOVW r0, #0 (placeholder)
347 0xc0, 0xf2, 0x00, 0x00, // MOVT r0, #0 (placeholder)
348 0x78, 0x44, // ADD r0, pc
349 };
350 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kUnpatchedPcRelativeCode(
351 kUnpatchedPcRelativeRawCode);
352 const uint32_t Thumb2RelativePatcherTest::kPcInsnOffset = 8u;
353
TestStringBssEntry(uint32_t bss_begin,uint32_t string_entry_offset)354 void Thumb2RelativePatcherTest::TestStringBssEntry(uint32_t bss_begin,
355 uint32_t string_entry_offset) {
356 constexpr uint32_t kStringIndex = 1u;
357 string_index_to_offset_map_.Put(kStringIndex, string_entry_offset);
358 bss_begin_ = bss_begin;
359 const LinkerPatch patches[] = {
360 LinkerPatch::StringBssEntryPatch(0u, nullptr, kPcInsnOffset, kStringIndex),
361 LinkerPatch::StringBssEntryPatch(4u, nullptr, kPcInsnOffset, kStringIndex),
362 };
363 CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), bss_begin_ + string_entry_offset);
364 }
365
TestStringReference(uint32_t string_offset)366 void Thumb2RelativePatcherTest::TestStringReference(uint32_t string_offset) {
367 constexpr uint32_t kStringIndex = 1u;
368 string_index_to_offset_map_.Put(kStringIndex, string_offset);
369 const LinkerPatch patches[] = {
370 LinkerPatch::RelativeStringPatch(0u, nullptr, kPcInsnOffset, kStringIndex),
371 LinkerPatch::RelativeStringPatch(4u, nullptr, kPcInsnOffset, kStringIndex),
372 };
373 CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), string_offset);
374 }
375
CheckPcRelativePatch(const ArrayRef<const LinkerPatch> & patches,uint32_t target_offset)376 void Thumb2RelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches,
377 uint32_t target_offset) {
378 AddCompiledMethod(MethodRef(1u), kUnpatchedPcRelativeCode, ArrayRef<const LinkerPatch>(patches));
379 Link();
380
381 uint32_t method1_offset = GetMethodOffset(1u);
382 uint32_t pc_base_offset = method1_offset + kPcInsnOffset + 4u /* PC adjustment */;
383 uint32_t diff = target_offset - pc_base_offset;
384 // Distribute the bits of the diff between the MOVW and MOVT:
385 uint32_t diffw = diff & 0xffffu;
386 uint32_t difft = diff >> 16;
387 uint32_t movw = 0xf2400000u | // MOVW r0, #0 (placeholder),
388 ((diffw & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
389 ((diffw & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
390 ((diffw & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
391 ((diffw & 0x00ffu)); // keep imm8 at bits 0-7.
392 uint32_t movt = 0xf2c00000u | // MOVT r0, #0 (placeholder),
393 ((difft & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
394 ((difft & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
395 ((difft & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
396 ((difft & 0x00ffu)); // keep imm8 at bits 0-7.
397 const uint8_t expected_code[] = {
398 static_cast<uint8_t>(movw >> 16), static_cast<uint8_t>(movw >> 24),
399 static_cast<uint8_t>(movw >> 0), static_cast<uint8_t>(movw >> 8),
400 static_cast<uint8_t>(movt >> 16), static_cast<uint8_t>(movt >> 24),
401 static_cast<uint8_t>(movt >> 0), static_cast<uint8_t>(movt >> 8),
402 0x78, 0x44,
403 };
404 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
405 }
406
TEST_F(Thumb2RelativePatcherTest,CallSelf)407 TEST_F(Thumb2RelativePatcherTest, CallSelf) {
408 const LinkerPatch patches[] = {
409 LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
410 };
411 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
412 Link();
413
414 static const uint8_t expected_code[] = {
415 0xff, 0xf7, 0xfe, 0xff
416 };
417 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
418 }
419
TEST_F(Thumb2RelativePatcherTest,CallOther)420 TEST_F(Thumb2RelativePatcherTest, CallOther) {
421 const LinkerPatch method1_patches[] = {
422 LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
423 };
424 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(method1_patches));
425 const LinkerPatch method2_patches[] = {
426 LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
427 };
428 AddCompiledMethod(MethodRef(2u), kCallCode, ArrayRef<const LinkerPatch>(method2_patches));
429 Link();
430
431 uint32_t method1_offset = GetMethodOffset(1u);
432 uint32_t method2_offset = GetMethodOffset(2u);
433 uint32_t diff_after = method2_offset - (method1_offset + 4u /* PC adjustment */);
434 ASSERT_EQ(diff_after & 1u, 0u);
435 ASSERT_LT(diff_after >> 1, 1u << 8); // Simple encoding, (diff_after >> 1) fits into 8 bits.
436 static const uint8_t method1_expected_code[] = {
437 0x00, 0xf0, static_cast<uint8_t>(diff_after >> 1), 0xf8
438 };
439 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code)));
440 uint32_t diff_before = method1_offset - (method2_offset + 4u /* PC adjustment */);
441 ASSERT_EQ(diff_before & 1u, 0u);
442 ASSERT_GE(diff_before, -1u << 9); // Simple encoding, -256 <= (diff >> 1) < 0.
443 auto method2_expected_code = GenNopsAndBl(0u, kBlMinus256 | ((diff_before >> 1) & 0xffu));
444 EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code)));
445 }
446
TEST_F(Thumb2RelativePatcherTest,CallTrampoline)447 TEST_F(Thumb2RelativePatcherTest, CallTrampoline) {
448 const LinkerPatch patches[] = {
449 LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
450 };
451 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
452 Link();
453
454 uint32_t method1_offset = GetMethodOffset(1u);
455 uint32_t diff = kTrampolineOffset - (method1_offset + 4u);
456 ASSERT_EQ(diff & 1u, 0u);
457 ASSERT_GE(diff, -1u << 9); // Simple encoding, -256 <= (diff >> 1) < 0 (checked as unsigned).
458 auto expected_code = GenNopsAndBl(0u, kBlMinus256 | ((diff >> 1) & 0xffu));
459 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
460 }
461
TEST_F(Thumb2RelativePatcherTest,CallTrampolineTooFar)462 TEST_F(Thumb2RelativePatcherTest, CallTrampolineTooFar) {
463 constexpr uint32_t missing_method_index = 1024u;
464 auto last_method_raw_code = GenNopsAndBl(3u, kBlPlus0);
465 constexpr uint32_t bl_offset_in_last_method = 3u * 2u; // After NOPs.
466 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
467 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
468 const LinkerPatch last_method_patches[] = {
469 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, missing_method_index),
470 };
471
472 constexpr uint32_t just_over_max_negative_disp = 16 * MB + 2 - 4u /* PC adjustment */;
473 uint32_t last_method_idx = Create2MethodsWithGap(
474 kNopCode,
475 ArrayRef<const LinkerPatch>(),
476 last_method_code,
477 ArrayRef<const LinkerPatch>(last_method_patches),
478 just_over_max_negative_disp - bl_offset_in_last_method);
479 uint32_t method1_offset = GetMethodOffset(1u);
480 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
481 ASSERT_EQ(method1_offset,
482 last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
483 ASSERT_FALSE(method_offset_map_.FindMethodOffset(MethodRef(missing_method_index)).first);
484
485 // Check linked code.
486 uint32_t thunk_offset = CompiledCode::AlignCode(
487 last_method_offset + last_method_code.size(), InstructionSet::kThumb2);
488 uint32_t diff =
489 thunk_offset - (last_method_offset + bl_offset_in_last_method + 4u /* PC adjustment */);
490 ASSERT_TRUE(IsAligned<2u>(diff));
491 ASSERT_LT(diff >> 1, 1u << 8); // Simple encoding, (diff >> 1) fits into 8 bits.
492 auto expected_code = GenNopsAndBl(3u, kBlPlus0 | ((diff >> 1) & 0xffu));
493 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
494 ArrayRef<const uint8_t>(expected_code)));
495 EXPECT_TRUE(CheckThunk(thunk_offset));
496 }
497
TEST_F(Thumb2RelativePatcherTest,CallOtherAlmostTooFarAfter)498 TEST_F(Thumb2RelativePatcherTest, CallOtherAlmostTooFarAfter) {
499 auto method1_raw_code = GenNopsAndBl(3u, kBlPlus0);
500 constexpr uint32_t bl_offset_in_method1 = 3u * 2u; // After NOPs.
501 ArrayRef<const uint8_t> method1_code(method1_raw_code);
502 ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
503 const uint32_t kExpectedLastMethodIdx = 9u; // Based on 2MiB chunks in Create2MethodsWithGap().
504 const LinkerPatch method1_patches[] = {
505 LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
506 };
507
508 constexpr uint32_t max_positive_disp = 16 * MB - 2u + 4u /* PC adjustment */;
509 uint32_t last_method_idx = Create2MethodsWithGap(method1_code,
510 ArrayRef<const LinkerPatch>(method1_patches),
511 kNopCode,
512 ArrayRef<const LinkerPatch>(),
513 bl_offset_in_method1 + max_positive_disp);
514 ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
515
516 // Check linked code.
517 auto expected_code = GenNopsAndBl(3u, kBlPlusMax);
518 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
519 }
520
TEST_F(Thumb2RelativePatcherTest,CallOtherAlmostTooFarBefore)521 TEST_F(Thumb2RelativePatcherTest, CallOtherAlmostTooFarBefore) {
522 auto last_method_raw_code = GenNopsAndBl(2u, kBlPlus0);
523 constexpr uint32_t bl_offset_in_last_method = 2u * 2u; // After NOPs.
524 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
525 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
526 const LinkerPatch last_method_patches[] = {
527 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
528 };
529
530 constexpr uint32_t max_negative_disp = 16 * MB - 4u /* PC adjustment */;
531 uint32_t last_method_idx = Create2MethodsWithGap(kNopCode,
532 ArrayRef<const LinkerPatch>(),
533 last_method_code,
534 ArrayRef<const LinkerPatch>(last_method_patches),
535 max_negative_disp - bl_offset_in_last_method);
536 uint32_t method1_offset = GetMethodOffset(1u);
537 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
538 ASSERT_EQ(method1_offset, last_method_offset + bl_offset_in_last_method - max_negative_disp);
539
540 // Check linked code.
541 auto expected_code = GenNopsAndBl(2u, kBlMinusMax);
542 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
543 ArrayRef<const uint8_t>(expected_code)));
544 }
545
TEST_F(Thumb2RelativePatcherTest,CallOtherJustTooFarAfter)546 TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarAfter) {
547 auto method1_raw_code = GenNopsAndBl(2u, kBlPlus0);
548 constexpr uint32_t bl_offset_in_method1 = 2u * 2u; // After NOPs.
549 ArrayRef<const uint8_t> method1_code(method1_raw_code);
550 ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
551 const uint32_t kExpectedLastMethodIdx = 9u; // Based on 2MiB chunks in Create2MethodsWithGap().
552 const LinkerPatch method1_patches[] = {
553 LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
554 };
555
556 constexpr uint32_t just_over_max_positive_disp = 16 * MB + 4u /* PC adjustment */;
557 uint32_t last_method_idx = Create2MethodsWithGap(
558 method1_code,
559 ArrayRef<const LinkerPatch>(method1_patches),
560 kNopCode,
561 ArrayRef<const LinkerPatch>(),
562 bl_offset_in_method1 + just_over_max_positive_disp);
563 ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
564 uint32_t method_after_thunk_idx = last_method_idx;
565 if (sizeof(OatQuickMethodHeader) < kArmAlignment) {
566 // The thunk needs to start on a kArmAlignment-aligned address before the address where the
567 // last method would have been if there was no thunk. If the size of the OatQuickMethodHeader
568 // is at least kArmAlignment, the thunk start shall fit between the previous filler method
569 // and that address. Otherwise, it shall be inserted before that filler method.
570 method_after_thunk_idx -= 1u;
571 }
572
573 uint32_t method1_offset = GetMethodOffset(1u);
574 uint32_t method_after_thunk_offset = GetMethodOffset(method_after_thunk_idx);
575 ASSERT_TRUE(IsAligned<kArmAlignment>(method_after_thunk_offset));
576 uint32_t method_after_thunk_header_offset =
577 method_after_thunk_offset - sizeof(OatQuickMethodHeader);
578 uint32_t thunk_size = MethodCallThunkSize();
579 uint32_t thunk_offset = RoundDown(method_after_thunk_header_offset - thunk_size, kArmAlignment);
580 DCHECK_EQ(thunk_offset + thunk_size + CodeAlignmentSize(thunk_offset + thunk_size),
581 method_after_thunk_header_offset);
582 ASSERT_TRUE(IsAligned<kArmAlignment>(thunk_offset));
583 uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1 + 4u /* PC adjustment */);
584 ASSERT_TRUE(IsAligned<2u>(diff));
585 ASSERT_GE(diff, 16 * MB - (1u << 22)); // Simple encoding, unknown bits fit into imm10:imm11:0.
586 auto expected_code =
587 GenNopsAndBl(2u, 0xf000d000 | ((diff >> 1) & 0x7ffu) | (((diff >> 12) & 0x3ffu) << 16));
588 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
589 CheckThunk(thunk_offset);
590 }
591
TEST_F(Thumb2RelativePatcherTest,CallOtherJustTooFarBefore)592 TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarBefore) {
593 auto last_method_raw_code = GenNopsAndBl(3u, kBlPlus0);
594 constexpr uint32_t bl_offset_in_last_method = 3u * 2u; // After NOPs.
595 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
596 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
597 const LinkerPatch last_method_patches[] = {
598 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
599 };
600
601 constexpr uint32_t just_over_max_negative_disp = 16 * MB + 2 - 4u /* PC adjustment */;
602 uint32_t last_method_idx = Create2MethodsWithGap(
603 kNopCode,
604 ArrayRef<const LinkerPatch>(),
605 last_method_code,
606 ArrayRef<const LinkerPatch>(last_method_patches),
607 just_over_max_negative_disp - bl_offset_in_last_method);
608 uint32_t method1_offset = GetMethodOffset(1u);
609 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
610 ASSERT_EQ(method1_offset,
611 last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
612
613 // Check linked code.
614 uint32_t thunk_offset = CompiledCode::AlignCode(
615 last_method_offset + last_method_code.size(), InstructionSet::kThumb2);
616 uint32_t diff =
617 thunk_offset - (last_method_offset + bl_offset_in_last_method + 4u /* PC adjustment */);
618 ASSERT_TRUE(IsAligned<2u>(diff));
619 ASSERT_LT(diff >> 1, 1u << 8); // Simple encoding, (diff >> 1) fits into 8 bits.
620 auto expected_code = GenNopsAndBl(3u, kBlPlus0 | ((diff >> 1) & 0xffu));
621 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
622 ArrayRef<const uint8_t>(expected_code)));
623 EXPECT_TRUE(CheckThunk(thunk_offset));
624 }
625
TEST_F(Thumb2RelativePatcherTest,StringBssEntry1)626 TEST_F(Thumb2RelativePatcherTest, StringBssEntry1) {
627 TestStringBssEntry(0x00ff0000u, 0x00fcu);
628 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
629 }
630
TEST_F(Thumb2RelativePatcherTest,StringBssEntry2)631 TEST_F(Thumb2RelativePatcherTest, StringBssEntry2) {
632 TestStringBssEntry(0x02ff0000u, 0x05fcu);
633 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
634 }
635
TEST_F(Thumb2RelativePatcherTest,StringBssEntry3)636 TEST_F(Thumb2RelativePatcherTest, StringBssEntry3) {
637 TestStringBssEntry(0x08ff0000u, 0x08fcu);
638 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
639 }
640
TEST_F(Thumb2RelativePatcherTest,StringBssEntry4)641 TEST_F(Thumb2RelativePatcherTest, StringBssEntry4) {
642 TestStringBssEntry(0xd0ff0000u, 0x60fcu);
643 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
644 }
645
TEST_F(Thumb2RelativePatcherTest,StringReference1)646 TEST_F(Thumb2RelativePatcherTest, StringReference1) {
647 TestStringReference(0x00ff00fcu);
648 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
649 }
650
TEST_F(Thumb2RelativePatcherTest,StringReference2)651 TEST_F(Thumb2RelativePatcherTest, StringReference2) {
652 TestStringReference(0x02ff05fcu);
653 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
654 }
655
TEST_F(Thumb2RelativePatcherTest,StringReference3)656 TEST_F(Thumb2RelativePatcherTest, StringReference3) {
657 TestStringReference(0x08ff08fcu);
658 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
659 }
660
TEST_F(Thumb2RelativePatcherTest,StringReference4)661 TEST_F(Thumb2RelativePatcherTest, StringReference4) {
662 TestStringReference(0xd0ff60fcu);
663 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
664 }
665
TEST_F(Thumb2RelativePatcherTest,EntrypointCall)666 TEST_F(Thumb2RelativePatcherTest, EntrypointCall) {
667 constexpr uint32_t kEntrypointOffset = 512;
668 const LinkerPatch patches[] = {
669 LinkerPatch::CallEntrypointPatch(0u, kEntrypointOffset),
670 };
671 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
672 Link();
673
674 uint32_t method_offset = GetMethodOffset(1u);
675 uint32_t thunk_offset = CompiledCode::AlignCode(method_offset + kCallCode.size(),
676 InstructionSet::kThumb2);
677 uint32_t diff = thunk_offset - method_offset - kPcAdjustment;
678 ASSERT_TRUE(IsAligned<2u>(diff));
679 ASSERT_LT(diff >> 1, 1u << 8); // Simple encoding, (diff >> 1) fits into 8 bits.
680 auto expected_code = GenNopsAndBl(0u, kBlPlus0 | ((diff >> 1) & 0xffu));
681 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
682
683 // Verify the thunk.
684 uint32_t ldr_pc_tr_offset =
685 0xf8d00000 | // LDR Rt, [Rn, #<imm12>]
686 (/* tr */ 9 << 16) | // Rn = TR
687 (/* pc */ 15 << 12) | // Rt = PC
688 kEntrypointOffset; // imm12
689 uint16_t bkpt = 0xbe00;
690 ASSERT_LE(6u, output_.size() - thunk_offset);
691 EXPECT_EQ(ldr_pc_tr_offset, GetOutputInsn32(thunk_offset));
692 EXPECT_EQ(bkpt, GetOutputInsn16(thunk_offset + 4u));
693 }
694
695 const uint32_t kBakerValidRegs[] = {
696 0, 1, 2, 3, 4, 5, 6, 7,
697 9, 10, 11, // r8 (rMR), IP, SP, LR and PC are reserved.
698 };
699
700 const uint32_t kBakerValidRegsNarrow[] = {
701 0, 1, 2, 3, 4, 5, 6, 7,
702 };
703
TestBakerFieldWide(uint32_t offset,uint32_t ref_reg)704 void Thumb2RelativePatcherTest::TestBakerFieldWide(uint32_t offset, uint32_t ref_reg) {
705 DCHECK_ALIGNED(offset, 4u);
706 DCHECK_LT(offset, 4 * KB);
707 constexpr size_t kMethodCodeSize = 8u;
708 constexpr size_t kLiteralOffset = 0u;
709 uint32_t method_idx = 0u;
710 for (uint32_t base_reg : kBakerValidRegs) {
711 for (uint32_t holder_reg : kBakerValidRegs) {
712 uint32_t ldr = kLdrWInsn | offset | (base_reg << 16) | (ref_reg << 12);
713 const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
714 ASSERT_EQ(kMethodCodeSize, raw_code.size());
715 ArrayRef<const uint8_t> code(raw_code);
716 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
717 base_reg, holder_reg, /* narrow */ false);
718 const LinkerPatch patches[] = {
719 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
720 };
721 ++method_idx;
722 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
723 }
724 }
725 Link();
726
727 // All thunks are at the end.
728 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
729 method_idx = 0u;
730 for (uint32_t base_reg : kBakerValidRegs) {
731 for (uint32_t holder_reg : kBakerValidRegs) {
732 ++method_idx;
733 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
734 uint32_t ldr = kLdrWInsn | offset | (base_reg << 16) | (ref_reg << 12);
735 const std::vector<uint8_t> expected_code = RawCode({bne, ldr});
736 ASSERT_EQ(kMethodCodeSize, expected_code.size()) << "bne=0x" << std::hex << bne;
737 ASSERT_TRUE(
738 CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
739
740 std::vector<uint8_t> expected_thunk =
741 CompileBakerOffsetThunk(base_reg, holder_reg, /* narrow */ false);
742 ASSERT_GT(output_.size(), thunk_offset);
743 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
744 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
745 expected_thunk.size());
746 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
747 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
748 ASSERT_TRUE(false);
749 }
750
751 size_t gray_check_offset = thunk_offset;
752 if (holder_reg == base_reg) {
753 // Verify that the null-check uses the correct register, i.e. holder_reg.
754 if (holder_reg < 8) {
755 ASSERT_GE(output_.size() - gray_check_offset, 2u);
756 ASSERT_EQ(0xb100 | holder_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
757 gray_check_offset +=2u;
758 } else {
759 ASSERT_GE(output_.size() - gray_check_offset, 6u);
760 ASSERT_EQ(0xf1b00f00u | (holder_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
761 ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u); // BEQ
762 gray_check_offset += 6u;
763 }
764 }
765 // Verify that the lock word for gray bit check is loaded from the holder address.
766 ASSERT_GE(output_.size() - gray_check_offset,
767 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
768 const uint32_t load_lock_word =
769 kLdrWInsn |
770 (holder_reg << 16) |
771 (/* IP */ 12 << 12) |
772 mirror::Object::MonitorOffset().Uint32Value();
773 ASSERT_EQ(load_lock_word, GetOutputInsn32(gray_check_offset));
774 // Verify the gray bit check.
775 DCHECK_GE(LockWord::kReadBarrierStateShift, 8u); // ROR modified immediate.
776 uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
777 const uint32_t tst_gray_bit_without_offset =
778 0xf0100f00 | (/* IP */ 12 << 16)
779 | (((ror_shift >> 4) & 1) << 26) // i
780 | (((ror_shift >> 1) & 7) << 12) // imm3
781 | ((ror_shift & 1) << 7); // imm8, ROR('1':imm8<7:0>, ror_shift).
782 EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(gray_check_offset + 4u));
783 EXPECT_EQ(0xd100u, GetOutputInsn16(gray_check_offset + 8u) & 0xff00u); // BNE
784 // Verify the fake dependency (skip "ADD LR, LR, #ldr_offset").
785 const uint32_t fake_dependency =
786 0xeb000010 | // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
787 (/* IP */ 12) | // Rm = IP
788 (base_reg << 16) | // Rn = base_reg
789 (base_reg << 8); // Rd = base_reg
790 EXPECT_EQ(fake_dependency, GetOutputInsn32(gray_check_offset + 14u));
791 // Do not check the rest of the implementation.
792
793 // The next thunk follows on the next aligned offset.
794 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
795 }
796 }
797 }
798
TestBakerFieldNarrow(uint32_t offset,uint32_t ref_reg)799 void Thumb2RelativePatcherTest::TestBakerFieldNarrow(uint32_t offset, uint32_t ref_reg) {
800 DCHECK_ALIGNED(offset, 4u);
801 DCHECK_LT(offset, 32u);
802 constexpr size_t kMethodCodeSize = 6u;
803 constexpr size_t kLiteralOffset = 0u;
804 uint32_t method_idx = 0u;
805 for (uint32_t base_reg : kBakerValidRegs) {
806 if (base_reg >= 8u) {
807 continue;
808 }
809 for (uint32_t holder_reg : kBakerValidRegs) {
810 uint32_t ldr = kLdrInsn | (offset << (6 - 2)) | (base_reg << 3) | ref_reg;
811 const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
812 ASSERT_EQ(kMethodCodeSize, raw_code.size());
813 ArrayRef<const uint8_t> code(raw_code);
814 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
815 base_reg, holder_reg, /* narrow */ true);
816 const LinkerPatch patches[] = {
817 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
818 };
819 ++method_idx;
820 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
821 }
822 }
823 Link();
824
825 // All thunks are at the end.
826 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
827 method_idx = 0u;
828 for (uint32_t base_reg : kBakerValidRegs) {
829 if (base_reg >= 8u) {
830 continue;
831 }
832 for (uint32_t holder_reg : kBakerValidRegs) {
833 ++method_idx;
834 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
835 uint32_t ldr = kLdrInsn | (offset << (6 - 2)) | (base_reg << 3) | ref_reg;
836 const std::vector<uint8_t> expected_code = RawCode({bne, ldr});
837 ASSERT_EQ(kMethodCodeSize, expected_code.size()) << "bne=0x" << std::hex << bne;
838 ASSERT_TRUE(
839 CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
840
841 std::vector<uint8_t> expected_thunk =
842 CompileBakerOffsetThunk(base_reg, holder_reg, /* narrow */ true);
843 ASSERT_GT(output_.size(), thunk_offset);
844 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
845 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
846 expected_thunk.size());
847 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
848 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
849 ASSERT_TRUE(false);
850 }
851
852 size_t gray_check_offset = thunk_offset;
853 if (holder_reg == base_reg) {
854 // Verify that the null-check uses the correct register, i.e. holder_reg.
855 if (holder_reg < 8) {
856 ASSERT_GE(output_.size() - gray_check_offset, 2u);
857 ASSERT_EQ(0xb100 | holder_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
858 gray_check_offset +=2u;
859 } else {
860 ASSERT_GE(output_.size() - gray_check_offset, 6u);
861 ASSERT_EQ(0xf1b00f00u | (holder_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
862 ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u); // BEQ
863 gray_check_offset += 6u;
864 }
865 }
866 // Verify that the lock word for gray bit check is loaded from the holder address.
867 ASSERT_GE(output_.size() - gray_check_offset,
868 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
869 const uint32_t load_lock_word =
870 kLdrWInsn |
871 (holder_reg << 16) |
872 (/* IP */ 12 << 12) |
873 mirror::Object::MonitorOffset().Uint32Value();
874 ASSERT_EQ(load_lock_word, GetOutputInsn32(gray_check_offset));
875 // Verify the gray bit check.
876 DCHECK_GE(LockWord::kReadBarrierStateShift, 8u); // ROR modified immediate.
877 uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
878 const uint32_t tst_gray_bit_without_offset =
879 0xf0100f00 | (/* IP */ 12 << 16)
880 | (((ror_shift >> 4) & 1) << 26) // i
881 | (((ror_shift >> 1) & 7) << 12) // imm3
882 | ((ror_shift & 1) << 7); // imm8, ROR('1':imm8<7:0>, ror_shift).
883 EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(gray_check_offset + 4u));
884 EXPECT_EQ(0xd100u, GetOutputInsn16(gray_check_offset + 8u) & 0xff00u); // BNE
885 // Verify the fake dependency (skip "ADD LR, LR, #ldr_offset").
886 const uint32_t fake_dependency =
887 0xeb000010 | // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
888 (/* IP */ 12) | // Rm = IP
889 (base_reg << 16) | // Rn = base_reg
890 (base_reg << 8); // Rd = base_reg
891 EXPECT_EQ(fake_dependency, GetOutputInsn32(gray_check_offset + 14u));
892 // Do not check the rest of the implementation.
893
894 // The next thunk follows on the next aligned offset.
895 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
896 }
897 }
898 }
899
TEST_F(Thumb2RelativePatcherTest,BakerOffsetWide)900 TEST_F(Thumb2RelativePatcherTest, BakerOffsetWide) {
901 struct TestCase {
902 uint32_t offset;
903 uint32_t ref_reg;
904 };
905 static const TestCase test_cases[] = {
906 { 0u, 0u },
907 { 8u, 3u },
908 { 28u, 7u },
909 { 0xffcu, 11u },
910 };
911 for (const TestCase& test_case : test_cases) {
912 Reset();
913 TestBakerFieldWide(test_case.offset, test_case.ref_reg);
914 }
915 }
916
TEST_F(Thumb2RelativePatcherTest,BakerOffsetNarrow)917 TEST_F(Thumb2RelativePatcherTest, BakerOffsetNarrow) {
918 struct TestCase {
919 uint32_t offset;
920 uint32_t ref_reg;
921 };
922 static const TestCase test_cases[] = {
923 { 0, 0u },
924 { 8, 3u },
925 { 28, 7u },
926 };
927 for (const TestCase& test_case : test_cases) {
928 Reset();
929 TestBakerFieldNarrow(test_case.offset, test_case.ref_reg);
930 }
931 }
932
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkInTheMiddle)933 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddle) {
934 // One thunk in the middle with maximum distance branches to it from both sides.
935 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
936 constexpr uint32_t kLiteralOffset1 = 6u;
937 const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
938 ArrayRef<const uint8_t> code1(raw_code1);
939 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
940 /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
941 const LinkerPatch patches1[] = {
942 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
943 };
944 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
945
946 constexpr uint32_t expected_thunk_offset =
947 kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement */ ((1 << 20) - 2u);
948 static_assert(IsAligned<kArmAlignment>(expected_thunk_offset), "Target offset must be aligned.");
949 size_t filler1_size = expected_thunk_offset -
950 RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment);
951 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
952 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
953 AddCompiledMethod(MethodRef(2u), filler1_code);
954
955 // Enforce thunk reservation with a tiny method.
956 AddCompiledMethod(MethodRef(3u), kNopCode);
957
958 constexpr uint32_t kLiteralOffset2 = 4;
959 static_assert(IsAligned<kArmAlignment>(kLiteralOffset2 + kPcAdjustment),
960 "PC for BNE must be aligned.");
961
962 // Allow reaching the thunk from the very beginning of a method almost 1MiB away. Backward branch
963 // reaches the full 1MiB but we need to take PC adjustment into account. Things to subtract:
964 // - thunk size and method 3 pre-header, rounded up (padding in between if needed)
965 // - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
966 // - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
967 size_t thunk_size =
968 CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false).size();
969 size_t filler2_size =
970 1 * MB - (kLiteralOffset2 + kPcAdjustment)
971 - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArmAlignment)
972 - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArmAlignment)
973 - sizeof(OatQuickMethodHeader);
974 std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 2u);
975 ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
976 AddCompiledMethod(MethodRef(4u), filler2_code);
977
978 const std::vector<uint8_t> raw_code2 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn});
979 ArrayRef<const uint8_t> code2(raw_code2);
980 const LinkerPatch patches2[] = {
981 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
982 };
983 AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
984
985 Link();
986
987 uint32_t first_method_offset = GetMethodOffset(1u);
988 uint32_t last_method_offset = GetMethodOffset(5u);
989 EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
990
991 const uint32_t bne_max_forward = kBneWPlus0 | 0x003f2fff;
992 const uint32_t bne_max_backward = kBneWPlus0 | 0x04000000;
993 const std::vector<uint8_t> expected_code1 =
994 RawCode({kNopWInsn, kNopInsn, bne_max_forward, kLdrWInsn});
995 const std::vector<uint8_t> expected_code2 = RawCode({kNopWInsn, bne_max_backward, kLdrWInsn});
996 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
997 ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
998 }
999
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkBeforeFiller)1000 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkBeforeFiller) {
1001 // Based on the first part of BakerOffsetThunkInTheMiddle but the BNE is one instruction
1002 // earlier, so the thunk is emitted before the filler.
1003 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1004 constexpr uint32_t kLiteralOffset1 = 4u;
1005 const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn});
1006 ArrayRef<const uint8_t> code1(raw_code1);
1007 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
1008 /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
1009 const LinkerPatch patches1[] = {
1010 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1011 };
1012 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1013
1014 constexpr uint32_t expected_thunk_offset =
1015 kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement + 2 */ (1u << 20);
1016 static_assert(IsAligned<kArmAlignment>(expected_thunk_offset), "Target offset must be aligned.");
1017 size_t filler1_size = expected_thunk_offset -
1018 RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment);
1019 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
1020 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1021 AddCompiledMethod(MethodRef(2u), filler1_code);
1022
1023 Link();
1024
1025 const uint32_t bne = BneWWithOffset(kLiteralOffset1, RoundUp(raw_code1.size(), kArmAlignment));
1026 const std::vector<uint8_t> expected_code1 = RawCode({kNopWInsn, bne, kLdrWInsn, kNopInsn});
1027 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1028 }
1029
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkInTheMiddleUnreachableFromLast)1030 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddleUnreachableFromLast) {
1031 // Based on the BakerOffsetThunkInTheMiddle but the BNE in the last method is preceded
1032 // by NOP and cannot reach the thunk in the middle, so we emit an extra thunk at the end.
1033 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1034 constexpr uint32_t kLiteralOffset1 = 6u;
1035 const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
1036 ArrayRef<const uint8_t> code1(raw_code1);
1037 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
1038 /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
1039 const LinkerPatch patches1[] = {
1040 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1041 };
1042 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1043
1044 constexpr uint32_t expected_thunk_offset =
1045 kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement */ ((1 << 20) - 2u);
1046 static_assert(IsAligned<kArmAlignment>(expected_thunk_offset), "Target offset must be aligned.");
1047 size_t filler1_size = expected_thunk_offset -
1048 RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment);
1049 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
1050 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1051 AddCompiledMethod(MethodRef(2u), filler1_code);
1052
1053 // Enforce thunk reservation with a tiny method.
1054 AddCompiledMethod(MethodRef(3u), kNopCode);
1055
1056 constexpr uint32_t kReachableFromOffset2 = 4;
1057 constexpr uint32_t kLiteralOffset2 = kReachableFromOffset2 + 2;
1058 static_assert(IsAligned<kArmAlignment>(kReachableFromOffset2 + kPcAdjustment),
1059 "PC for BNE must be aligned.");
1060
1061 // If not for the extra NOP, this would allow reaching the thunk from the BNE
1062 // of a method 1MiB away. Backward branch reaches the full 1MiB but we need to take
1063 // PC adjustment into account. Things to subtract:
1064 // - thunk size and method 3 pre-header, rounded up (padding in between if needed)
1065 // - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
1066 // - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
1067 size_t thunk_size =
1068 CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false).size();
1069 size_t filler2_size =
1070 1 * MB - (kReachableFromOffset2 + kPcAdjustment)
1071 - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArmAlignment)
1072 - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArmAlignment)
1073 - sizeof(OatQuickMethodHeader);
1074 std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 2u);
1075 ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
1076 AddCompiledMethod(MethodRef(4u), filler2_code);
1077
1078 // Extra 16-bit NOP compared to BakerOffsetThunkInTheMiddle.
1079 const std::vector<uint8_t> raw_code2 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
1080 ArrayRef<const uint8_t> code2(raw_code2);
1081 const LinkerPatch patches2[] = {
1082 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
1083 };
1084 AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
1085
1086 Link();
1087
1088 uint32_t first_method_offset = GetMethodOffset(1u);
1089 uint32_t last_method_offset = GetMethodOffset(5u);
1090 EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
1091
1092 const uint32_t bne_max_forward = kBneWPlus0 | 0x003f2fff;
1093 const uint32_t bne_last =
1094 BneWWithOffset(kLiteralOffset2, RoundUp(raw_code2.size(), kArmAlignment));
1095 const std::vector<uint8_t> expected_code1 =
1096 RawCode({kNopWInsn, kNopInsn, bne_max_forward, kLdrWInsn});
1097 const std::vector<uint8_t> expected_code2 =
1098 RawCode({kNopWInsn, kNopInsn, bne_last, kLdrWInsn});
1099 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1100 ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
1101 }
1102
TEST_F(Thumb2RelativePatcherTest,BakerArray)1103 TEST_F(Thumb2RelativePatcherTest, BakerArray) {
1104 auto ldr = [](uint32_t base_reg) {
1105 uint32_t index_reg = (base_reg == 0u) ? 1u : 0u;
1106 uint32_t ref_reg = (base_reg == 2) ? 3u : 2u;
1107 return kLdrRegLsl2 | index_reg | (base_reg << 16) | (ref_reg << 12);
1108 };
1109 constexpr size_t kMethodCodeSize = 8u;
1110 constexpr size_t kLiteralOffset = 0u;
1111 uint32_t method_idx = 0u;
1112 for (uint32_t base_reg : kBakerValidRegs) {
1113 ++method_idx;
1114 const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr(base_reg)});
1115 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1116 ArrayRef<const uint8_t> code(raw_code);
1117 const LinkerPatch patches[] = {
1118 LinkerPatch::BakerReadBarrierBranchPatch(
1119 kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
1120 };
1121 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1122 }
1123 Link();
1124
1125 // All thunks are at the end.
1126 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
1127 method_idx = 0u;
1128 for (uint32_t base_reg : kBakerValidRegs) {
1129 ++method_idx;
1130 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1131 const std::vector<uint8_t> expected_code = RawCode({bne, ldr(base_reg)});
1132 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1133 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1134
1135 std::vector<uint8_t> expected_thunk = CompileBakerArrayThunk(base_reg);
1136 ASSERT_GT(output_.size(), thunk_offset);
1137 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1138 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1139 expected_thunk.size());
1140 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1141 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1142 ASSERT_TRUE(false);
1143 }
1144
1145 // Verify that the lock word for gray bit check is loaded from the correct address
1146 // before the base_reg which points to the array data.
1147 ASSERT_GE(output_.size() - thunk_offset,
1148 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
1149 int32_t data_offset =
1150 mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
1151 int32_t offset = mirror::Object::MonitorOffset().Int32Value() - data_offset;
1152 ASSERT_LT(offset, 0);
1153 ASSERT_GT(offset, -256);
1154 const uint32_t load_lock_word =
1155 kLdrNegativeOffset |
1156 (-offset & 0xffu) |
1157 (base_reg << 16) |
1158 (/* IP */ 12 << 12);
1159 EXPECT_EQ(load_lock_word, GetOutputInsn32(thunk_offset));
1160 // Verify the gray bit check.
1161 DCHECK_GE(LockWord::kReadBarrierStateShift, 8u); // ROR modified immediate.
1162 uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
1163 const uint32_t tst_gray_bit_without_offset =
1164 0xf0100f00 | (/* IP */ 12 << 16)
1165 | (((ror_shift >> 4) & 1) << 26) // i
1166 | (((ror_shift >> 1) & 7) << 12) // imm3
1167 | ((ror_shift & 1) << 7); // imm8, ROR('1':imm8<7:0>, ror_shift).
1168 EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(thunk_offset + 4u));
1169 EXPECT_EQ(0xd100u, GetOutputInsn16(thunk_offset + 8u) & 0xff00u); // BNE
1170 // Verify the fake dependency.
1171 const uint32_t fake_dependency =
1172 0xeb000010 | // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
1173 (/* IP */ 12) | // Rm = IP
1174 (base_reg << 16) | // Rn = base_reg
1175 (base_reg << 8); // Rd = base_reg
1176 EXPECT_EQ(fake_dependency, GetOutputInsn32(thunk_offset + 14u));
1177 // Do not check the rest of the implementation.
1178
1179 // The next thunk follows on the next aligned offset.
1180 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
1181 }
1182 }
1183
TEST_F(Thumb2RelativePatcherTest,BakerGcRootWide)1184 TEST_F(Thumb2RelativePatcherTest, BakerGcRootWide) {
1185 constexpr size_t kMethodCodeSize = 8u;
1186 constexpr size_t kLiteralOffset = 4u;
1187 uint32_t method_idx = 0u;
1188 for (uint32_t root_reg : kBakerValidRegs) {
1189 ++method_idx;
1190 uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (root_reg << 12);
1191 const std::vector<uint8_t> raw_code = RawCode({ldr, kBneWPlus0});
1192 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1193 ArrayRef<const uint8_t> code(raw_code);
1194 const LinkerPatch patches[] = {
1195 LinkerPatch::BakerReadBarrierBranchPatch(
1196 kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
1197 };
1198 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1199 }
1200 Link();
1201
1202 // All thunks are at the end.
1203 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
1204 method_idx = 0u;
1205 for (uint32_t root_reg : kBakerValidRegs) {
1206 ++method_idx;
1207 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1208 uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (root_reg << 12);
1209 const std::vector<uint8_t> expected_code = RawCode({ldr, bne});
1210 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1211 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1212
1213 std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg, /* narrow */ false);
1214 ASSERT_GT(output_.size(), thunk_offset);
1215 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1216 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1217 expected_thunk.size());
1218 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1219 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1220 ASSERT_TRUE(false);
1221 }
1222
1223 // Verify that the fast-path null-check uses the correct register, i.e. root_reg.
1224 if (root_reg < 8) {
1225 ASSERT_GE(output_.size() - thunk_offset, 2u);
1226 ASSERT_EQ(0xb100 | root_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
1227 } else {
1228 ASSERT_GE(output_.size() - thunk_offset, 6u);
1229 ASSERT_EQ(0xf1b00f00u | (root_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
1230 ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u); // BEQ
1231 }
1232 // Do not check the rest of the implementation.
1233
1234 // The next thunk follows on the next aligned offset.
1235 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
1236 }
1237 }
1238
TEST_F(Thumb2RelativePatcherTest,BakerGcRootNarrow)1239 TEST_F(Thumb2RelativePatcherTest, BakerGcRootNarrow) {
1240 constexpr size_t kMethodCodeSize = 6u;
1241 constexpr size_t kLiteralOffset = 2u;
1242 uint32_t method_idx = 0u;
1243 for (uint32_t root_reg : kBakerValidRegsNarrow) {
1244 ++method_idx;
1245 uint32_t ldr = kLdrInsn | (/* offset */ 8 << (6 - 2)) | (/* base_reg */ 0 << 3) | root_reg;
1246 const std::vector<uint8_t> raw_code = RawCode({ldr, kBneWPlus0});
1247 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1248 ArrayRef<const uint8_t> code(raw_code);
1249 const LinkerPatch patches[] = {
1250 LinkerPatch::BakerReadBarrierBranchPatch(
1251 kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
1252 };
1253 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1254 }
1255 Link();
1256
1257 // All thunks are at the end.
1258 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
1259 method_idx = 0u;
1260 for (uint32_t root_reg : kBakerValidRegsNarrow) {
1261 ++method_idx;
1262 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1263 uint32_t ldr = kLdrInsn | (/* offset */ 8 << (6 - 2)) | (/* base_reg */ 0 << 3) | root_reg;
1264 const std::vector<uint8_t> expected_code = RawCode({ldr, bne});
1265 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1266 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1267
1268 std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg, /* narrow */ true);
1269 ASSERT_GT(output_.size(), thunk_offset);
1270 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1271 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1272 expected_thunk.size());
1273 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1274 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1275 ASSERT_TRUE(false);
1276 }
1277
1278 // Verify that the fast-path null-check CBZ uses the correct register, i.e. root_reg.
1279 ASSERT_GE(output_.size() - thunk_offset, 2u);
1280 ASSERT_EQ(0xb100 | root_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
1281 // Do not check the rest of the implementation.
1282
1283 // The next thunk follows on the next aligned offset.
1284 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
1285 }
1286 }
1287
TEST_F(Thumb2RelativePatcherTest,BakerGcRootOffsetBits)1288 TEST_F(Thumb2RelativePatcherTest, BakerGcRootOffsetBits) {
1289 // Test 1MiB of patches to the same thunk to stress-test different large offsets.
1290 // (The low bits are not that important but the location of the high bits is easy to get wrong.)
1291 std::vector<uint8_t> code;
1292 code.reserve(1 * MB);
1293 const size_t num_patches = 1 * MB / 8u;
1294 std::vector<LinkerPatch> patches;
1295 patches.reserve(num_patches);
1296 const uint32_t ldr =
1297 kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12);
1298 uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
1299 for (size_t i = 0; i != num_patches; ++i) {
1300 PushBackInsn(&code, ldr);
1301 PushBackInsn(&code, kBneWPlus0);
1302 patches.push_back(LinkerPatch::BakerReadBarrierBranchPatch(8u * i + 4u, encoded_data));
1303 }
1304 ASSERT_EQ(1 * MB, code.size());
1305 ASSERT_EQ(num_patches, patches.size());
1306 AddCompiledMethod(MethodRef(1u),
1307 ArrayRef<const uint8_t>(code),
1308 ArrayRef<const LinkerPatch>(patches));
1309 Link();
1310
1311 // The thunk is right after the method code.
1312 DCHECK_ALIGNED(1 * MB, kArmAlignment);
1313 std::vector<uint8_t> expected_code;
1314 for (size_t i = 0; i != num_patches; ++i) {
1315 PushBackInsn(&expected_code, ldr);
1316 PushBackInsn(&expected_code, BneWWithOffset(8u * i + 4u, 1 * MB));
1317 patches.push_back(LinkerPatch::BakerReadBarrierBranchPatch(8u * i + 4u, encoded_data));
1318 }
1319 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
1320 }
1321
TEST_F(Thumb2RelativePatcherTest,BakerAndMethodCallInteraction)1322 TEST_F(Thumb2RelativePatcherTest, BakerAndMethodCallInteraction) {
1323 // During development, there was a `DCHECK_LE(MaxNextOffset(), next_thunk.MaxNextOffset());`
1324 // in `ArmBaseRelativePatcher::ThunkData::MakeSpaceBefore()` which does not necessarily
1325 // hold when we're reserving thunks of different sizes. This test exposes the situation
1326 // by using Baker thunks and a method call thunk.
1327
1328 // Add a method call patch that can reach to method 1 offset + 16MiB.
1329 uint32_t method_idx = 0u;
1330 constexpr size_t kMethodCallLiteralOffset = 2u;
1331 constexpr uint32_t kMissingMethodIdx = 2u;
1332 const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kBlPlus0});
1333 const LinkerPatch method1_patches[] = {
1334 LinkerPatch::RelativeCodePatch(kMethodCallLiteralOffset, nullptr, 2u),
1335 };
1336 ArrayRef<const uint8_t> code1(raw_code1);
1337 ++method_idx;
1338 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(method1_patches));
1339
1340 // Skip kMissingMethodIdx.
1341 ++method_idx;
1342 ASSERT_EQ(kMissingMethodIdx, method_idx);
1343 // Add a method with the right size that the method code for the next one starts 1MiB
1344 // after code for method 1.
1345 size_t filler_size =
1346 1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment)
1347 - sizeof(OatQuickMethodHeader);
1348 std::vector<uint8_t> filler_code = GenNops(filler_size / 2u);
1349 ++method_idx;
1350 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1351 // Add 14 methods with 1MiB code+header, making the code for the next method start 1MiB
1352 // before the currently scheduled MaxNextOffset() for the method call thunk.
1353 for (uint32_t i = 0; i != 14; ++i) {
1354 filler_size = 1 * MB - sizeof(OatQuickMethodHeader);
1355 filler_code = GenNops(filler_size / 2u);
1356 ++method_idx;
1357 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1358 }
1359
1360 // Add 2 Baker GC root patches to the last method, one that would allow the thunk at
1361 // 1MiB + kArmAlignment, i.e. kArmAlignment after the method call thunk, and the
1362 // second that needs it kArmAlignment after that. Given the size of the GC root thunk
1363 // is more than the space required by the method call thunk plus kArmAlignment,
1364 // this pushes the first GC root thunk's pending MaxNextOffset() before the method call
1365 // thunk's pending MaxNextOffset() which needs to be adjusted.
1366 ASSERT_LT(RoundUp(CompileMethodCallThunk().size(), kArmAlignment) + kArmAlignment,
1367 CompileBakerGcRootThunk(/* root_reg */ 0, /* narrow */ false).size());
1368 static_assert(kArmAlignment == 8, "Code below assumes kArmAlignment == 8");
1369 constexpr size_t kBakerLiteralOffset1 = kArmAlignment + 2u - kPcAdjustment;
1370 constexpr size_t kBakerLiteralOffset2 = kBakerLiteralOffset1 + kArmAlignment;
1371 // Use offset = 0, base_reg = 0, the LDR is simply `kLdrWInsn | (root_reg << 12)`.
1372 const uint32_t ldr1 = kLdrWInsn | (/* root_reg */ 1 << 12);
1373 const uint32_t ldr2 = kLdrWInsn | (/* root_reg */ 2 << 12);
1374 const std::vector<uint8_t> last_method_raw_code = RawCode({
1375 kNopInsn, // Padding before first GC root read barrier.
1376 ldr1, kBneWPlus0, // First GC root LDR with read barrier.
1377 ldr2, kBneWPlus0, // Second GC root LDR with read barrier.
1378 });
1379 uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
1380 uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
1381 const LinkerPatch last_method_patches[] = {
1382 LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
1383 LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
1384 };
1385 ++method_idx;
1386 AddCompiledMethod(MethodRef(method_idx),
1387 ArrayRef<const uint8_t>(last_method_raw_code),
1388 ArrayRef<const LinkerPatch>(last_method_patches));
1389
1390 // The main purpose of the test is to check that Link() does not cause a crash.
1391 Link();
1392
1393 ASSERT_EQ(15 * MB, GetMethodOffset(method_idx) - GetMethodOffset(1u));
1394 }
1395
1396 } // namespace linker
1397 } // namespace art
1398