1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "linker/arm/relative_patcher_thumb2.h"
18 
19 #include "arch/arm/instruction_set_features_arm.h"
20 #include "base/casts.h"
21 #include "driver/compiler_options.h"
22 #include "linker/relative_patcher_test.h"
23 #include "lock_word.h"
24 #include "mirror/array-inl.h"
25 #include "mirror/object.h"
26 #include "oat_quick_method_header.h"
27 #include "optimizing/code_generator_arm_vixl.h"
28 #include "optimizing/optimizing_unit_test.h"
29 
30 namespace art {
31 namespace linker {
32 
33 class Thumb2RelativePatcherTest : public RelativePatcherTest {
34  public:
Thumb2RelativePatcherTest()35   Thumb2RelativePatcherTest() : RelativePatcherTest(InstructionSet::kThumb2, "default") { }
36 
37  protected:
38   static const uint8_t kCallRawCode[];
39   static const ArrayRef<const uint8_t> kCallCode;
40   static const uint8_t kNopRawCode[];
41   static const ArrayRef<const uint8_t> kNopCode;
42   static const uint8_t kUnpatchedPcRelativeRawCode[];
43   static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCode;
44   static const uint32_t kPcInsnOffset;
45 
46   // The PC in Thumb mode is 4 bytes after the instruction location.
47   static constexpr uint32_t kPcAdjustment = 4u;
48 
49   // Branches within range [-256, 256) can be created from these by adding the low 8 bits.
50   static constexpr uint32_t kBlPlus0 = 0xf000f800u;
51   static constexpr uint32_t kBlMinus256 = 0xf7ffff00u;
52 
53   // Special BL values.
54   static constexpr uint32_t kBlPlusMax = 0xf3ffd7ffu;
55   static constexpr uint32_t kBlMinusMax = 0xf400d000u;
56 
57   // BNE +0, 32-bit, encoding T3. Bits 0-10, 11, 13, 16-21, 26 are placeholder for target offset.
58   static constexpr uint32_t kBneWPlus0 = 0xf0408000u;
59 
60   // LDR immediate, 16-bit, encoding T1. Bits 6-10 are imm5, 0-2 are Rt, 3-5 are Rn.
61   static constexpr uint32_t kLdrInsn = 0x6800u;
62 
63   // LDR immediate, 32-bit, encoding T3. Bits 0-11 are offset, 12-15 are Rt, 16-20 are Rn.
64   static constexpr uint32_t kLdrWInsn = 0xf8d00000u;
65 
66   // LDR immediate, negative offset, encoding T4. Bits 0-7 are the offset to subtract.
67   static constexpr uint32_t kLdrNegativeOffset = 0xf8500c00u;
68 
69   // LDR register, lsl #2. Bits 4-5 are the imm2, i.e. the lsl shift.
70   static constexpr uint32_t kLdrRegLsl2 = 0xf8500020u;
71 
72   // NOP instructions.
73   static constexpr uint32_t kNopInsn = 0xbf00u;
74   static constexpr uint32_t kNopWInsn = 0xf3af8000u;
75 
InsertInsn(std::vector<uint8_t> * code,size_t pos,uint32_t insn)76   void InsertInsn(std::vector<uint8_t>* code, size_t pos, uint32_t insn) {
77     CHECK_LE(pos, code->size());
78     if (IsUint<16>(insn)) {
79       const uint8_t insn_code[] = {
80           static_cast<uint8_t>(insn),
81           static_cast<uint8_t>(insn >> 8),
82       };
83       static_assert(sizeof(insn_code) == 2u, "Invalid sizeof(insn_code).");
84       code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
85     } else {
86       const uint8_t insn_code[] = {
87           static_cast<uint8_t>(insn >> 16),
88           static_cast<uint8_t>(insn >> 24),
89           static_cast<uint8_t>(insn),
90           static_cast<uint8_t>(insn >> 8),
91       };
92       static_assert(sizeof(insn_code) == 4u, "Invalid sizeof(insn_code).");
93       code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
94     }
95   }
96 
PushBackInsn(std::vector<uint8_t> * code,uint32_t insn)97   void PushBackInsn(std::vector<uint8_t>* code, uint32_t insn) {
98     InsertInsn(code, code->size(), insn);
99   }
100 
GenNops(size_t num_nops)101   std::vector<uint8_t> GenNops(size_t num_nops) {
102     std::vector<uint8_t> result;
103     result.reserve(num_nops * 2u);
104     for (size_t i = 0; i != num_nops; ++i) {
105       PushBackInsn(&result, kNopInsn);
106     }
107     return result;
108   }
109 
RawCode(std::initializer_list<uint32_t> insns)110   std::vector<uint8_t> RawCode(std::initializer_list<uint32_t> insns) {
111     std::vector<uint8_t> raw_code;
112     size_t number_of_16_bit_insns =
113         std::count_if(insns.begin(), insns.end(), [](uint32_t x) { return IsUint<16>(x); });
114     raw_code.reserve(insns.size() * 4u - number_of_16_bit_insns * 2u);
115     for (uint32_t insn : insns) {
116       PushBackInsn(&raw_code, insn);
117     }
118     return raw_code;
119   }
120 
BneWWithOffset(uint32_t bne_offset,uint32_t target_offset)121   uint32_t BneWWithOffset(uint32_t bne_offset, uint32_t target_offset) {
122     if (!IsAligned<2u>(bne_offset)) {
123       LOG(ERROR) << "Unaligned bne_offset: " << bne_offset;
124       return 0xffffffffu;  // Fails code diff later.
125     }
126     if (!IsAligned<2u>(target_offset)) {
127       LOG(ERROR) << "Unaligned target_offset: " << target_offset;
128       return 0xffffffffu;  // Fails code diff later.
129     }
130     uint32_t diff = target_offset - bne_offset - kPcAdjustment;
131     DCHECK_ALIGNED(diff, 2u);
132     if ((diff >> 20) != 0 && (diff >> 20) != 0xfffu) {
133       LOG(ERROR) << "Target out of range: " << diff;
134       return 0xffffffffu;  // Fails code diff later.
135     }
136     return kBneWPlus0 | ((diff >> 1) & 0x7ffu)          // imm11
137                       | (((diff >> 12) & 0x3fu) << 16)  // imm6
138                       | (((diff >> 18) & 1) << 13)      // J1
139                       | (((diff >> 19) & 1) << 11)      // J2
140                       | (((diff >> 20) & 1) << 26);     // S
141   }
142 
Create2MethodsWithGap(const ArrayRef<const uint8_t> & method1_code,const ArrayRef<const LinkerPatch> & method1_patches,const ArrayRef<const uint8_t> & last_method_code,const ArrayRef<const LinkerPatch> & last_method_patches,uint32_t distance_without_thunks)143   uint32_t Create2MethodsWithGap(const ArrayRef<const uint8_t>& method1_code,
144                                  const ArrayRef<const LinkerPatch>& method1_patches,
145                                  const ArrayRef<const uint8_t>& last_method_code,
146                                  const ArrayRef<const LinkerPatch>& last_method_patches,
147                                  uint32_t distance_without_thunks) {
148     CHECK_EQ(distance_without_thunks % kArmAlignment, 0u);
149     uint32_t method1_offset =
150         kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
151     AddCompiledMethod(MethodRef(1u), method1_code, method1_patches);
152     const uint32_t gap_start = method1_offset + method1_code.size();
153 
154     // We want to put the last method at a very precise offset.
155     const uint32_t last_method_offset = method1_offset + distance_without_thunks;
156     CHECK_ALIGNED(last_method_offset, kArmAlignment);
157     const uint32_t gap_end = last_method_offset - sizeof(OatQuickMethodHeader);
158 
159     // Fill the gap with intermediate methods in chunks of 2MiB and the first in [2MiB, 4MiB).
160     // (This allows deduplicating the small chunks to avoid using 32MiB of memory for +-16MiB
161     // offsets by this test. Making the first chunk bigger makes it easy to give all intermediate
162     // methods the same alignment of the end, so the thunk insertion adds a predictable size as
163     // long as it's after the first chunk.)
164     uint32_t method_idx = 2u;
165     constexpr uint32_t kSmallChunkSize = 2 * MB;
166     std::vector<uint8_t> gap_code;
167     uint32_t gap_size = gap_end - gap_start;
168     uint32_t num_small_chunks = std::max(gap_size / kSmallChunkSize, 1u) - 1u;
169     uint32_t chunk_start = gap_start;
170     uint32_t chunk_size = gap_size - num_small_chunks * kSmallChunkSize;
171     for (uint32_t i = 0; i <= num_small_chunks; ++i) {  // num_small_chunks+1 iterations.
172       uint32_t chunk_code_size =
173           chunk_size - CodeAlignmentSize(chunk_start) - sizeof(OatQuickMethodHeader);
174       gap_code.resize(chunk_code_size, 0u);
175       AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(gap_code));
176       method_idx += 1u;
177       chunk_start += chunk_size;
178       chunk_size = kSmallChunkSize;  // For all but the first chunk.
179       DCHECK_EQ(CodeAlignmentSize(gap_end), CodeAlignmentSize(chunk_start));
180     }
181 
182     // Add the last method and link
183     AddCompiledMethod(MethodRef(method_idx), last_method_code, last_method_patches);
184     Link();
185 
186     // Check assumptions.
187     CHECK_EQ(GetMethodOffset(1), method1_offset);
188     auto last_result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
189     CHECK(last_result.first);
190     // There may be a thunk before method2.
191     if (last_result.second != last_method_offset + 1 /* thumb mode */) {
192       // Thunk present. Check that there's only one.
193       uint32_t thunk_end =
194           CompiledCode::AlignCode(gap_end, InstructionSet::kThumb2) + MethodCallThunkSize();
195       uint32_t header_offset = thunk_end + CodeAlignmentSize(thunk_end);
196       CHECK_EQ(last_result.second,
197                header_offset + sizeof(OatQuickMethodHeader) + 1 /* thumb mode */);
198     }
199     return method_idx;
200   }
201 
GetMethodOffset(uint32_t method_idx)202   uint32_t GetMethodOffset(uint32_t method_idx) {
203     auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
204     CHECK(result.first);
205     CHECK_NE(result.second & 1u, 0u);
206     return result.second - 1 /* thumb mode */;
207   }
208 
CompileThunk(const LinkerPatch & patch,std::string * debug_name=nullptr)209   std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
210                                     /*out*/ std::string* debug_name = nullptr) {
211     OptimizingUnitTestHelper helper;
212     HGraph* graph = helper.CreateGraph();
213     CompilerOptions compiler_options;
214     arm::CodeGeneratorARMVIXL codegen(graph, compiler_options);
215     ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
216     codegen.EmitThunkCode(patch, &code, debug_name);
217     return std::vector<uint8_t>(code.begin(), code.end());
218   }
219 
AddCompiledMethod(MethodReference method_ref,const ArrayRef<const uint8_t> & code,const ArrayRef<const LinkerPatch> & patches=ArrayRef<const LinkerPatch> ())220   void AddCompiledMethod(
221       MethodReference method_ref,
222       const ArrayRef<const uint8_t>& code,
223       const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
224     RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
225 
226     // Make sure the ThunkProvider has all the necessary thunks.
227     for (const LinkerPatch& patch : patches) {
228       if (patch.GetType() == LinkerPatch::Type::kCallEntrypoint ||
229           patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
230           patch.GetType() == LinkerPatch::Type::kCallRelative) {
231         std::string debug_name;
232         std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
233         thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
234       }
235     }
236   }
237 
CompileMethodCallThunk()238   std::vector<uint8_t> CompileMethodCallThunk() {
239     LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
240                                                        /* target_dex_file*/ nullptr,
241                                                        /* target_method_idx */ 0u);
242     return CompileThunk(patch);
243   }
244 
MethodCallThunkSize()245   uint32_t MethodCallThunkSize() {
246     return CompileMethodCallThunk().size();
247   }
248 
CheckThunk(uint32_t thunk_offset)249   bool CheckThunk(uint32_t thunk_offset) {
250     const std::vector<uint8_t> expected_code = CompileMethodCallThunk();
251     if (output_.size() < thunk_offset + expected_code.size()) {
252       LOG(ERROR) << "output_.size() == " << output_.size() << " < "
253           << "thunk_offset + expected_code.size() == " << (thunk_offset + expected_code.size());
254       return false;
255     }
256     ArrayRef<const uint8_t> linked_code(&output_[thunk_offset], expected_code.size());
257     if (linked_code == ArrayRef<const uint8_t>(expected_code)) {
258       return true;
259     }
260     // Log failure info.
261     DumpDiff(ArrayRef<const uint8_t>(expected_code), linked_code);
262     return false;
263   }
264 
GenNopsAndBl(size_t num_nops,uint32_t bl)265   std::vector<uint8_t> GenNopsAndBl(size_t num_nops, uint32_t bl) {
266     std::vector<uint8_t> result;
267     result.reserve(num_nops * 2u + 4u);
268     for (size_t i = 0; i != num_nops; ++i) {
269       PushBackInsn(&result, kNopInsn);
270     }
271     PushBackInsn(&result, bl);
272     return result;
273   }
274 
275   void TestStringBssEntry(uint32_t bss_begin, uint32_t string_entry_offset);
276   void TestStringReference(uint32_t string_offset);
277   void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset);
278 
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg,bool narrow)279   static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
280                                                   uint32_t holder_reg,
281                                                   bool narrow) {
282     return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow);
283   }
284 
EncodeBakerReadBarrierArrayData(uint32_t base_reg)285   static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
286     return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg);
287   }
288 
EncodeBakerReadBarrierGcRootData(uint32_t root_reg,bool narrow)289   static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
290     return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow);
291   }
292 
CompileBakerOffsetThunk(uint32_t base_reg,uint32_t holder_reg,bool narrow)293   std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg,
294                                                uint32_t holder_reg,
295                                                bool narrow) {
296     const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
297         /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
298     return CompileThunk(patch);
299   }
300 
CompileBakerArrayThunk(uint32_t base_reg)301   std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
302     LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
303         /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
304     return CompileThunk(patch);
305   }
306 
CompileBakerGcRootThunk(uint32_t root_reg,bool narrow)307   std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) {
308     LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
309         /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow));
310     return CompileThunk(patch);
311   }
312 
GetOutputInsn32(uint32_t offset)313   uint32_t GetOutputInsn32(uint32_t offset) {
314     CHECK_LE(offset, output_.size());
315     CHECK_GE(output_.size() - offset, 4u);
316     return (static_cast<uint32_t>(output_[offset]) << 16) |
317            (static_cast<uint32_t>(output_[offset + 1]) << 24) |
318            (static_cast<uint32_t>(output_[offset + 2]) << 0) |
319            (static_cast<uint32_t>(output_[offset + 3]) << 8);
320   }
321 
GetOutputInsn16(uint32_t offset)322   uint16_t GetOutputInsn16(uint32_t offset) {
323     CHECK_LE(offset, output_.size());
324     CHECK_GE(output_.size() - offset, 2u);
325     return (static_cast<uint32_t>(output_[offset]) << 0) |
326            (static_cast<uint32_t>(output_[offset + 1]) << 8);
327   }
328 
329   void TestBakerFieldWide(uint32_t offset, uint32_t ref_reg);
330   void TestBakerFieldNarrow(uint32_t offset, uint32_t ref_reg);
331 };
332 
333 const uint8_t Thumb2RelativePatcherTest::kCallRawCode[] = {
334     0x00, 0xf0, 0x00, 0xf8
335 };
336 
337 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kCallCode(kCallRawCode);
338 
339 const uint8_t Thumb2RelativePatcherTest::kNopRawCode[] = {
340     0x00, 0xbf
341 };
342 
343 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kNopCode(kNopRawCode);
344 
345 const uint8_t Thumb2RelativePatcherTest::kUnpatchedPcRelativeRawCode[] = {
346     0x40, 0xf2, 0x00, 0x00,   // MOVW r0, #0 (placeholder)
347     0xc0, 0xf2, 0x00, 0x00,   // MOVT r0, #0 (placeholder)
348     0x78, 0x44,               // ADD r0, pc
349 };
350 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kUnpatchedPcRelativeCode(
351     kUnpatchedPcRelativeRawCode);
352 const uint32_t Thumb2RelativePatcherTest::kPcInsnOffset = 8u;
353 
TestStringBssEntry(uint32_t bss_begin,uint32_t string_entry_offset)354 void Thumb2RelativePatcherTest::TestStringBssEntry(uint32_t bss_begin,
355                                                    uint32_t string_entry_offset) {
356   constexpr uint32_t kStringIndex = 1u;
357   string_index_to_offset_map_.Put(kStringIndex, string_entry_offset);
358   bss_begin_ = bss_begin;
359   const LinkerPatch patches[] = {
360       LinkerPatch::StringBssEntryPatch(0u, nullptr, kPcInsnOffset, kStringIndex),
361       LinkerPatch::StringBssEntryPatch(4u, nullptr, kPcInsnOffset, kStringIndex),
362   };
363   CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), bss_begin_ + string_entry_offset);
364 }
365 
TestStringReference(uint32_t string_offset)366 void Thumb2RelativePatcherTest::TestStringReference(uint32_t string_offset) {
367   constexpr uint32_t kStringIndex = 1u;
368   string_index_to_offset_map_.Put(kStringIndex, string_offset);
369   const LinkerPatch patches[] = {
370       LinkerPatch::RelativeStringPatch(0u, nullptr, kPcInsnOffset, kStringIndex),
371       LinkerPatch::RelativeStringPatch(4u, nullptr, kPcInsnOffset, kStringIndex),
372   };
373   CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), string_offset);
374 }
375 
CheckPcRelativePatch(const ArrayRef<const LinkerPatch> & patches,uint32_t target_offset)376 void Thumb2RelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches,
377                                                      uint32_t target_offset) {
378   AddCompiledMethod(MethodRef(1u), kUnpatchedPcRelativeCode, ArrayRef<const LinkerPatch>(patches));
379   Link();
380 
381   uint32_t method1_offset = GetMethodOffset(1u);
382   uint32_t pc_base_offset = method1_offset + kPcInsnOffset + 4u /* PC adjustment */;
383   uint32_t diff = target_offset - pc_base_offset;
384   // Distribute the bits of the diff between the MOVW and MOVT:
385   uint32_t diffw = diff & 0xffffu;
386   uint32_t difft = diff >> 16;
387   uint32_t movw = 0xf2400000u |           // MOVW r0, #0 (placeholder),
388       ((diffw & 0xf000u) << (16 - 12)) |  // move imm4 from bits 12-15 to bits 16-19,
389       ((diffw & 0x0800u) << (26 - 11)) |  // move imm from bit 11 to bit 26,
390       ((diffw & 0x0700u) << (12 - 8)) |   // move imm3 from bits 8-10 to bits 12-14,
391       ((diffw & 0x00ffu));                // keep imm8 at bits 0-7.
392   uint32_t movt = 0xf2c00000u |           // MOVT r0, #0 (placeholder),
393       ((difft & 0xf000u) << (16 - 12)) |  // move imm4 from bits 12-15 to bits 16-19,
394       ((difft & 0x0800u) << (26 - 11)) |  // move imm from bit 11 to bit 26,
395       ((difft & 0x0700u) << (12 - 8)) |   // move imm3 from bits 8-10 to bits 12-14,
396       ((difft & 0x00ffu));                // keep imm8 at bits 0-7.
397   const uint8_t expected_code[] = {
398       static_cast<uint8_t>(movw >> 16), static_cast<uint8_t>(movw >> 24),
399       static_cast<uint8_t>(movw >> 0), static_cast<uint8_t>(movw >> 8),
400       static_cast<uint8_t>(movt >> 16), static_cast<uint8_t>(movt >> 24),
401       static_cast<uint8_t>(movt >> 0), static_cast<uint8_t>(movt >> 8),
402       0x78, 0x44,
403   };
404   EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
405 }
406 
TEST_F(Thumb2RelativePatcherTest,CallSelf)407 TEST_F(Thumb2RelativePatcherTest, CallSelf) {
408   const LinkerPatch patches[] = {
409       LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
410   };
411   AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
412   Link();
413 
414   static const uint8_t expected_code[] = {
415       0xff, 0xf7, 0xfe, 0xff
416   };
417   EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
418 }
419 
TEST_F(Thumb2RelativePatcherTest,CallOther)420 TEST_F(Thumb2RelativePatcherTest, CallOther) {
421   const LinkerPatch method1_patches[] = {
422       LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
423   };
424   AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(method1_patches));
425   const LinkerPatch method2_patches[] = {
426       LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
427   };
428   AddCompiledMethod(MethodRef(2u), kCallCode, ArrayRef<const LinkerPatch>(method2_patches));
429   Link();
430 
431   uint32_t method1_offset = GetMethodOffset(1u);
432   uint32_t method2_offset = GetMethodOffset(2u);
433   uint32_t diff_after = method2_offset - (method1_offset + 4u /* PC adjustment */);
434   ASSERT_EQ(diff_after & 1u, 0u);
435   ASSERT_LT(diff_after >> 1, 1u << 8);  // Simple encoding, (diff_after >> 1) fits into 8 bits.
436   static const uint8_t method1_expected_code[] = {
437       0x00, 0xf0, static_cast<uint8_t>(diff_after >> 1), 0xf8
438   };
439   EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code)));
440   uint32_t diff_before = method1_offset - (method2_offset + 4u /* PC adjustment */);
441   ASSERT_EQ(diff_before & 1u, 0u);
442   ASSERT_GE(diff_before, -1u << 9);  // Simple encoding, -256 <= (diff >> 1) < 0.
443   auto method2_expected_code = GenNopsAndBl(0u, kBlMinus256 | ((diff_before >> 1) & 0xffu));
444   EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code)));
445 }
446 
TEST_F(Thumb2RelativePatcherTest,CallTrampoline)447 TEST_F(Thumb2RelativePatcherTest, CallTrampoline) {
448   const LinkerPatch patches[] = {
449       LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
450   };
451   AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
452   Link();
453 
454   uint32_t method1_offset = GetMethodOffset(1u);
455   uint32_t diff = kTrampolineOffset - (method1_offset + 4u);
456   ASSERT_EQ(diff & 1u, 0u);
457   ASSERT_GE(diff, -1u << 9);  // Simple encoding, -256 <= (diff >> 1) < 0 (checked as unsigned).
458   auto expected_code = GenNopsAndBl(0u, kBlMinus256 | ((diff >> 1) & 0xffu));
459   EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
460 }
461 
TEST_F(Thumb2RelativePatcherTest,CallTrampolineTooFar)462 TEST_F(Thumb2RelativePatcherTest, CallTrampolineTooFar) {
463   constexpr uint32_t missing_method_index = 1024u;
464   auto last_method_raw_code = GenNopsAndBl(3u, kBlPlus0);
465   constexpr uint32_t bl_offset_in_last_method = 3u * 2u;  // After NOPs.
466   ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
467   ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
468   const LinkerPatch last_method_patches[] = {
469       LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, missing_method_index),
470   };
471 
472   constexpr uint32_t just_over_max_negative_disp = 16 * MB + 2 - 4u /* PC adjustment */;
473   uint32_t last_method_idx = Create2MethodsWithGap(
474       kNopCode,
475       ArrayRef<const LinkerPatch>(),
476       last_method_code,
477       ArrayRef<const LinkerPatch>(last_method_patches),
478       just_over_max_negative_disp - bl_offset_in_last_method);
479   uint32_t method1_offset = GetMethodOffset(1u);
480   uint32_t last_method_offset = GetMethodOffset(last_method_idx);
481   ASSERT_EQ(method1_offset,
482             last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
483   ASSERT_FALSE(method_offset_map_.FindMethodOffset(MethodRef(missing_method_index)).first);
484 
485   // Check linked code.
486   uint32_t thunk_offset = CompiledCode::AlignCode(
487       last_method_offset + last_method_code.size(), InstructionSet::kThumb2);
488   uint32_t diff =
489       thunk_offset - (last_method_offset + bl_offset_in_last_method + 4u /* PC adjustment */);
490   ASSERT_TRUE(IsAligned<2u>(diff));
491   ASSERT_LT(diff >> 1, 1u << 8);  // Simple encoding, (diff >> 1) fits into 8 bits.
492   auto expected_code = GenNopsAndBl(3u, kBlPlus0 | ((diff >> 1) & 0xffu));
493   EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
494                                 ArrayRef<const uint8_t>(expected_code)));
495   EXPECT_TRUE(CheckThunk(thunk_offset));
496 }
497 
TEST_F(Thumb2RelativePatcherTest,CallOtherAlmostTooFarAfter)498 TEST_F(Thumb2RelativePatcherTest, CallOtherAlmostTooFarAfter) {
499   auto method1_raw_code = GenNopsAndBl(3u, kBlPlus0);
500   constexpr uint32_t bl_offset_in_method1 = 3u * 2u;  // After NOPs.
501   ArrayRef<const uint8_t> method1_code(method1_raw_code);
502   ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
503   const uint32_t kExpectedLastMethodIdx = 9u;  // Based on 2MiB chunks in Create2MethodsWithGap().
504   const LinkerPatch method1_patches[] = {
505       LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
506   };
507 
508   constexpr uint32_t max_positive_disp = 16 * MB - 2u + 4u /* PC adjustment */;
509   uint32_t last_method_idx = Create2MethodsWithGap(method1_code,
510                                                    ArrayRef<const LinkerPatch>(method1_patches),
511                                                    kNopCode,
512                                                    ArrayRef<const LinkerPatch>(),
513                                                    bl_offset_in_method1 + max_positive_disp);
514   ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
515 
516   // Check linked code.
517   auto expected_code = GenNopsAndBl(3u, kBlPlusMax);
518   EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
519 }
520 
TEST_F(Thumb2RelativePatcherTest,CallOtherAlmostTooFarBefore)521 TEST_F(Thumb2RelativePatcherTest, CallOtherAlmostTooFarBefore) {
522   auto last_method_raw_code = GenNopsAndBl(2u, kBlPlus0);
523   constexpr uint32_t bl_offset_in_last_method = 2u * 2u;  // After NOPs.
524   ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
525   ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
526   const LinkerPatch last_method_patches[] = {
527       LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
528   };
529 
530   constexpr uint32_t max_negative_disp = 16 * MB - 4u /* PC adjustment */;
531   uint32_t last_method_idx = Create2MethodsWithGap(kNopCode,
532                                                    ArrayRef<const LinkerPatch>(),
533                                                    last_method_code,
534                                                    ArrayRef<const LinkerPatch>(last_method_patches),
535                                                    max_negative_disp - bl_offset_in_last_method);
536   uint32_t method1_offset = GetMethodOffset(1u);
537   uint32_t last_method_offset = GetMethodOffset(last_method_idx);
538   ASSERT_EQ(method1_offset, last_method_offset + bl_offset_in_last_method - max_negative_disp);
539 
540   // Check linked code.
541   auto expected_code = GenNopsAndBl(2u, kBlMinusMax);
542   EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
543                                 ArrayRef<const uint8_t>(expected_code)));
544 }
545 
TEST_F(Thumb2RelativePatcherTest,CallOtherJustTooFarAfter)546 TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarAfter) {
547   auto method1_raw_code = GenNopsAndBl(2u, kBlPlus0);
548   constexpr uint32_t bl_offset_in_method1 = 2u * 2u;  // After NOPs.
549   ArrayRef<const uint8_t> method1_code(method1_raw_code);
550   ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
551   const uint32_t kExpectedLastMethodIdx = 9u;  // Based on 2MiB chunks in Create2MethodsWithGap().
552   const LinkerPatch method1_patches[] = {
553       LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
554   };
555 
556   constexpr uint32_t just_over_max_positive_disp = 16 * MB + 4u /* PC adjustment */;
557   uint32_t last_method_idx = Create2MethodsWithGap(
558       method1_code,
559       ArrayRef<const LinkerPatch>(method1_patches),
560       kNopCode,
561       ArrayRef<const LinkerPatch>(),
562       bl_offset_in_method1 + just_over_max_positive_disp);
563   ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
564   uint32_t method_after_thunk_idx = last_method_idx;
565   if (sizeof(OatQuickMethodHeader) < kArmAlignment) {
566     // The thunk needs to start on a kArmAlignment-aligned address before the address where the
567     // last method would have been if there was no thunk. If the size of the OatQuickMethodHeader
568     // is at least kArmAlignment, the thunk start shall fit between the previous filler method
569     // and that address. Otherwise, it shall be inserted before that filler method.
570     method_after_thunk_idx -= 1u;
571   }
572 
573   uint32_t method1_offset = GetMethodOffset(1u);
574   uint32_t method_after_thunk_offset = GetMethodOffset(method_after_thunk_idx);
575   ASSERT_TRUE(IsAligned<kArmAlignment>(method_after_thunk_offset));
576   uint32_t method_after_thunk_header_offset =
577       method_after_thunk_offset - sizeof(OatQuickMethodHeader);
578   uint32_t thunk_size = MethodCallThunkSize();
579   uint32_t thunk_offset = RoundDown(method_after_thunk_header_offset - thunk_size, kArmAlignment);
580   DCHECK_EQ(thunk_offset + thunk_size + CodeAlignmentSize(thunk_offset + thunk_size),
581             method_after_thunk_header_offset);
582   ASSERT_TRUE(IsAligned<kArmAlignment>(thunk_offset));
583   uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1 + 4u /* PC adjustment */);
584   ASSERT_TRUE(IsAligned<2u>(diff));
585   ASSERT_GE(diff, 16 * MB - (1u << 22));  // Simple encoding, unknown bits fit into imm10:imm11:0.
586   auto expected_code =
587       GenNopsAndBl(2u, 0xf000d000 | ((diff >> 1) & 0x7ffu) | (((diff >> 12) & 0x3ffu) << 16));
588   EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
589   CheckThunk(thunk_offset);
590 }
591 
TEST_F(Thumb2RelativePatcherTest,CallOtherJustTooFarBefore)592 TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarBefore) {
593   auto last_method_raw_code = GenNopsAndBl(3u, kBlPlus0);
594   constexpr uint32_t bl_offset_in_last_method = 3u * 2u;  // After NOPs.
595   ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
596   ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
597   const LinkerPatch last_method_patches[] = {
598       LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
599   };
600 
601   constexpr uint32_t just_over_max_negative_disp = 16 * MB + 2 - 4u /* PC adjustment */;
602   uint32_t last_method_idx = Create2MethodsWithGap(
603       kNopCode,
604       ArrayRef<const LinkerPatch>(),
605       last_method_code,
606       ArrayRef<const LinkerPatch>(last_method_patches),
607       just_over_max_negative_disp - bl_offset_in_last_method);
608   uint32_t method1_offset = GetMethodOffset(1u);
609   uint32_t last_method_offset = GetMethodOffset(last_method_idx);
610   ASSERT_EQ(method1_offset,
611             last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
612 
613   // Check linked code.
614   uint32_t thunk_offset = CompiledCode::AlignCode(
615       last_method_offset + last_method_code.size(), InstructionSet::kThumb2);
616   uint32_t diff =
617       thunk_offset - (last_method_offset + bl_offset_in_last_method + 4u /* PC adjustment */);
618   ASSERT_TRUE(IsAligned<2u>(diff));
619   ASSERT_LT(diff >> 1, 1u << 8);  // Simple encoding, (diff >> 1) fits into 8 bits.
620   auto expected_code = GenNopsAndBl(3u, kBlPlus0 | ((diff >> 1) & 0xffu));
621   EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
622                                 ArrayRef<const uint8_t>(expected_code)));
623   EXPECT_TRUE(CheckThunk(thunk_offset));
624 }
625 
TEST_F(Thumb2RelativePatcherTest,StringBssEntry1)626 TEST_F(Thumb2RelativePatcherTest, StringBssEntry1) {
627   TestStringBssEntry(0x00ff0000u, 0x00fcu);
628   ASSERT_LT(GetMethodOffset(1u), 0xfcu);
629 }
630 
TEST_F(Thumb2RelativePatcherTest,StringBssEntry2)631 TEST_F(Thumb2RelativePatcherTest, StringBssEntry2) {
632   TestStringBssEntry(0x02ff0000u, 0x05fcu);
633   ASSERT_LT(GetMethodOffset(1u), 0xfcu);
634 }
635 
TEST_F(Thumb2RelativePatcherTest,StringBssEntry3)636 TEST_F(Thumb2RelativePatcherTest, StringBssEntry3) {
637   TestStringBssEntry(0x08ff0000u, 0x08fcu);
638   ASSERT_LT(GetMethodOffset(1u), 0xfcu);
639 }
640 
TEST_F(Thumb2RelativePatcherTest,StringBssEntry4)641 TEST_F(Thumb2RelativePatcherTest, StringBssEntry4) {
642   TestStringBssEntry(0xd0ff0000u, 0x60fcu);
643   ASSERT_LT(GetMethodOffset(1u), 0xfcu);
644 }
645 
TEST_F(Thumb2RelativePatcherTest,StringReference1)646 TEST_F(Thumb2RelativePatcherTest, StringReference1) {
647   TestStringReference(0x00ff00fcu);
648   ASSERT_LT(GetMethodOffset(1u), 0xfcu);
649 }
650 
TEST_F(Thumb2RelativePatcherTest,StringReference2)651 TEST_F(Thumb2RelativePatcherTest, StringReference2) {
652   TestStringReference(0x02ff05fcu);
653   ASSERT_LT(GetMethodOffset(1u), 0xfcu);
654 }
655 
TEST_F(Thumb2RelativePatcherTest,StringReference3)656 TEST_F(Thumb2RelativePatcherTest, StringReference3) {
657   TestStringReference(0x08ff08fcu);
658   ASSERT_LT(GetMethodOffset(1u), 0xfcu);
659 }
660 
TEST_F(Thumb2RelativePatcherTest,StringReference4)661 TEST_F(Thumb2RelativePatcherTest, StringReference4) {
662   TestStringReference(0xd0ff60fcu);
663   ASSERT_LT(GetMethodOffset(1u), 0xfcu);
664 }
665 
TEST_F(Thumb2RelativePatcherTest,EntrypointCall)666 TEST_F(Thumb2RelativePatcherTest, EntrypointCall) {
667   constexpr uint32_t kEntrypointOffset = 512;
668   const LinkerPatch patches[] = {
669       LinkerPatch::CallEntrypointPatch(0u, kEntrypointOffset),
670   };
671   AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
672   Link();
673 
674   uint32_t method_offset = GetMethodOffset(1u);
675   uint32_t thunk_offset = CompiledCode::AlignCode(method_offset + kCallCode.size(),
676                                                   InstructionSet::kThumb2);
677   uint32_t diff = thunk_offset - method_offset - kPcAdjustment;
678   ASSERT_TRUE(IsAligned<2u>(diff));
679   ASSERT_LT(diff >> 1, 1u << 8);  // Simple encoding, (diff >> 1) fits into 8 bits.
680   auto expected_code = GenNopsAndBl(0u, kBlPlus0 | ((diff >> 1) & 0xffu));
681   EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
682 
683   // Verify the thunk.
684   uint32_t ldr_pc_tr_offset =
685       0xf8d00000 |                        // LDR Rt, [Rn, #<imm12>]
686       (/* tr */ 9 << 16) |                // Rn = TR
687       (/* pc */ 15 << 12) |               // Rt = PC
688       kEntrypointOffset;                  // imm12
689   uint16_t bkpt = 0xbe00;
690   ASSERT_LE(6u, output_.size() - thunk_offset);
691   EXPECT_EQ(ldr_pc_tr_offset, GetOutputInsn32(thunk_offset));
692   EXPECT_EQ(bkpt, GetOutputInsn16(thunk_offset + 4u));
693 }
694 
695 const uint32_t kBakerValidRegs[] = {
696     0,  1,  2,  3,  4,  5,  6,  7,
697     9, 10, 11,                      // r8 (rMR), IP, SP, LR and PC are reserved.
698 };
699 
700 const uint32_t kBakerValidRegsNarrow[] = {
701     0,  1,  2,  3,  4,  5,  6,  7,
702 };
703 
TestBakerFieldWide(uint32_t offset,uint32_t ref_reg)704 void Thumb2RelativePatcherTest::TestBakerFieldWide(uint32_t offset, uint32_t ref_reg) {
705   DCHECK_ALIGNED(offset, 4u);
706   DCHECK_LT(offset, 4 * KB);
707   constexpr size_t kMethodCodeSize = 8u;
708   constexpr size_t kLiteralOffset = 0u;
709   uint32_t method_idx = 0u;
710   for (uint32_t base_reg : kBakerValidRegs) {
711     for (uint32_t holder_reg : kBakerValidRegs) {
712       uint32_t ldr = kLdrWInsn | offset | (base_reg << 16) | (ref_reg << 12);
713       const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
714       ASSERT_EQ(kMethodCodeSize, raw_code.size());
715       ArrayRef<const uint8_t> code(raw_code);
716       uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
717           base_reg, holder_reg, /* narrow */ false);
718       const LinkerPatch patches[] = {
719           LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
720       };
721       ++method_idx;
722       AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
723     }
724   }
725   Link();
726 
727   // All thunks are at the end.
728   uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
729   method_idx = 0u;
730   for (uint32_t base_reg : kBakerValidRegs) {
731     for (uint32_t holder_reg : kBakerValidRegs) {
732       ++method_idx;
733       uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
734       uint32_t ldr = kLdrWInsn | offset | (base_reg << 16) | (ref_reg << 12);
735       const std::vector<uint8_t> expected_code = RawCode({bne, ldr});
736       ASSERT_EQ(kMethodCodeSize, expected_code.size()) << "bne=0x" << std::hex << bne;
737       ASSERT_TRUE(
738           CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
739 
740       std::vector<uint8_t> expected_thunk =
741           CompileBakerOffsetThunk(base_reg, holder_reg, /* narrow */ false);
742       ASSERT_GT(output_.size(), thunk_offset);
743       ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
744       ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
745                                              expected_thunk.size());
746       if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
747         DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
748         ASSERT_TRUE(false);
749       }
750 
751       size_t gray_check_offset = thunk_offset;
752       if (holder_reg == base_reg) {
753         // Verify that the null-check uses the correct register, i.e. holder_reg.
754         if (holder_reg < 8) {
755           ASSERT_GE(output_.size() - gray_check_offset, 2u);
756           ASSERT_EQ(0xb100 | holder_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
757           gray_check_offset +=2u;
758         } else {
759           ASSERT_GE(output_.size() - gray_check_offset, 6u);
760           ASSERT_EQ(0xf1b00f00u | (holder_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
761           ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u);  // BEQ
762           gray_check_offset += 6u;
763         }
764       }
765       // Verify that the lock word for gray bit check is loaded from the holder address.
766       ASSERT_GE(output_.size() - gray_check_offset,
767                 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
768       const uint32_t load_lock_word =
769           kLdrWInsn |
770           (holder_reg << 16) |
771           (/* IP */ 12 << 12) |
772           mirror::Object::MonitorOffset().Uint32Value();
773       ASSERT_EQ(load_lock_word, GetOutputInsn32(gray_check_offset));
774       // Verify the gray bit check.
775       DCHECK_GE(LockWord::kReadBarrierStateShift, 8u);  // ROR modified immediate.
776       uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
777       const uint32_t tst_gray_bit_without_offset =
778           0xf0100f00 | (/* IP */ 12 << 16)
779                      | (((ror_shift >> 4) & 1) << 26)   // i
780                      | (((ror_shift >> 1) & 7) << 12)   // imm3
781                      | ((ror_shift & 1) << 7);          // imm8, ROR('1':imm8<7:0>, ror_shift).
782       EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(gray_check_offset + 4u));
783       EXPECT_EQ(0xd100u, GetOutputInsn16(gray_check_offset + 8u) & 0xff00u);  // BNE
784       // Verify the fake dependency (skip "ADD LR, LR, #ldr_offset").
785       const uint32_t fake_dependency =
786           0xeb000010 |              // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
787           (/* IP */ 12) |           // Rm = IP
788           (base_reg << 16) |        // Rn = base_reg
789           (base_reg << 8);          // Rd = base_reg
790       EXPECT_EQ(fake_dependency, GetOutputInsn32(gray_check_offset + 14u));
791       // Do not check the rest of the implementation.
792 
793       // The next thunk follows on the next aligned offset.
794       thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
795     }
796   }
797 }
798 
TestBakerFieldNarrow(uint32_t offset,uint32_t ref_reg)799 void Thumb2RelativePatcherTest::TestBakerFieldNarrow(uint32_t offset, uint32_t ref_reg) {
800   DCHECK_ALIGNED(offset, 4u);
801   DCHECK_LT(offset, 32u);
802   constexpr size_t kMethodCodeSize = 6u;
803   constexpr size_t kLiteralOffset = 0u;
804   uint32_t method_idx = 0u;
805   for (uint32_t base_reg : kBakerValidRegs) {
806     if (base_reg >= 8u) {
807       continue;
808     }
809     for (uint32_t holder_reg : kBakerValidRegs) {
810       uint32_t ldr = kLdrInsn | (offset << (6 - 2)) | (base_reg << 3) | ref_reg;
811       const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
812       ASSERT_EQ(kMethodCodeSize, raw_code.size());
813       ArrayRef<const uint8_t> code(raw_code);
814       uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
815           base_reg, holder_reg, /* narrow */ true);
816       const LinkerPatch patches[] = {
817           LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
818       };
819       ++method_idx;
820       AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
821     }
822   }
823   Link();
824 
825   // All thunks are at the end.
826   uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
827   method_idx = 0u;
828   for (uint32_t base_reg : kBakerValidRegs) {
829     if (base_reg >= 8u) {
830       continue;
831     }
832     for (uint32_t holder_reg : kBakerValidRegs) {
833       ++method_idx;
834       uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
835       uint32_t ldr = kLdrInsn | (offset << (6 - 2)) | (base_reg << 3) | ref_reg;
836       const std::vector<uint8_t> expected_code = RawCode({bne, ldr});
837       ASSERT_EQ(kMethodCodeSize, expected_code.size()) << "bne=0x" << std::hex << bne;
838       ASSERT_TRUE(
839           CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
840 
841       std::vector<uint8_t> expected_thunk =
842           CompileBakerOffsetThunk(base_reg, holder_reg, /* narrow */ true);
843       ASSERT_GT(output_.size(), thunk_offset);
844       ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
845       ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
846                                              expected_thunk.size());
847       if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
848         DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
849         ASSERT_TRUE(false);
850       }
851 
852       size_t gray_check_offset = thunk_offset;
853       if (holder_reg == base_reg) {
854         // Verify that the null-check uses the correct register, i.e. holder_reg.
855         if (holder_reg < 8) {
856           ASSERT_GE(output_.size() - gray_check_offset, 2u);
857           ASSERT_EQ(0xb100 | holder_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
858           gray_check_offset +=2u;
859         } else {
860           ASSERT_GE(output_.size() - gray_check_offset, 6u);
861           ASSERT_EQ(0xf1b00f00u | (holder_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
862           ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u);  // BEQ
863           gray_check_offset += 6u;
864         }
865       }
866       // Verify that the lock word for gray bit check is loaded from the holder address.
867       ASSERT_GE(output_.size() - gray_check_offset,
868                 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
869       const uint32_t load_lock_word =
870           kLdrWInsn |
871           (holder_reg << 16) |
872           (/* IP */ 12 << 12) |
873           mirror::Object::MonitorOffset().Uint32Value();
874       ASSERT_EQ(load_lock_word, GetOutputInsn32(gray_check_offset));
875       // Verify the gray bit check.
876       DCHECK_GE(LockWord::kReadBarrierStateShift, 8u);  // ROR modified immediate.
877       uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
878       const uint32_t tst_gray_bit_without_offset =
879           0xf0100f00 | (/* IP */ 12 << 16)
880                      | (((ror_shift >> 4) & 1) << 26)   // i
881                      | (((ror_shift >> 1) & 7) << 12)   // imm3
882                      | ((ror_shift & 1) << 7);          // imm8, ROR('1':imm8<7:0>, ror_shift).
883       EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(gray_check_offset + 4u));
884       EXPECT_EQ(0xd100u, GetOutputInsn16(gray_check_offset + 8u) & 0xff00u);  // BNE
885       // Verify the fake dependency (skip "ADD LR, LR, #ldr_offset").
886       const uint32_t fake_dependency =
887           0xeb000010 |              // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
888           (/* IP */ 12) |           // Rm = IP
889           (base_reg << 16) |        // Rn = base_reg
890           (base_reg << 8);          // Rd = base_reg
891       EXPECT_EQ(fake_dependency, GetOutputInsn32(gray_check_offset + 14u));
892       // Do not check the rest of the implementation.
893 
894       // The next thunk follows on the next aligned offset.
895       thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
896     }
897   }
898 }
899 
TEST_F(Thumb2RelativePatcherTest,BakerOffsetWide)900 TEST_F(Thumb2RelativePatcherTest, BakerOffsetWide) {
901   struct TestCase {
902     uint32_t offset;
903     uint32_t ref_reg;
904   };
905   static const TestCase test_cases[] = {
906       { 0u, 0u },
907       { 8u, 3u },
908       { 28u, 7u },
909       { 0xffcu, 11u },
910   };
911   for (const TestCase& test_case : test_cases) {
912     Reset();
913     TestBakerFieldWide(test_case.offset, test_case.ref_reg);
914   }
915 }
916 
TEST_F(Thumb2RelativePatcherTest,BakerOffsetNarrow)917 TEST_F(Thumb2RelativePatcherTest, BakerOffsetNarrow) {
918   struct TestCase {
919     uint32_t offset;
920     uint32_t ref_reg;
921   };
922   static const TestCase test_cases[] = {
923       { 0, 0u },
924       { 8, 3u },
925       { 28, 7u },
926   };
927   for (const TestCase& test_case : test_cases) {
928     Reset();
929     TestBakerFieldNarrow(test_case.offset, test_case.ref_reg);
930   }
931 }
932 
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkInTheMiddle)933 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddle) {
934   // One thunk in the middle with maximum distance branches to it from both sides.
935   // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
936   constexpr uint32_t kLiteralOffset1 = 6u;
937   const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
938   ArrayRef<const uint8_t> code1(raw_code1);
939   uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
940       /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
941   const LinkerPatch patches1[] = {
942       LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
943   };
944   AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
945 
946   constexpr uint32_t expected_thunk_offset =
947       kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement */ ((1 << 20) - 2u);
948   static_assert(IsAligned<kArmAlignment>(expected_thunk_offset), "Target offset must be aligned.");
949   size_t filler1_size = expected_thunk_offset -
950                         RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment);
951   std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
952   ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
953   AddCompiledMethod(MethodRef(2u), filler1_code);
954 
955   // Enforce thunk reservation with a tiny method.
956   AddCompiledMethod(MethodRef(3u), kNopCode);
957 
958   constexpr uint32_t kLiteralOffset2 = 4;
959   static_assert(IsAligned<kArmAlignment>(kLiteralOffset2 + kPcAdjustment),
960                 "PC for BNE must be aligned.");
961 
962   // Allow reaching the thunk from the very beginning of a method almost 1MiB away. Backward branch
963   // reaches the full 1MiB but we need to take PC adjustment into account. Things to subtract:
964   //   - thunk size and method 3 pre-header, rounded up (padding in between if needed)
965   //   - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
966   //   - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
967   size_t thunk_size =
968       CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false).size();
969   size_t filler2_size =
970       1 * MB - (kLiteralOffset2 + kPcAdjustment)
971              - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArmAlignment)
972              - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArmAlignment)
973              - sizeof(OatQuickMethodHeader);
974   std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 2u);
975   ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
976   AddCompiledMethod(MethodRef(4u), filler2_code);
977 
978   const std::vector<uint8_t> raw_code2 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn});
979   ArrayRef<const uint8_t> code2(raw_code2);
980   const LinkerPatch patches2[] = {
981       LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
982   };
983   AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
984 
985   Link();
986 
987   uint32_t first_method_offset = GetMethodOffset(1u);
988   uint32_t last_method_offset = GetMethodOffset(5u);
989   EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
990 
991   const uint32_t bne_max_forward = kBneWPlus0 | 0x003f2fff;
992   const uint32_t bne_max_backward = kBneWPlus0 | 0x04000000;
993   const std::vector<uint8_t> expected_code1 =
994       RawCode({kNopWInsn, kNopInsn, bne_max_forward, kLdrWInsn});
995   const std::vector<uint8_t> expected_code2 = RawCode({kNopWInsn, bne_max_backward, kLdrWInsn});
996   ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
997   ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
998 }
999 
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkBeforeFiller)1000 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkBeforeFiller) {
1001   // Based on the first part of BakerOffsetThunkInTheMiddle but the BNE is one instruction
1002   // earlier, so the thunk is emitted before the filler.
1003   // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1004   constexpr uint32_t kLiteralOffset1 = 4u;
1005   const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn});
1006   ArrayRef<const uint8_t> code1(raw_code1);
1007   uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
1008       /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
1009   const LinkerPatch patches1[] = {
1010       LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1011   };
1012   AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1013 
1014   constexpr uint32_t expected_thunk_offset =
1015       kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement + 2 */ (1u << 20);
1016   static_assert(IsAligned<kArmAlignment>(expected_thunk_offset), "Target offset must be aligned.");
1017   size_t filler1_size = expected_thunk_offset -
1018                         RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment);
1019   std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
1020   ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1021   AddCompiledMethod(MethodRef(2u), filler1_code);
1022 
1023   Link();
1024 
1025   const uint32_t bne = BneWWithOffset(kLiteralOffset1, RoundUp(raw_code1.size(), kArmAlignment));
1026   const std::vector<uint8_t> expected_code1 = RawCode({kNopWInsn, bne, kLdrWInsn, kNopInsn});
1027   ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1028 }
1029 
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkInTheMiddleUnreachableFromLast)1030 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddleUnreachableFromLast) {
1031   // Based on the BakerOffsetThunkInTheMiddle but the BNE in the last method is preceded
1032   // by NOP and cannot reach the thunk in the middle, so we emit an extra thunk at the end.
1033   // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1034   constexpr uint32_t kLiteralOffset1 = 6u;
1035   const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
1036   ArrayRef<const uint8_t> code1(raw_code1);
1037   uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
1038       /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
1039   const LinkerPatch patches1[] = {
1040       LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1041   };
1042   AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1043 
1044   constexpr uint32_t expected_thunk_offset =
1045       kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement */ ((1 << 20) - 2u);
1046   static_assert(IsAligned<kArmAlignment>(expected_thunk_offset), "Target offset must be aligned.");
1047   size_t filler1_size = expected_thunk_offset -
1048                         RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment);
1049   std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
1050   ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1051   AddCompiledMethod(MethodRef(2u), filler1_code);
1052 
1053   // Enforce thunk reservation with a tiny method.
1054   AddCompiledMethod(MethodRef(3u), kNopCode);
1055 
1056   constexpr uint32_t kReachableFromOffset2 = 4;
1057   constexpr uint32_t kLiteralOffset2 = kReachableFromOffset2 + 2;
1058   static_assert(IsAligned<kArmAlignment>(kReachableFromOffset2 + kPcAdjustment),
1059                 "PC for BNE must be aligned.");
1060 
1061   // If not for the extra NOP, this would allow reaching the thunk from the BNE
1062   // of a method 1MiB away. Backward branch reaches the full 1MiB  but we need to take
1063   // PC adjustment into account. Things to subtract:
1064   //   - thunk size and method 3 pre-header, rounded up (padding in between if needed)
1065   //   - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
1066   //   - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
1067   size_t thunk_size =
1068       CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false).size();
1069   size_t filler2_size =
1070       1 * MB - (kReachableFromOffset2 + kPcAdjustment)
1071              - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArmAlignment)
1072              - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArmAlignment)
1073              - sizeof(OatQuickMethodHeader);
1074   std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 2u);
1075   ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
1076   AddCompiledMethod(MethodRef(4u), filler2_code);
1077 
1078   // Extra 16-bit NOP compared to BakerOffsetThunkInTheMiddle.
1079   const std::vector<uint8_t> raw_code2 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
1080   ArrayRef<const uint8_t> code2(raw_code2);
1081   const LinkerPatch patches2[] = {
1082       LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
1083   };
1084   AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
1085 
1086   Link();
1087 
1088   uint32_t first_method_offset = GetMethodOffset(1u);
1089   uint32_t last_method_offset = GetMethodOffset(5u);
1090   EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
1091 
1092   const uint32_t bne_max_forward = kBneWPlus0 | 0x003f2fff;
1093   const uint32_t bne_last =
1094       BneWWithOffset(kLiteralOffset2, RoundUp(raw_code2.size(), kArmAlignment));
1095   const std::vector<uint8_t> expected_code1 =
1096       RawCode({kNopWInsn, kNopInsn, bne_max_forward, kLdrWInsn});
1097   const std::vector<uint8_t> expected_code2 =
1098       RawCode({kNopWInsn, kNopInsn, bne_last, kLdrWInsn});
1099   ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1100   ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
1101 }
1102 
TEST_F(Thumb2RelativePatcherTest,BakerArray)1103 TEST_F(Thumb2RelativePatcherTest, BakerArray) {
1104   auto ldr = [](uint32_t base_reg) {
1105     uint32_t index_reg = (base_reg == 0u) ? 1u : 0u;
1106     uint32_t ref_reg = (base_reg == 2) ? 3u : 2u;
1107     return kLdrRegLsl2 | index_reg | (base_reg << 16) | (ref_reg << 12);
1108   };
1109   constexpr size_t kMethodCodeSize = 8u;
1110   constexpr size_t kLiteralOffset = 0u;
1111   uint32_t method_idx = 0u;
1112   for (uint32_t base_reg : kBakerValidRegs) {
1113     ++method_idx;
1114     const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr(base_reg)});
1115     ASSERT_EQ(kMethodCodeSize, raw_code.size());
1116     ArrayRef<const uint8_t> code(raw_code);
1117     const LinkerPatch patches[] = {
1118         LinkerPatch::BakerReadBarrierBranchPatch(
1119             kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
1120     };
1121     AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1122   }
1123   Link();
1124 
1125   // All thunks are at the end.
1126   uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
1127   method_idx = 0u;
1128   for (uint32_t base_reg : kBakerValidRegs) {
1129     ++method_idx;
1130     uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1131     const std::vector<uint8_t> expected_code = RawCode({bne, ldr(base_reg)});
1132     ASSERT_EQ(kMethodCodeSize, expected_code.size());
1133     EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1134 
1135     std::vector<uint8_t> expected_thunk = CompileBakerArrayThunk(base_reg);
1136     ASSERT_GT(output_.size(), thunk_offset);
1137     ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1138     ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1139                                            expected_thunk.size());
1140     if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1141       DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1142       ASSERT_TRUE(false);
1143     }
1144 
1145     // Verify that the lock word for gray bit check is loaded from the correct address
1146     // before the base_reg which points to the array data.
1147     ASSERT_GE(output_.size() - thunk_offset,
1148               4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
1149     int32_t data_offset =
1150         mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
1151     int32_t offset = mirror::Object::MonitorOffset().Int32Value() - data_offset;
1152     ASSERT_LT(offset, 0);
1153     ASSERT_GT(offset, -256);
1154     const uint32_t load_lock_word =
1155         kLdrNegativeOffset |
1156         (-offset & 0xffu) |
1157         (base_reg << 16) |
1158         (/* IP */ 12 << 12);
1159     EXPECT_EQ(load_lock_word, GetOutputInsn32(thunk_offset));
1160     // Verify the gray bit check.
1161     DCHECK_GE(LockWord::kReadBarrierStateShift, 8u);  // ROR modified immediate.
1162     uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
1163     const uint32_t tst_gray_bit_without_offset =
1164         0xf0100f00 | (/* IP */ 12 << 16)
1165                    | (((ror_shift >> 4) & 1) << 26)   // i
1166                    | (((ror_shift >> 1) & 7) << 12)   // imm3
1167                    | ((ror_shift & 1) << 7);          // imm8, ROR('1':imm8<7:0>, ror_shift).
1168     EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(thunk_offset + 4u));
1169     EXPECT_EQ(0xd100u, GetOutputInsn16(thunk_offset + 8u) & 0xff00u);  // BNE
1170     // Verify the fake dependency.
1171     const uint32_t fake_dependency =
1172         0xeb000010 |              // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
1173         (/* IP */ 12) |           // Rm = IP
1174         (base_reg << 16) |        // Rn = base_reg
1175         (base_reg << 8);          // Rd = base_reg
1176     EXPECT_EQ(fake_dependency, GetOutputInsn32(thunk_offset + 14u));
1177     // Do not check the rest of the implementation.
1178 
1179     // The next thunk follows on the next aligned offset.
1180     thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
1181   }
1182 }
1183 
TEST_F(Thumb2RelativePatcherTest,BakerGcRootWide)1184 TEST_F(Thumb2RelativePatcherTest, BakerGcRootWide) {
1185   constexpr size_t kMethodCodeSize = 8u;
1186   constexpr size_t kLiteralOffset = 4u;
1187   uint32_t method_idx = 0u;
1188   for (uint32_t root_reg : kBakerValidRegs) {
1189     ++method_idx;
1190     uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (root_reg << 12);
1191     const std::vector<uint8_t> raw_code = RawCode({ldr, kBneWPlus0});
1192     ASSERT_EQ(kMethodCodeSize, raw_code.size());
1193     ArrayRef<const uint8_t> code(raw_code);
1194     const LinkerPatch patches[] = {
1195         LinkerPatch::BakerReadBarrierBranchPatch(
1196             kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
1197     };
1198     AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1199   }
1200   Link();
1201 
1202   // All thunks are at the end.
1203   uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
1204   method_idx = 0u;
1205   for (uint32_t root_reg : kBakerValidRegs) {
1206     ++method_idx;
1207     uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1208     uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (root_reg << 12);
1209     const std::vector<uint8_t> expected_code = RawCode({ldr, bne});
1210     ASSERT_EQ(kMethodCodeSize, expected_code.size());
1211     EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1212 
1213     std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg, /* narrow */ false);
1214     ASSERT_GT(output_.size(), thunk_offset);
1215     ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1216     ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1217                                            expected_thunk.size());
1218     if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1219       DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1220       ASSERT_TRUE(false);
1221     }
1222 
1223     // Verify that the fast-path null-check uses the correct register, i.e. root_reg.
1224     if (root_reg < 8) {
1225       ASSERT_GE(output_.size() - thunk_offset, 2u);
1226       ASSERT_EQ(0xb100 | root_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
1227     } else {
1228       ASSERT_GE(output_.size() - thunk_offset, 6u);
1229       ASSERT_EQ(0xf1b00f00u | (root_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
1230       ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u);  // BEQ
1231     }
1232     // Do not check the rest of the implementation.
1233 
1234     // The next thunk follows on the next aligned offset.
1235     thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
1236   }
1237 }
1238 
TEST_F(Thumb2RelativePatcherTest,BakerGcRootNarrow)1239 TEST_F(Thumb2RelativePatcherTest, BakerGcRootNarrow) {
1240   constexpr size_t kMethodCodeSize = 6u;
1241   constexpr size_t kLiteralOffset = 2u;
1242   uint32_t method_idx = 0u;
1243   for (uint32_t root_reg : kBakerValidRegsNarrow) {
1244     ++method_idx;
1245     uint32_t ldr = kLdrInsn | (/* offset */ 8 << (6 - 2)) | (/* base_reg */ 0 << 3) | root_reg;
1246     const std::vector<uint8_t> raw_code = RawCode({ldr, kBneWPlus0});
1247     ASSERT_EQ(kMethodCodeSize, raw_code.size());
1248     ArrayRef<const uint8_t> code(raw_code);
1249     const LinkerPatch patches[] = {
1250         LinkerPatch::BakerReadBarrierBranchPatch(
1251             kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
1252     };
1253     AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1254   }
1255   Link();
1256 
1257   // All thunks are at the end.
1258   uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
1259   method_idx = 0u;
1260   for (uint32_t root_reg : kBakerValidRegsNarrow) {
1261     ++method_idx;
1262     uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1263     uint32_t ldr = kLdrInsn | (/* offset */ 8 << (6 - 2)) | (/* base_reg */ 0 << 3) | root_reg;
1264     const std::vector<uint8_t> expected_code = RawCode({ldr, bne});
1265     ASSERT_EQ(kMethodCodeSize, expected_code.size());
1266     EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1267 
1268     std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg, /* narrow */ true);
1269     ASSERT_GT(output_.size(), thunk_offset);
1270     ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1271     ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1272                                            expected_thunk.size());
1273     if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1274       DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1275       ASSERT_TRUE(false);
1276     }
1277 
1278     // Verify that the fast-path null-check CBZ uses the correct register, i.e. root_reg.
1279     ASSERT_GE(output_.size() - thunk_offset, 2u);
1280     ASSERT_EQ(0xb100 | root_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
1281     // Do not check the rest of the implementation.
1282 
1283     // The next thunk follows on the next aligned offset.
1284     thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
1285   }
1286 }
1287 
TEST_F(Thumb2RelativePatcherTest,BakerGcRootOffsetBits)1288 TEST_F(Thumb2RelativePatcherTest, BakerGcRootOffsetBits) {
1289   // Test 1MiB of patches to the same thunk to stress-test different large offsets.
1290   // (The low bits are not that important but the location of the high bits is easy to get wrong.)
1291   std::vector<uint8_t> code;
1292   code.reserve(1 * MB);
1293   const size_t num_patches = 1 * MB / 8u;
1294   std::vector<LinkerPatch> patches;
1295   patches.reserve(num_patches);
1296   const uint32_t ldr =
1297       kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12);
1298   uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
1299   for (size_t i = 0; i != num_patches; ++i) {
1300     PushBackInsn(&code, ldr);
1301     PushBackInsn(&code, kBneWPlus0);
1302     patches.push_back(LinkerPatch::BakerReadBarrierBranchPatch(8u * i + 4u, encoded_data));
1303   }
1304   ASSERT_EQ(1 * MB, code.size());
1305   ASSERT_EQ(num_patches, patches.size());
1306   AddCompiledMethod(MethodRef(1u),
1307                     ArrayRef<const uint8_t>(code),
1308                     ArrayRef<const LinkerPatch>(patches));
1309   Link();
1310 
1311   // The thunk is right after the method code.
1312   DCHECK_ALIGNED(1 * MB, kArmAlignment);
1313   std::vector<uint8_t> expected_code;
1314   for (size_t i = 0; i != num_patches; ++i) {
1315     PushBackInsn(&expected_code, ldr);
1316     PushBackInsn(&expected_code, BneWWithOffset(8u * i + 4u, 1 * MB));
1317     patches.push_back(LinkerPatch::BakerReadBarrierBranchPatch(8u * i + 4u, encoded_data));
1318   }
1319   EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
1320 }
1321 
TEST_F(Thumb2RelativePatcherTest,BakerAndMethodCallInteraction)1322 TEST_F(Thumb2RelativePatcherTest, BakerAndMethodCallInteraction) {
1323   // During development, there was a `DCHECK_LE(MaxNextOffset(), next_thunk.MaxNextOffset());`
1324   // in `ArmBaseRelativePatcher::ThunkData::MakeSpaceBefore()` which does not necessarily
1325   // hold when we're reserving thunks of different sizes. This test exposes the situation
1326   // by using Baker thunks and a method call thunk.
1327 
1328   // Add a method call patch that can reach to method 1 offset + 16MiB.
1329   uint32_t method_idx = 0u;
1330   constexpr size_t kMethodCallLiteralOffset = 2u;
1331   constexpr uint32_t kMissingMethodIdx = 2u;
1332   const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kBlPlus0});
1333   const LinkerPatch method1_patches[] = {
1334       LinkerPatch::RelativeCodePatch(kMethodCallLiteralOffset, nullptr, 2u),
1335   };
1336   ArrayRef<const uint8_t> code1(raw_code1);
1337   ++method_idx;
1338   AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(method1_patches));
1339 
1340   // Skip kMissingMethodIdx.
1341   ++method_idx;
1342   ASSERT_EQ(kMissingMethodIdx, method_idx);
1343   // Add a method with the right size that the method code for the next one starts 1MiB
1344   // after code for method 1.
1345   size_t filler_size =
1346       1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment)
1347              - sizeof(OatQuickMethodHeader);
1348   std::vector<uint8_t> filler_code = GenNops(filler_size / 2u);
1349   ++method_idx;
1350   AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1351   // Add 14 methods with 1MiB code+header, making the code for the next method start 1MiB
1352   // before the currently scheduled MaxNextOffset() for the method call thunk.
1353   for (uint32_t i = 0; i != 14; ++i) {
1354     filler_size = 1 * MB - sizeof(OatQuickMethodHeader);
1355     filler_code = GenNops(filler_size / 2u);
1356     ++method_idx;
1357     AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1358   }
1359 
1360   // Add 2 Baker GC root patches to the last method, one that would allow the thunk at
1361   // 1MiB + kArmAlignment, i.e. kArmAlignment after the method call thunk, and the
1362   // second that needs it kArmAlignment after that. Given the size of the GC root thunk
1363   // is more than the space required by the method call thunk plus kArmAlignment,
1364   // this pushes the first GC root thunk's pending MaxNextOffset() before the method call
1365   // thunk's pending MaxNextOffset() which needs to be adjusted.
1366   ASSERT_LT(RoundUp(CompileMethodCallThunk().size(), kArmAlignment) + kArmAlignment,
1367             CompileBakerGcRootThunk(/* root_reg */ 0, /* narrow */ false).size());
1368   static_assert(kArmAlignment == 8, "Code below assumes kArmAlignment == 8");
1369   constexpr size_t kBakerLiteralOffset1 = kArmAlignment + 2u - kPcAdjustment;
1370   constexpr size_t kBakerLiteralOffset2 = kBakerLiteralOffset1 + kArmAlignment;
1371   // Use offset = 0, base_reg = 0, the LDR is simply `kLdrWInsn | (root_reg << 12)`.
1372   const uint32_t ldr1 = kLdrWInsn | (/* root_reg */ 1 << 12);
1373   const uint32_t ldr2 = kLdrWInsn | (/* root_reg */ 2 << 12);
1374   const std::vector<uint8_t> last_method_raw_code = RawCode({
1375       kNopInsn,                                 // Padding before first GC root read barrier.
1376       ldr1, kBneWPlus0,                         // First GC root LDR with read barrier.
1377       ldr2, kBneWPlus0,                         // Second GC root LDR with read barrier.
1378   });
1379   uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
1380   uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
1381   const LinkerPatch last_method_patches[] = {
1382       LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
1383       LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
1384   };
1385   ++method_idx;
1386   AddCompiledMethod(MethodRef(method_idx),
1387                     ArrayRef<const uint8_t>(last_method_raw_code),
1388                     ArrayRef<const LinkerPatch>(last_method_patches));
1389 
1390   // The main purpose of the test is to check that Link() does not cause a crash.
1391   Link();
1392 
1393   ASSERT_EQ(15 * MB, GetMethodOffset(method_idx) - GetMethodOffset(1u));
1394 }
1395 
1396 }  // namespace linker
1397 }  // namespace art
1398