62 static ElfW(Addr) get_elf_exec_load_bias(const ElfW(Ehdr)* elf);
64 static void get_elf_base_from_phdr(const ElfW(Phdr)* phdr_table, size_t phdr_count,
65                                    ElfW(Addr)* base, ElfW(Addr)* load_bias);
158   ElfW(Ehdr)* ehdr_vdso = reinterpret_cast<ElfW(Ehdr)*>(getauxval(AT_SYSINFO_EHDR));  in add_vdso()
165   si->phdr = reinterpret_cast<ElfW(Phdr)*>(reinterpret_cast<char*>(ehdr_vdso) + ehdr_vdso->e_phoff);  in add_vdso()
167   si->base = reinterpret_cast<ElfW(Addr)>(ehdr_vdso);  in add_vdso()
196   const ElfW(Phdr)* phdr;
198   ElfW(Addr) entry_point;
232   result.phdr = reinterpret_cast<const ElfW(Phdr)*>(getauxval(AT_PHDR));  in get_executable_info()
300 static ElfW(Addr) linker_main(KernelArgumentBlock& args, const char* exe_to_load) {  in linker_main()
398   ElfW(Ehdr)* elf_hdr = reinterpret_cast<ElfW(Ehdr)*>(si->base);  in linker_main()
499   ElfW(Addr) entry = exe_info.entry_point;  in linker_main()
514 static ElfW(Addr) get_elf_exec_load_bias(const ElfW(Ehdr)* elf) {  in get_elf_exec_load_bias()
515   ElfW(Addr) offset = elf->e_phoff;  in get_elf_exec_load_bias()
516   const ElfW(Phdr)* phdr_table =  in get_elf_exec_load_bias()  local
517       reinterpret_cast<const ElfW(Phdr)*>(reinterpret_cast<uintptr_t>(elf) + offset);  in get_elf_exec_load_bias()
518   const ElfW(Phdr)* phdr_end = phdr_table + elf->e_phnum;  in get_elf_exec_load_bias()  local
520   for (const ElfW(Phdr)* phdr = phdr_table; phdr < phdr_end; phdr++) {  in get_elf_exec_load_bias()  local
522       return reinterpret_cast<ElfW(Addr)>(elf) + phdr->p_offset - phdr->p_vaddr;  in get_elf_exec_load_bias()
533 static void get_elf_base_from_phdr(const ElfW(Phdr)* phdr_table, size_t phdr_count,  in get_elf_base_from_phdr()
534                                    ElfW(Addr)* base, ElfW(Addr)* load_bias) {  in get_elf_base_from_phdr()
537       *load_bias = reinterpret_cast<ElfW(Addr)>(phdr_table) - phdr_table[i].p_vaddr;  in get_elf_base_from_phdr()
538       *base = reinterpret_cast<ElfW(Addr)>(phdr_table) - phdr_table[i].p_offset;  in get_elf_base_from_phdr()
558     ElfW(Addr) seg_start = phdr->p_vaddr + si->load_bias;  in set_bss_vma_name()
559     ElfW(Addr) seg_page_end = PAGE_END(seg_start + phdr->p_memsz);  in set_bss_vma_name()
560     ElfW(Addr) seg_file_end = PAGE_END(seg_start + phdr->p_filesz);  in set_bss_vma_name()
575 extern __LIBC_HIDDEN__ ElfW(Rela) __rela_iplt_start[], __rela_iplt_end[];
577 static void call_ifunc_resolvers(ElfW(Addr) load_bias) {  in call_ifunc_resolvers()
578   for (ElfW(Rela) *r = __rela_iplt_start; r != __rela_iplt_end; ++r) {  in call_ifunc_resolvers()
579     ElfW(Addr)* offset = reinterpret_cast<ElfW(Addr)*>(r->r_offset + load_bias);  in call_ifunc_resolvers()
580     ElfW(Addr) resolver = r->r_addend + load_bias;  in call_ifunc_resolvers()
585 extern __LIBC_HIDDEN__ ElfW(Rel) __rel_iplt_start[], __rel_iplt_end[];
587 static void call_ifunc_resolvers(ElfW(Addr) load_bias) {  in call_ifunc_resolvers()
588   for (ElfW(Rel) *r = __rel_iplt_start; r != __rel_iplt_end; ++r) {  in call_ifunc_resolvers()
589     ElfW(Addr)* offset = reinterpret_cast<ElfW(Addr)*>(r->r_offset + load_bias);  in call_ifunc_resolvers()
590     ElfW(Addr) resolver = *offset + load_bias;  in call_ifunc_resolvers()
621 static ElfW(Addr) __attribute__((noinline))
633 extern "C" ElfW(Addr) __linker_init(void* raw_args) {  in __linker_init()
642   ElfW(Addr) linker_addr = getauxval(AT_BASE);  in __linker_init()
646     ElfW(Addr) load_bias;  in __linker_init()
648       reinterpret_cast<ElfW(Phdr)*>(getauxval(AT_PHDR)), getauxval(AT_PHNUM),  in __linker_init()
652   ElfW(Ehdr)* elf_hdr = reinterpret_cast<ElfW(Ehdr)*>(linker_addr);  in __linker_init()
653   ElfW(Phdr)* phdr = reinterpret_cast<ElfW(Phdr)*>(linker_addr + elf_hdr->e_phoff);  in __linker_init()
656   const ElfW(Addr) load_bias = get_elf_exec_load_bias(elf_hdr);  in __linker_init()  local
682 static ElfW(Addr) __attribute__((noinline))
743   ElfW(Addr) start_address = linker_main(args, exe_to_load);  in __linker_init_post_relocation()