/* * Based on arch/arm/include/asm/uaccess.h * * Copyright (C) 2012 ARM Ltd. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ #ifndef __ASM_UACCESS_H #define __ASM_UACCESS_H #include <asm/alternative.h> #include <asm/kernel-pgtable.h> #include <asm/sysreg.h> /* * User space memory access functions */ #include <linux/bitops.h> #include <linux/kasan-checks.h> #include <linux/string.h> #include <asm/cpufeature.h> #include <asm/ptrace.h> #include <asm/memory.h> #include <asm/compiler.h> #include <asm/extable.h> #define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) static inline void set_fs(mm_segment_t fs) { current_thread_info()->addr_limit = fs; /* * Prevent a mispredicted conditional call to set_fs from forwarding * the wrong address limit to access_ok under speculation. */ dsb(nsh); isb(); /* On user-mode return, check fs is correct */ set_thread_flag(TIF_FSCHECK); /* * Enable/disable UAO so that copy_to_user() etc can access * kernel memory with the unprivileged instructions. */ if (IS_ENABLED(CONFIG_ARM64_UAO) && fs == KERNEL_DS) asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO)); else asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO, CONFIG_ARM64_UAO)); } #define segment_eq(a, b) ((a) == (b)) /* * Test whether a block of memory is a valid user space address. * Returns 1 if the range is valid, 0 otherwise. * * This is equivalent to the following test: * (u65)addr + (u65)size <= (u65)current->addr_limit + 1 */ static inline unsigned long __range_ok(unsigned long addr, unsigned long size) { unsigned long limit = current_thread_info()->addr_limit; __chk_user_ptr(addr); asm volatile( // A + B <= C + 1 for all A,B,C, in four easy steps: // 1: X = A + B; X' = X % 2^64 " adds %0, %0, %2\n" // 2: Set C = 0 if X > 2^64, to guarantee X' > C in step 4 " csel %1, xzr, %1, hi\n" // 3: Set X' = ~0 if X >= 2^64. For X == 2^64, this decrements X' // to compensate for the carry flag being set in step 4. For // X > 2^64, X' merely has to remain nonzero, which it does. " csinv %0, %0, xzr, cc\n" // 4: For X < 2^64, this gives us X' - C - 1 <= 0, where the -1 // comes from the carry in being clear. Otherwise, we are // testing X' - C == 0, subject to the previous adjustments. " sbcs xzr, %0, %1\n" " cset %0, ls\n" : "+r" (addr), "+r" (limit) : "Ir" (size) : "cc"); return addr; } /* * When dealing with data aborts, watchpoints, or instruction traps we may end * up with a tagged userland pointer. Clear the tag to get a sane pointer to * pass on to access_ok(), for instance. */ #define untagged_addr(addr) sign_extend64(addr, 55) #define access_ok(type, addr, size) __range_ok((unsigned long)(addr), size) #define user_addr_max get_fs #define _ASM_EXTABLE(from, to) \ " .pushsection __ex_table, \"a\"\n" \ " .align 3\n" \ " .long (" #from " - .), (" #to " - .)\n" \ " .popsection\n" /* * User access enabling/disabling. */ #ifdef CONFIG_ARM64_SW_TTBR0_PAN static inline void __uaccess_ttbr0_disable(void) { unsigned long flags, ttbr; local_irq_save(flags); ttbr = read_sysreg(ttbr1_el1); ttbr &= ~TTBR_ASID_MASK; /* reserved_ttbr0 placed at the end of swapper_pg_dir */ write_sysreg(ttbr + SWAPPER_DIR_SIZE, ttbr0_el1); isb(); /* Set reserved ASID */ write_sysreg(ttbr, ttbr1_el1); isb(); local_irq_restore(flags); } static inline void __uaccess_ttbr0_enable(void) { unsigned long flags, ttbr0, ttbr1; /* * Disable interrupts to avoid preemption between reading the 'ttbr0' * variable and the MSR. A context switch could trigger an ASID * roll-over and an update of 'ttbr0'. */ local_irq_save(flags); ttbr0 = READ_ONCE(current_thread_info()->ttbr0); /* Restore active ASID */ ttbr1 = read_sysreg(ttbr1_el1); ttbr1 &= ~TTBR_ASID_MASK; /* safety measure */ ttbr1 |= ttbr0 & TTBR_ASID_MASK; write_sysreg(ttbr1, ttbr1_el1); isb(); /* Restore user page table */ write_sysreg(ttbr0, ttbr0_el1); isb(); local_irq_restore(flags); } static inline bool uaccess_ttbr0_disable(void) { if (!system_uses_ttbr0_pan()) return false; __uaccess_ttbr0_disable(); return true; } static inline bool uaccess_ttbr0_enable(void) { if (!system_uses_ttbr0_pan()) return false; __uaccess_ttbr0_enable(); return true; } #else static inline bool uaccess_ttbr0_disable(void) { return false; } static inline bool uaccess_ttbr0_enable(void) { return false; } #endif static inline void __uaccess_disable_hw_pan(void) { asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN)); } static inline void __uaccess_enable_hw_pan(void) { asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN)); } #define __uaccess_disable(alt) \ do { \ if (!uaccess_ttbr0_disable()) \ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt, \ CONFIG_ARM64_PAN)); \ } while (0) #define __uaccess_enable(alt) \ do { \ if (!uaccess_ttbr0_enable()) \ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt, \ CONFIG_ARM64_PAN)); \ } while (0) static inline void uaccess_disable(void) { __uaccess_disable(ARM64_HAS_PAN); } static inline void uaccess_enable(void) { __uaccess_enable(ARM64_HAS_PAN); } /* * These functions are no-ops when UAO is present. */ static inline void uaccess_disable_not_uao(void) { __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); } static inline void uaccess_enable_not_uao(void) { __uaccess_enable(ARM64_ALT_PAN_NOT_UAO); } /* * Sanitise a uaccess pointer such that it becomes NULL if above the * current addr_limit. */ #define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr) static inline void __user *__uaccess_mask_ptr(const void __user *ptr) { void __user *safe_ptr; asm volatile( " bics xzr, %1, %2\n" " csel %0, %1, xzr, eq\n" : "=&r" (safe_ptr) : "r" (ptr), "r" (current_thread_info()->addr_limit) : "cc"); csdb(); return safe_ptr; } /* * The "__xxx" versions of the user access functions do not verify the address * space - it must have been done previously with a separate "access_ok()" * call. * * The "__xxx_error" versions set the third argument to -EFAULT if an error * occurs, and leave it unchanged on success. */ #define __get_user_asm(instr, alt_instr, reg, x, addr, err, feature) \ asm volatile( \ "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \ alt_instr " " reg "1, [%2]\n", feature) \ "2:\n" \ " .section .fixup, \"ax\"\n" \ " .align 2\n" \ "3: mov %w0, %3\n" \ " mov %1, #0\n" \ " b 2b\n" \ " .previous\n" \ _ASM_EXTABLE(1b, 3b) \ : "+r" (err), "=&r" (x) \ : "r" (addr), "i" (-EFAULT)) #define __get_user_err(x, ptr, err) \ do { \ unsigned long __gu_val; \ __chk_user_ptr(ptr); \ uaccess_enable_not_uao(); \ switch (sizeof(*(ptr))) { \ case 1: \ __get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr), \ (err), ARM64_HAS_UAO); \ break; \ case 2: \ __get_user_asm("ldrh", "ldtrh", "%w", __gu_val, (ptr), \ (err), ARM64_HAS_UAO); \ break; \ case 4: \ __get_user_asm("ldr", "ldtr", "%w", __gu_val, (ptr), \ (err), ARM64_HAS_UAO); \ break; \ case 8: \ __get_user_asm("ldr", "ldtr", "%x", __gu_val, (ptr), \ (err), ARM64_HAS_UAO); \ break; \ default: \ BUILD_BUG(); \ } \ uaccess_disable_not_uao(); \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ } while (0) #define __get_user_check(x, ptr, err) \ ({ \ __typeof__(*(ptr)) __user *__p = (ptr); \ might_fault(); \ if (access_ok(VERIFY_READ, __p, sizeof(*__p))) { \ __p = uaccess_mask_ptr(__p); \ __get_user_err((x), __p, (err)); \ } else { \ (x) = 0; (err) = -EFAULT; \ } \ }) #define __get_user_error(x, ptr, err) \ ({ \ __get_user_check((x), (ptr), (err)); \ (void)0; \ }) #define __get_user(x, ptr) \ ({ \ int __gu_err = 0; \ __get_user_check((x), (ptr), __gu_err); \ __gu_err; \ }) #define get_user __get_user #define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature) \ asm volatile( \ "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \ alt_instr " " reg "1, [%2]\n", feature) \ "2:\n" \ " .section .fixup,\"ax\"\n" \ " .align 2\n" \ "3: mov %w0, %3\n" \ " b 2b\n" \ " .previous\n" \ _ASM_EXTABLE(1b, 3b) \ : "+r" (err) \ : "r" (x), "r" (addr), "i" (-EFAULT)) #define __put_user_err(x, ptr, err) \ do { \ __typeof__(*(ptr)) __pu_val = (x); \ __chk_user_ptr(ptr); \ uaccess_enable_not_uao(); \ switch (sizeof(*(ptr))) { \ case 1: \ __put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr), \ (err), ARM64_HAS_UAO); \ break; \ case 2: \ __put_user_asm("strh", "sttrh", "%w", __pu_val, (ptr), \ (err), ARM64_HAS_UAO); \ break; \ case 4: \ __put_user_asm("str", "sttr", "%w", __pu_val, (ptr), \ (err), ARM64_HAS_UAO); \ break; \ case 8: \ __put_user_asm("str", "sttr", "%x", __pu_val, (ptr), \ (err), ARM64_HAS_UAO); \ break; \ default: \ BUILD_BUG(); \ } \ uaccess_disable_not_uao(); \ } while (0) #define __put_user_check(x, ptr, err) \ ({ \ __typeof__(*(ptr)) __user *__p = (ptr); \ might_fault(); \ if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) { \ __p = uaccess_mask_ptr(__p); \ __put_user_err((x), __p, (err)); \ } else { \ (err) = -EFAULT; \ } \ }) #define __put_user_error(x, ptr, err) \ ({ \ __put_user_check((x), (ptr), (err)); \ (void)0; \ }) #define __put_user(x, ptr) \ ({ \ int __pu_err = 0; \ __put_user_check((x), (ptr), __pu_err); \ __pu_err; \ }) #define put_user __put_user extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); #define raw_copy_from_user(to, from, n) \ ({ \ __arch_copy_from_user((to), __uaccess_mask_ptr(from), (n)); \ }) extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n); #define raw_copy_to_user(to, from, n) \ ({ \ __arch_copy_to_user(__uaccess_mask_ptr(to), (from), (n)); \ }) extern unsigned long __must_check __arch_copy_in_user(void __user *to, const void __user *from, unsigned long n); #define raw_copy_in_user(to, from, n) \ ({ \ __arch_copy_in_user(__uaccess_mask_ptr(to), \ __uaccess_mask_ptr(from), (n)); \ }) #define INLINE_COPY_TO_USER #define INLINE_COPY_FROM_USER extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n); static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n) { if (access_ok(VERIFY_WRITE, to, n)) n = __arch_clear_user(__uaccess_mask_ptr(to), n); return n; } #define clear_user __clear_user extern long strncpy_from_user(char *dest, const char __user *src, long count); extern __must_check long strnlen_user(const char __user *str, long n); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE struct page; void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len); extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n); static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) { kasan_check_write(dst, size); return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size); } #endif #endif /* __ASM_UACCESS_H */
Name | Type | Size | Permission | Actions |
---|---|---|---|---|
xen | Folder | 0755 |
|
|
Kbuild | File | 703 B | 0644 |
|
acenv.h | File | 541 B | 0644 |
|
acpi.h | File | 4.34 KB | 0644 |
|
alternative.h | File | 7.63 KB | 0644 |
|
arch_gicv3.h | File | 3.44 KB | 0644 |
|
arch_timer.h | File | 4.87 KB | 0644 |
|
arm-cci.h | File | 794 B | 0644 |
|
asm-bug.h | File | 1.45 KB | 0644 |
|
asm-offsets.h | File | 35 B | 0644 |
|
asm-uaccess.h | File | 2.09 KB | 0644 |
|
assembler.h | File | 13.51 KB | 0644 |
|
atomic.h | File | 8.35 KB | 0644 |
|
atomic_ll_sc.h | File | 10.62 KB | 0644 |
|
atomic_lse.h | File | 14.82 KB | 0644 |
|
barrier.h | File | 3.78 KB | 0644 |
|
bitops.h | File | 1.9 KB | 0644 |
|
bitrev.h | File | 452 B | 0644 |
|
boot.h | File | 384 B | 0644 |
|
brk-imm.h | File | 706 B | 0644 |
|
bug.h | File | 1.09 KB | 0644 |
|
cache.h | File | 2.23 KB | 0644 |
|
cacheflush.h | File | 4.87 KB | 0644 |
|
checksum.h | File | 1.35 KB | 0644 |
|
clocksource.h | File | 192 B | 0644 |
|
cmpxchg.h | File | 7.98 KB | 0644 |
|
compat.h | File | 7.15 KB | 0644 |
|
compiler.h | File | 1.18 KB | 0644 |
|
cpu.h | File | 1.84 KB | 0644 |
|
cpu_ops.h | File | 2.73 KB | 0644 |
|
cpucaps.h | File | 1.83 KB | 0644 |
|
cpufeature.h | File | 19.14 KB | 0644 |
|
cpuidle.h | File | 401 B | 0644 |
|
cputype.h | File | 8.1 KB | 0644 |
|
current.h | File | 517 B | 0644 |
|
daifflags.h | File | 1.59 KB | 0644 |
|
dcc.h | File | 1.36 KB | 0644 |
|
debug-monitors.h | File | 3.76 KB | 0644 |
|
device.h | File | 886 B | 0644 |
|
dma-mapping.h | File | 2.42 KB | 0644 |
|
dmi.h | File | 850 B | 0644 |
|
efi.h | File | 4.57 KB | 0644 |
|
elf.h | File | 5.7 KB | 0644 |
|
esr.h | File | 9.02 KB | 0644 |
|
exception.h | File | 1.21 KB | 0644 |
|
exec.h | File | 868 B | 0644 |
|
extable.h | File | 815 B | 0644 |
|
fb.h | File | 1000 B | 0644 |
|
fixmap.h | File | 2.91 KB | 0644 |
|
fpsimd.h | File | 4.21 KB | 0644 |
|
fpsimdmacros.h | File | 5.62 KB | 0644 |
|
ftrace.h | File | 1.92 KB | 0644 |
|
futex.h | File | 3.41 KB | 0644 |
|
hardirq.h | File | 2.08 KB | 0644 |
|
hugetlb.h | File | 2.71 KB | 0644 |
|
hw_breakpoint.h | File | 4.46 KB | 0644 |
|
hwcap.h | File | 1.86 KB | 0644 |
|
hypervisor.h | File | 144 B | 0644 |
|
insn.h | File | 16.03 KB | 0644 |
|
io.h | File | 7.72 KB | 0644 |
|
irq.h | File | 307 B | 0644 |
|
irq_work.h | File | 228 B | 0644 |
|
irqflags.h | File | 2.3 KB | 0644 |
|
jump_label.h | File | 1.68 KB | 0644 |
|
kasan.h | File | 1.16 KB | 0644 |
|
kernel-pgtable.h | File | 4.03 KB | 0644 |
|
kexec.h | File | 2.42 KB | 0644 |
|
kgdb.h | File | 3.79 KB | 0644 |
|
kprobes.h | File | 1.74 KB | 0644 |
|
kvm_arm.h | File | 8.38 KB | 0644 |
|
kvm_asm.h | File | 4.26 KB | 0644 |
|
kvm_coproc.h | File | 2.04 KB | 0644 |
|
kvm_emulate.h | File | 10.38 KB | 0644 |
|
kvm_host.h | File | 15.73 KB | 0644 |
|
kvm_hyp.h | File | 5.79 KB | 0644 |
|
kvm_mmio.h | File | 1.3 KB | 0644 |
|
kvm_mmu.h | File | 11.72 KB | 0644 |
|
linkage.h | File | 114 B | 0644 |
|
lse.h | File | 1.26 KB | 0644 |
|
memblock.h | File | 720 B | 0644 |
|
memory.h | File | 9.32 KB | 0644 |
|
mmu.h | File | 2.74 KB | 0644 |
|
mmu_context.h | File | 6.35 KB | 0644 |
|
mmzone.h | File | 266 B | 0644 |
|
module.h | File | 2.8 KB | 0644 |
|
neon.h | File | 815 B | 0644 |
|
numa.h | File | 1.33 KB | 0644 |
|
page-def.h | File | 1.17 KB | 0644 |
|
page.h | File | 1.61 KB | 0644 |
|
paravirt.h | File | 458 B | 0644 |
|
pci.h | File | 878 B | 0644 |
|
percpu.h | File | 7.48 KB | 0644 |
|
perf_event.h | File | 3.17 KB | 0644 |
|
pgalloc.h | File | 3.71 KB | 0644 |
|
pgtable-hwdef.h | File | 9.4 KB | 0644 |
|
pgtable-prot.h | File | 4.38 KB | 0644 |
|
pgtable-types.h | File | 1.84 KB | 0644 |
|
pgtable.h | File | 21.55 KB | 0644 |
|
probes.h | File | 1022 B | 0644 |
|
proc-fns.h | File | 1.21 KB | 0644 |
|
processor.h | File | 6.52 KB | 0644 |
|
ptdump.h | File | 1.42 KB | 0644 |
|
ptrace.h | File | 9 KB | 0644 |
|
sdei.h | File | 1.46 KB | 0644 |
|
seccomp.h | File | 714 B | 0644 |
|
sections.h | File | 1.46 KB | 0644 |
|
shmparam.h | File | 965 B | 0644 |
|
signal32.h | File | 1.45 KB | 0644 |
|
simd.h | File | 1.39 KB | 0644 |
|
smp.h | File | 4.23 KB | 0644 |
|
smp_plat.h | File | 1.43 KB | 0644 |
|
sparsemem.h | File | 771 B | 0644 |
|
spinlock.h | File | 3.33 KB | 0644 |
|
spinlock_types.h | File | 1.06 KB | 0644 |
|
stack_pointer.h | File | 247 B | 0644 |
|
stackprotector.h | File | 1.11 KB | 0644 |
|
stacktrace.h | File | 2.53 KB | 0644 |
|
stage2_pgtable-nopmd.h | File | 1.3 KB | 0644 |
|
stage2_pgtable-nopud.h | File | 1.24 KB | 0644 |
|
stage2_pgtable.h | File | 4.89 KB | 0644 |
|
stat.h | File | 1.43 KB | 0644 |
|
string.h | File | 2.33 KB | 0644 |
|
suspend.h | File | 1.65 KB | 0644 |
|
sync_bitops.h | File | 1.11 KB | 0644 |
|
syscall.h | File | 2.87 KB | 0644 |
|
sysreg.h | File | 25.1 KB | 0644 |
|
system_misc.h | File | 1.86 KB | 0644 |
|
thread_info.h | File | 3.93 KB | 0644 |
|
timex.h | File | 883 B | 0644 |
|
tlb.h | File | 2.22 KB | 0644 |
|
tlbflush.h | File | 5.38 KB | 0644 |
|
topology.h | File | 1.29 KB | 0644 |
|
traps.h | File | 3.33 KB | 0644 |
|
uaccess.h | File | 12.01 KB | 0644 |
|
unistd.h | File | 1.6 KB | 0644 |
|
unistd32.h | File | 27.53 KB | 0644 |
|
uprobes.h | File | 777 B | 0644 |
|
vdso.h | File | 1.09 KB | 0644 |
|
vdso_datapage.h | File | 1.53 KB | 0644 |
|
vectors.h | File | 1.75 KB | 0644 |
|
virt.h | File | 3 KB | 0644 |
|
vmap_stack.h | File | 769 B | 0644 |
|
word-at-a-time.h | File | 2.22 KB | 0644 |
|