404

[ Avaa Bypassed ]




Upload:

Command:

botdev@18.191.197.56: ~ $
/*
 * Copyright (C) 2012,2013 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef __ARM64_KVM_MMU_H__
#define __ARM64_KVM_MMU_H__

#include <asm/page.h>
#include <asm/memory.h>
#include <asm/cpufeature.h>

/*
 * As ARMv8.0 only has the TTBR0_EL2 register, we cannot express
 * "negative" addresses. This makes it impossible to directly share
 * mappings with the kernel.
 *
 * Instead, give the HYP mode its own VA region at a fixed offset from
 * the kernel by just masking the top bits (which are all ones for a
 * kernel address). We need to find out how many bits to mask.
 *
 * We want to build a set of page tables that cover both parts of the
 * idmap (the trampoline page used to initialize EL2), and our normal
 * runtime VA space, at the same time.
 *
 * Given that the kernel uses VA_BITS for its entire address space,
 * and that half of that space (VA_BITS - 1) is used for the linear
 * mapping, we can also limit the EL2 space to (VA_BITS - 1).
 *
 * The main question is "Within the VA_BITS space, does EL2 use the
 * top or the bottom half of that space to shadow the kernel's linear
 * mapping?". As we need to idmap the trampoline page, this is
 * determined by the range in which this page lives.
 *
 * If the page is in the bottom half, we have to use the top half. If
 * the page is in the top half, we have to use the bottom half:
 *
 * T = __pa_symbol(__hyp_idmap_text_start)
 * if (T & BIT(VA_BITS - 1))
 *	HYP_VA_MIN = 0  //idmap in upper half
 * else
 *	HYP_VA_MIN = 1 << (VA_BITS - 1)
 * HYP_VA_MAX = HYP_VA_MIN + (1 << (VA_BITS - 1)) - 1
 *
 * This of course assumes that the trampoline page exists within the
 * VA_BITS range. If it doesn't, then it means we're in the odd case
 * where the kernel idmap (as well as HYP) uses more levels than the
 * kernel runtime page tables (as seen when the kernel is configured
 * for 4k pages, 39bits VA, and yet memory lives just above that
 * limit, forcing the idmap to use 4 levels of page tables while the
 * kernel itself only uses 3). In this particular case, it doesn't
 * matter which side of VA_BITS we use, as we're guaranteed not to
 * conflict with anything.
 *
 * When using VHE, there are no separate hyp mappings and all KVM
 * functionality is already mapped as part of the main kernel
 * mappings, and none of this applies in that case.
 */

#define HYP_PAGE_OFFSET_HIGH_MASK	((UL(1) << VA_BITS) - 1)
#define HYP_PAGE_OFFSET_LOW_MASK	((UL(1) << (VA_BITS - 1)) - 1)

#ifdef __ASSEMBLY__

#include <asm/alternative.h>
#include <asm/cpufeature.h>

/*
 * Convert a kernel VA into a HYP VA.
 * reg: VA to be converted.
 *
 * This generates the following sequences:
 * - High mask:
 *		and x0, x0, #HYP_PAGE_OFFSET_HIGH_MASK
 *		nop
 * - Low mask:
 *		and x0, x0, #HYP_PAGE_OFFSET_HIGH_MASK
 *		and x0, x0, #HYP_PAGE_OFFSET_LOW_MASK
 * - VHE:
 *		nop
 *		nop
 *
 * The "low mask" version works because the mask is a strict subset of
 * the "high mask", hence performing the first mask for nothing.
 * Should be completely invisible on any viable CPU.
 */
.macro kern_hyp_va	reg
alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
	and     \reg, \reg, #HYP_PAGE_OFFSET_HIGH_MASK
alternative_else_nop_endif
alternative_if ARM64_HYP_OFFSET_LOW
	and     \reg, \reg, #HYP_PAGE_OFFSET_LOW_MASK
alternative_else_nop_endif
.endm

#else

#include <asm/pgalloc.h>
#include <asm/cache.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>

static inline unsigned long __kern_hyp_va(unsigned long v)
{
	asm volatile(ALTERNATIVE("and %0, %0, %1",
				 "nop",
				 ARM64_HAS_VIRT_HOST_EXTN)
		     : "+r" (v)
		     : "i" (HYP_PAGE_OFFSET_HIGH_MASK));
	asm volatile(ALTERNATIVE("nop",
				 "and %0, %0, %1",
				 ARM64_HYP_OFFSET_LOW)
		     : "+r" (v)
		     : "i" (HYP_PAGE_OFFSET_LOW_MASK));
	return v;
}

#define kern_hyp_va(v) 	((typeof(v))(__kern_hyp_va((unsigned long)(v))))

/*
 * Obtain the PC-relative address of a kernel symbol
 * s: symbol
 *
 * The goal of this macro is to return a symbol's address based on a
 * PC-relative computation, as opposed to a loading the VA from a
 * constant pool or something similar. This works well for HYP, as an
 * absolute VA is guaranteed to be wrong. Only use this if trying to
 * obtain the address of a symbol (i.e. not something you obtained by
 * following a pointer).
 */
#define hyp_symbol_addr(s)						\
	({								\
		typeof(s) *addr;					\
		asm("adrp	%0, %1\n"				\
		    "add	%0, %0, :lo12:%1\n"			\
		    : "=r" (addr) : "S" (&s));				\
		addr;							\
	})

/*
 * We currently only support a 40bit IPA.
 */
#define KVM_PHYS_SHIFT	(40)
#define KVM_PHYS_SIZE	(1UL << KVM_PHYS_SHIFT)
#define KVM_PHYS_MASK	(KVM_PHYS_SIZE - 1UL)

#include <asm/stage2_pgtable.h>

int create_hyp_mappings(void *from, void *to, pgprot_t prot);
int create_hyp_io_mappings(void *from, void *to, phys_addr_t);
void free_hyp_pgds(void);

void stage2_unmap_vm(struct kvm *kvm);
int kvm_alloc_stage2_pgd(struct kvm *kvm);
void kvm_free_stage2_pgd(struct kvm *kvm);
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
			  phys_addr_t pa, unsigned long size, bool writable);

int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);

void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);

phys_addr_t kvm_mmu_get_httbr(void);
phys_addr_t kvm_get_idmap_vector(void);
int kvm_mmu_init(void);
void kvm_clear_hyp_idmap(void);

#define	kvm_set_pte(ptep, pte)		set_pte(ptep, pte)
#define	kvm_set_pmd(pmdp, pmd)		set_pmd(pmdp, pmd)

static inline pte_t kvm_s2pte_mkwrite(pte_t pte)
{
	pte_val(pte) |= PTE_S2_RDWR;
	return pte;
}

static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd)
{
	pmd_val(pmd) |= PMD_S2_RDWR;
	return pmd;
}

static inline void kvm_set_s2pte_readonly(pte_t *pte)
{
	pteval_t old_pteval, pteval;

	pteval = READ_ONCE(pte_val(*pte));
	do {
		old_pteval = pteval;
		pteval &= ~PTE_S2_RDWR;
		pteval |= PTE_S2_RDONLY;
		pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval);
	} while (pteval != old_pteval);
}

static inline bool kvm_s2pte_readonly(pte_t *pte)
{
	return (pte_val(*pte) & PTE_S2_RDWR) == PTE_S2_RDONLY;
}

static inline void kvm_set_s2pmd_readonly(pmd_t *pmd)
{
	kvm_set_s2pte_readonly((pte_t *)pmd);
}

static inline bool kvm_s2pmd_readonly(pmd_t *pmd)
{
	return kvm_s2pte_readonly((pte_t *)pmd);
}

static inline bool kvm_page_empty(void *ptr)
{
	struct page *ptr_page = virt_to_page(ptr);
	return page_count(ptr_page) == 1;
}

#define hyp_pte_table_empty(ptep) kvm_page_empty(ptep)

#ifdef __PAGETABLE_PMD_FOLDED
#define hyp_pmd_table_empty(pmdp) (0)
#else
#define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp)
#endif

#ifdef __PAGETABLE_PUD_FOLDED
#define hyp_pud_table_empty(pudp) (0)
#else
#define hyp_pud_table_empty(pudp) kvm_page_empty(pudp)
#endif

struct kvm;

#define kvm_flush_dcache_to_poc(a,l)	__flush_dcache_area((a), (l))

static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
{
	return (vcpu_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101;
}

static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
					       kvm_pfn_t pfn,
					       unsigned long size)
{
	void *va = page_address(pfn_to_page(pfn));

	kvm_flush_dcache_to_poc(va, size);

	if (icache_is_aliasing()) {
		/* any kind of VIPT cache */
		__flush_icache_all();
	} else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
		/* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
		flush_icache_range((unsigned long)va,
				   (unsigned long)va + size);
	}
}

static inline void __kvm_flush_dcache_pte(pte_t pte)
{
	struct page *page = pte_page(pte);
	kvm_flush_dcache_to_poc(page_address(page), PAGE_SIZE);
}

static inline void __kvm_flush_dcache_pmd(pmd_t pmd)
{
	struct page *page = pmd_page(pmd);
	kvm_flush_dcache_to_poc(page_address(page), PMD_SIZE);
}

static inline void __kvm_flush_dcache_pud(pud_t pud)
{
	struct page *page = pud_page(pud);
	kvm_flush_dcache_to_poc(page_address(page), PUD_SIZE);
}

#define kvm_virt_to_phys(x)		__pa_symbol(x)

void kvm_set_way_flush(struct kvm_vcpu *vcpu);
void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);

static inline bool __kvm_cpu_uses_extended_idmap(void)
{
	return __cpu_uses_extended_idmap();
}

/*
 * Can't use pgd_populate here, because the extended idmap adds an extra level
 * above CONFIG_PGTABLE_LEVELS (which is 2 or 3 if we're using the extended
 * idmap), and pgd_populate is only available if CONFIG_PGTABLE_LEVELS = 4.
 */
static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
				       pgd_t *hyp_pgd,
				       pgd_t *merged_hyp_pgd,
				       unsigned long hyp_idmap_start)
{
	int idmap_idx;

	/*
	 * Use the first entry to access the HYP mappings. It is
	 * guaranteed to be free, otherwise we wouldn't use an
	 * extended idmap.
	 */
	VM_BUG_ON(pgd_val(merged_hyp_pgd[0]));
	merged_hyp_pgd[0] = __pgd(__pa(hyp_pgd) | PMD_TYPE_TABLE);

	/*
	 * Create another extended level entry that points to the boot HYP map,
	 * which contains an ID mapping of the HYP init code. We essentially
	 * merge the boot and runtime HYP maps by doing so, but they don't
	 * overlap anyway, so this is fine.
	 */
	idmap_idx = hyp_idmap_start >> VA_BITS;
	VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx]));
	merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) | PMD_TYPE_TABLE);
}

static inline unsigned int kvm_get_vmid_bits(void)
{
	int reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);

	return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
}

/*
 * We are not in the kvm->srcu critical section most of the time, so we take
 * the SRCU read lock here. Since we copy the data from the user page, we
 * can immediately drop the lock again.
 */
static inline int kvm_read_guest_lock(struct kvm *kvm,
				      gpa_t gpa, void *data, unsigned long len)
{
	int srcu_idx = srcu_read_lock(&kvm->srcu);
	int ret = kvm_read_guest(kvm, gpa, data, len);

	srcu_read_unlock(&kvm->srcu, srcu_idx);

	return ret;
}

static inline int kvm_write_guest_lock(struct kvm *kvm, gpa_t gpa,
				       const void *data, unsigned long len)
{
	int srcu_idx = srcu_read_lock(&kvm->srcu);
	int ret = kvm_write_guest(kvm, gpa, data, len);

	srcu_read_unlock(&kvm->srcu, srcu_idx);

	return ret;
}

#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
#include <asm/mmu.h>

static inline void *kvm_get_hyp_vector(void)
{
	struct bp_hardening_data *data = arm64_get_bp_hardening_data();
	void *vect = kvm_ksym_ref(__kvm_hyp_vector);

	if (data->template_start) {
		vect = __bp_harden_hyp_vecs_start +
		       data->hyp_vectors_slot * SZ_2K;

		if (!has_vhe())
			vect = lm_alias(vect);
	}

	return vect;
}

static inline int kvm_map_vectors(void)
{
	return create_hyp_mappings(kvm_ksym_ref(__bp_harden_hyp_vecs_start),
				   kvm_ksym_ref(__bp_harden_hyp_vecs_end),
				   PAGE_HYP_EXEC);
}

#else
static inline void *kvm_get_hyp_vector(void)
{
	return kvm_ksym_ref(__kvm_hyp_vector);
}

static inline int kvm_map_vectors(void)
{
	return 0;
}
#endif

#ifdef CONFIG_ARM64_SSBD
DECLARE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);

static inline int hyp_map_aux_data(void)
{
	int cpu, err;

	for_each_possible_cpu(cpu) {
		u64 *ptr;

		ptr = per_cpu_ptr(&arm64_ssbd_callback_required, cpu);
		err = create_hyp_mappings(ptr, ptr + 1, PAGE_HYP);
		if (err)
			return err;
	}
	return 0;
}
#else
static inline int hyp_map_aux_data(void)
{
	return 0;
}
#endif

#endif /* __ASSEMBLY__ */
#endif /* __ARM64_KVM_MMU_H__ */

Filemanager

Name Type Size Permission Actions
xen Folder 0755
Kbuild File 703 B 0644
acenv.h File 541 B 0644
acpi.h File 4.34 KB 0644
alternative.h File 7.63 KB 0644
arch_gicv3.h File 3.44 KB 0644
arch_timer.h File 4.87 KB 0644
arm-cci.h File 794 B 0644
asm-bug.h File 1.45 KB 0644
asm-offsets.h File 35 B 0644
asm-uaccess.h File 2.09 KB 0644
assembler.h File 13.51 KB 0644
atomic.h File 8.35 KB 0644
atomic_ll_sc.h File 10.62 KB 0644
atomic_lse.h File 14.82 KB 0644
barrier.h File 3.78 KB 0644
bitops.h File 1.9 KB 0644
bitrev.h File 452 B 0644
boot.h File 384 B 0644
brk-imm.h File 706 B 0644
bug.h File 1.09 KB 0644
cache.h File 2.23 KB 0644
cacheflush.h File 4.87 KB 0644
checksum.h File 1.35 KB 0644
clocksource.h File 192 B 0644
cmpxchg.h File 7.98 KB 0644
compat.h File 7.15 KB 0644
compiler.h File 1.18 KB 0644
cpu.h File 1.84 KB 0644
cpu_ops.h File 2.73 KB 0644
cpucaps.h File 1.83 KB 0644
cpufeature.h File 19.14 KB 0644
cpuidle.h File 401 B 0644
cputype.h File 8.1 KB 0644
current.h File 517 B 0644
daifflags.h File 1.59 KB 0644
dcc.h File 1.36 KB 0644
debug-monitors.h File 3.76 KB 0644
device.h File 886 B 0644
dma-mapping.h File 2.42 KB 0644
dmi.h File 850 B 0644
efi.h File 4.57 KB 0644
elf.h File 5.7 KB 0644
esr.h File 9.02 KB 0644
exception.h File 1.21 KB 0644
exec.h File 868 B 0644
extable.h File 815 B 0644
fb.h File 1000 B 0644
fixmap.h File 2.91 KB 0644
fpsimd.h File 4.21 KB 0644
fpsimdmacros.h File 5.62 KB 0644
ftrace.h File 1.92 KB 0644
futex.h File 3.41 KB 0644
hardirq.h File 2.08 KB 0644
hugetlb.h File 2.71 KB 0644
hw_breakpoint.h File 4.46 KB 0644
hwcap.h File 1.86 KB 0644
hypervisor.h File 144 B 0644
insn.h File 16.03 KB 0644
io.h File 7.72 KB 0644
irq.h File 307 B 0644
irq_work.h File 228 B 0644
irqflags.h File 2.3 KB 0644
jump_label.h File 1.68 KB 0644
kasan.h File 1.16 KB 0644
kernel-pgtable.h File 4.03 KB 0644
kexec.h File 2.42 KB 0644
kgdb.h File 3.79 KB 0644
kprobes.h File 1.74 KB 0644
kvm_arm.h File 8.38 KB 0644
kvm_asm.h File 4.26 KB 0644
kvm_coproc.h File 2.04 KB 0644
kvm_emulate.h File 10.38 KB 0644
kvm_host.h File 15.73 KB 0644
kvm_hyp.h File 5.79 KB 0644
kvm_mmio.h File 1.3 KB 0644
kvm_mmu.h File 11.72 KB 0644
linkage.h File 114 B 0644
lse.h File 1.26 KB 0644
memblock.h File 720 B 0644
memory.h File 9.32 KB 0644
mmu.h File 2.74 KB 0644
mmu_context.h File 6.35 KB 0644
mmzone.h File 266 B 0644
module.h File 2.8 KB 0644
neon.h File 815 B 0644
numa.h File 1.33 KB 0644
page-def.h File 1.17 KB 0644
page.h File 1.61 KB 0644
paravirt.h File 458 B 0644
pci.h File 878 B 0644
percpu.h File 7.48 KB 0644
perf_event.h File 3.17 KB 0644
pgalloc.h File 3.71 KB 0644
pgtable-hwdef.h File 9.4 KB 0644
pgtable-prot.h File 4.38 KB 0644
pgtable-types.h File 1.84 KB 0644
pgtable.h File 21.55 KB 0644
probes.h File 1022 B 0644
proc-fns.h File 1.21 KB 0644
processor.h File 6.52 KB 0644
ptdump.h File 1.42 KB 0644
ptrace.h File 9 KB 0644
sdei.h File 1.46 KB 0644
seccomp.h File 714 B 0644
sections.h File 1.46 KB 0644
shmparam.h File 965 B 0644
signal32.h File 1.45 KB 0644
simd.h File 1.39 KB 0644
smp.h File 4.23 KB 0644
smp_plat.h File 1.43 KB 0644
sparsemem.h File 771 B 0644
spinlock.h File 3.33 KB 0644
spinlock_types.h File 1.06 KB 0644
stack_pointer.h File 247 B 0644
stackprotector.h File 1.11 KB 0644
stacktrace.h File 2.53 KB 0644
stage2_pgtable-nopmd.h File 1.3 KB 0644
stage2_pgtable-nopud.h File 1.24 KB 0644
stage2_pgtable.h File 4.89 KB 0644
stat.h File 1.43 KB 0644
string.h File 2.33 KB 0644
suspend.h File 1.65 KB 0644
sync_bitops.h File 1.11 KB 0644
syscall.h File 2.87 KB 0644
sysreg.h File 25.1 KB 0644
system_misc.h File 1.86 KB 0644
thread_info.h File 3.93 KB 0644
timex.h File 883 B 0644
tlb.h File 2.22 KB 0644
tlbflush.h File 5.38 KB 0644
topology.h File 1.29 KB 0644
traps.h File 3.33 KB 0644
uaccess.h File 12.01 KB 0644
unistd.h File 1.6 KB 0644
unistd32.h File 27.53 KB 0644
uprobes.h File 777 B 0644
vdso.h File 1.09 KB 0644
vdso_datapage.h File 1.53 KB 0644
vectors.h File 1.75 KB 0644
virt.h File 3 KB 0644
vmap_stack.h File 769 B 0644
word-at-a-time.h File 2.22 KB 0644