404

[ Avaa Bypassed ]




Upload:

Command:

botdev@3.144.37.229: ~ $
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PERCPU_H
#define _ASM_X86_PERCPU_H

#ifdef CONFIG_X86_64
#define __percpu_seg		gs
#define __percpu_mov_op		movq
#else
#define __percpu_seg		fs
#define __percpu_mov_op		movl
#endif

#ifdef __ASSEMBLY__

/*
 * PER_CPU finds an address of a per-cpu variable.
 *
 * Args:
 *    var - variable name
 *    reg - 32bit register
 *
 * The resulting address is stored in the "reg" argument.
 *
 * Example:
 *    PER_CPU(cpu_gdt_descr, %ebx)
 */
#ifdef CONFIG_SMP
#define PER_CPU(var, reg)						\
	__percpu_mov_op %__percpu_seg:this_cpu_off, reg;		\
	lea var(reg), reg
#define PER_CPU_VAR(var)	%__percpu_seg:var
#else /* ! SMP */
#define PER_CPU(var, reg)	__percpu_mov_op $var, reg
#define PER_CPU_VAR(var)	var
#endif	/* SMP */

#ifdef CONFIG_X86_64_SMP
#define INIT_PER_CPU_VAR(var)  init_per_cpu__##var
#else
#define INIT_PER_CPU_VAR(var)  var
#endif

#else /* ...!ASSEMBLY */

#include <linux/kernel.h>
#include <linux/stringify.h>

#ifdef CONFIG_SMP
#define __percpu_prefix		"%%"__stringify(__percpu_seg)":"
#define __my_cpu_offset		this_cpu_read(this_cpu_off)

/*
 * Compared to the generic __my_cpu_offset version, the following
 * saves one instruction and avoids clobbering a temp register.
 */
#define arch_raw_cpu_ptr(ptr)				\
({							\
	unsigned long tcp_ptr__;			\
	asm volatile("add " __percpu_arg(1) ", %0"	\
		     : "=r" (tcp_ptr__)			\
		     : "m" (this_cpu_off), "0" (ptr));	\
	(typeof(*(ptr)) __kernel __force *)tcp_ptr__;	\
})
#else
#define __percpu_prefix		""
#endif

#define __percpu_arg(x)		__percpu_prefix "%" #x

/*
 * Initialized pointers to per-cpu variables needed for the boot
 * processor need to use these macros to get the proper address
 * offset from __per_cpu_load on SMP.
 *
 * There also must be an entry in vmlinux_64.lds.S
 */
#define DECLARE_INIT_PER_CPU(var) \
       extern typeof(var) init_per_cpu_var(var)

#ifdef CONFIG_X86_64_SMP
#define init_per_cpu_var(var)  init_per_cpu__##var
#else
#define init_per_cpu_var(var)  var
#endif

/* For arch-specific code, we can use direct single-insn ops (they
 * don't give an lvalue though). */
extern void __bad_percpu_size(void);

#define percpu_to_op(op, var, val)			\
do {							\
	typedef typeof(var) pto_T__;			\
	if (0) {					\
		pto_T__ pto_tmp__;			\
		pto_tmp__ = (val);			\
		(void)pto_tmp__;			\
	}						\
	switch (sizeof(var)) {				\
	case 1:						\
		asm(op "b %1,"__percpu_arg(0)		\
		    : "+m" (var)			\
		    : "qi" ((pto_T__)(val)));		\
		break;					\
	case 2:						\
		asm(op "w %1,"__percpu_arg(0)		\
		    : "+m" (var)			\
		    : "ri" ((pto_T__)(val)));		\
		break;					\
	case 4:						\
		asm(op "l %1,"__percpu_arg(0)		\
		    : "+m" (var)			\
		    : "ri" ((pto_T__)(val)));		\
		break;					\
	case 8:						\
		asm(op "q %1,"__percpu_arg(0)		\
		    : "+m" (var)			\
		    : "re" ((pto_T__)(val)));		\
		break;					\
	default: __bad_percpu_size();			\
	}						\
} while (0)

/*
 * Generate a percpu add to memory instruction and optimize code
 * if one is added or subtracted.
 */
#define percpu_add_op(var, val)						\
do {									\
	typedef typeof(var) pao_T__;					\
	const int pao_ID__ = (__builtin_constant_p(val) &&		\
			      ((val) == 1 || (val) == -1)) ?		\
				(int)(val) : 0;				\
	if (0) {							\
		pao_T__ pao_tmp__;					\
		pao_tmp__ = (val);					\
		(void)pao_tmp__;					\
	}								\
	switch (sizeof(var)) {						\
	case 1:								\
		if (pao_ID__ == 1)					\
			asm("incb "__percpu_arg(0) : "+m" (var));	\
		else if (pao_ID__ == -1)				\
			asm("decb "__percpu_arg(0) : "+m" (var));	\
		else							\
			asm("addb %1, "__percpu_arg(0)			\
			    : "+m" (var)				\
			    : "qi" ((pao_T__)(val)));			\
		break;							\
	case 2:								\
		if (pao_ID__ == 1)					\
			asm("incw "__percpu_arg(0) : "+m" (var));	\
		else if (pao_ID__ == -1)				\
			asm("decw "__percpu_arg(0) : "+m" (var));	\
		else							\
			asm("addw %1, "__percpu_arg(0)			\
			    : "+m" (var)				\
			    : "ri" ((pao_T__)(val)));			\
		break;							\
	case 4:								\
		if (pao_ID__ == 1)					\
			asm("incl "__percpu_arg(0) : "+m" (var));	\
		else if (pao_ID__ == -1)				\
			asm("decl "__percpu_arg(0) : "+m" (var));	\
		else							\
			asm("addl %1, "__percpu_arg(0)			\
			    : "+m" (var)				\
			    : "ri" ((pao_T__)(val)));			\
		break;							\
	case 8:								\
		if (pao_ID__ == 1)					\
			asm("incq "__percpu_arg(0) : "+m" (var));	\
		else if (pao_ID__ == -1)				\
			asm("decq "__percpu_arg(0) : "+m" (var));	\
		else							\
			asm("addq %1, "__percpu_arg(0)			\
			    : "+m" (var)				\
			    : "re" ((pao_T__)(val)));			\
		break;							\
	default: __bad_percpu_size();					\
	}								\
} while (0)

#define percpu_from_op(op, var)				\
({							\
	typeof(var) pfo_ret__;				\
	switch (sizeof(var)) {				\
	case 1:						\
		asm volatile(op "b "__percpu_arg(1)",%0"\
		    : "=q" (pfo_ret__)			\
		    : "m" (var));			\
		break;					\
	case 2:						\
		asm volatile(op "w "__percpu_arg(1)",%0"\
		    : "=r" (pfo_ret__)			\
		    : "m" (var));			\
		break;					\
	case 4:						\
		asm volatile(op "l "__percpu_arg(1)",%0"\
		    : "=r" (pfo_ret__)			\
		    : "m" (var));			\
		break;					\
	case 8:						\
		asm volatile(op "q "__percpu_arg(1)",%0"\
		    : "=r" (pfo_ret__)			\
		    : "m" (var));			\
		break;					\
	default: __bad_percpu_size();			\
	}						\
	pfo_ret__;					\
})

#define percpu_stable_op(op, var)			\
({							\
	typeof(var) pfo_ret__;				\
	switch (sizeof(var)) {				\
	case 1:						\
		asm(op "b "__percpu_arg(P1)",%0"	\
		    : "=q" (pfo_ret__)			\
		    : "p" (&(var)));			\
		break;					\
	case 2:						\
		asm(op "w "__percpu_arg(P1)",%0"	\
		    : "=r" (pfo_ret__)			\
		    : "p" (&(var)));			\
		break;					\
	case 4:						\
		asm(op "l "__percpu_arg(P1)",%0"	\
		    : "=r" (pfo_ret__)			\
		    : "p" (&(var)));			\
		break;					\
	case 8:						\
		asm(op "q "__percpu_arg(P1)",%0"	\
		    : "=r" (pfo_ret__)			\
		    : "p" (&(var)));			\
		break;					\
	default: __bad_percpu_size();			\
	}						\
	pfo_ret__;					\
})

#define percpu_unary_op(op, var)			\
({							\
	switch (sizeof(var)) {				\
	case 1:						\
		asm(op "b "__percpu_arg(0)		\
		    : "+m" (var));			\
		break;					\
	case 2:						\
		asm(op "w "__percpu_arg(0)		\
		    : "+m" (var));			\
		break;					\
	case 4:						\
		asm(op "l "__percpu_arg(0)		\
		    : "+m" (var));			\
		break;					\
	case 8:						\
		asm(op "q "__percpu_arg(0)		\
		    : "+m" (var));			\
		break;					\
	default: __bad_percpu_size();			\
	}						\
})

/*
 * Add return operation
 */
#define percpu_add_return_op(var, val)					\
({									\
	typeof(var) paro_ret__ = val;					\
	switch (sizeof(var)) {						\
	case 1:								\
		asm("xaddb %0, "__percpu_arg(1)				\
			    : "+q" (paro_ret__), "+m" (var)		\
			    : : "memory");				\
		break;							\
	case 2:								\
		asm("xaddw %0, "__percpu_arg(1)				\
			    : "+r" (paro_ret__), "+m" (var)		\
			    : : "memory");				\
		break;							\
	case 4:								\
		asm("xaddl %0, "__percpu_arg(1)				\
			    : "+r" (paro_ret__), "+m" (var)		\
			    : : "memory");				\
		break;							\
	case 8:								\
		asm("xaddq %0, "__percpu_arg(1)				\
			    : "+re" (paro_ret__), "+m" (var)		\
			    : : "memory");				\
		break;							\
	default: __bad_percpu_size();					\
	}								\
	paro_ret__ += val;						\
	paro_ret__;							\
})

/*
 * xchg is implemented using cmpxchg without a lock prefix. xchg is
 * expensive due to the implied lock prefix.  The processor cannot prefetch
 * cachelines if xchg is used.
 */
#define percpu_xchg_op(var, nval)					\
({									\
	typeof(var) pxo_ret__;						\
	typeof(var) pxo_new__ = (nval);					\
	switch (sizeof(var)) {						\
	case 1:								\
		asm("\n\tmov "__percpu_arg(1)",%%al"			\
		    "\n1:\tcmpxchgb %2, "__percpu_arg(1)		\
		    "\n\tjnz 1b"					\
			    : "=&a" (pxo_ret__), "+m" (var)		\
			    : "q" (pxo_new__)				\
			    : "memory");				\
		break;							\
	case 2:								\
		asm("\n\tmov "__percpu_arg(1)",%%ax"			\
		    "\n1:\tcmpxchgw %2, "__percpu_arg(1)		\
		    "\n\tjnz 1b"					\
			    : "=&a" (pxo_ret__), "+m" (var)		\
			    : "r" (pxo_new__)				\
			    : "memory");				\
		break;							\
	case 4:								\
		asm("\n\tmov "__percpu_arg(1)",%%eax"			\
		    "\n1:\tcmpxchgl %2, "__percpu_arg(1)		\
		    "\n\tjnz 1b"					\
			    : "=&a" (pxo_ret__), "+m" (var)		\
			    : "r" (pxo_new__)				\
			    : "memory");				\
		break;							\
	case 8:								\
		asm("\n\tmov "__percpu_arg(1)",%%rax"			\
		    "\n1:\tcmpxchgq %2, "__percpu_arg(1)		\
		    "\n\tjnz 1b"					\
			    : "=&a" (pxo_ret__), "+m" (var)		\
			    : "r" (pxo_new__)				\
			    : "memory");				\
		break;							\
	default: __bad_percpu_size();					\
	}								\
	pxo_ret__;							\
})

/*
 * cmpxchg has no such implied lock semantics as a result it is much
 * more efficient for cpu local operations.
 */
#define percpu_cmpxchg_op(var, oval, nval)				\
({									\
	typeof(var) pco_ret__;						\
	typeof(var) pco_old__ = (oval);					\
	typeof(var) pco_new__ = (nval);					\
	switch (sizeof(var)) {						\
	case 1:								\
		asm("cmpxchgb %2, "__percpu_arg(1)			\
			    : "=a" (pco_ret__), "+m" (var)		\
			    : "q" (pco_new__), "0" (pco_old__)		\
			    : "memory");				\
		break;							\
	case 2:								\
		asm("cmpxchgw %2, "__percpu_arg(1)			\
			    : "=a" (pco_ret__), "+m" (var)		\
			    : "r" (pco_new__), "0" (pco_old__)		\
			    : "memory");				\
		break;							\
	case 4:								\
		asm("cmpxchgl %2, "__percpu_arg(1)			\
			    : "=a" (pco_ret__), "+m" (var)		\
			    : "r" (pco_new__), "0" (pco_old__)		\
			    : "memory");				\
		break;							\
	case 8:								\
		asm("cmpxchgq %2, "__percpu_arg(1)			\
			    : "=a" (pco_ret__), "+m" (var)		\
			    : "r" (pco_new__), "0" (pco_old__)		\
			    : "memory");				\
		break;							\
	default: __bad_percpu_size();					\
	}								\
	pco_ret__;							\
})

/*
 * this_cpu_read() makes gcc load the percpu variable every time it is
 * accessed while this_cpu_read_stable() allows the value to be cached.
 * this_cpu_read_stable() is more efficient and can be used if its value
 * is guaranteed to be valid across cpus.  The current users include
 * get_current() and get_thread_info() both of which are actually
 * per-thread variables implemented as per-cpu variables and thus
 * stable for the duration of the respective task.
 */
#define this_cpu_read_stable(var)	percpu_stable_op("mov", var)

#define raw_cpu_read_1(pcp)		percpu_from_op("mov", pcp)
#define raw_cpu_read_2(pcp)		percpu_from_op("mov", pcp)
#define raw_cpu_read_4(pcp)		percpu_from_op("mov", pcp)

#define raw_cpu_write_1(pcp, val)	percpu_to_op("mov", (pcp), val)
#define raw_cpu_write_2(pcp, val)	percpu_to_op("mov", (pcp), val)
#define raw_cpu_write_4(pcp, val)	percpu_to_op("mov", (pcp), val)
#define raw_cpu_add_1(pcp, val)		percpu_add_op((pcp), val)
#define raw_cpu_add_2(pcp, val)		percpu_add_op((pcp), val)
#define raw_cpu_add_4(pcp, val)		percpu_add_op((pcp), val)
#define raw_cpu_and_1(pcp, val)		percpu_to_op("and", (pcp), val)
#define raw_cpu_and_2(pcp, val)		percpu_to_op("and", (pcp), val)
#define raw_cpu_and_4(pcp, val)		percpu_to_op("and", (pcp), val)
#define raw_cpu_or_1(pcp, val)		percpu_to_op("or", (pcp), val)
#define raw_cpu_or_2(pcp, val)		percpu_to_op("or", (pcp), val)
#define raw_cpu_or_4(pcp, val)		percpu_to_op("or", (pcp), val)
#define raw_cpu_xchg_1(pcp, val)	percpu_xchg_op(pcp, val)
#define raw_cpu_xchg_2(pcp, val)	percpu_xchg_op(pcp, val)
#define raw_cpu_xchg_4(pcp, val)	percpu_xchg_op(pcp, val)

#define this_cpu_read_1(pcp)		percpu_from_op("mov", pcp)
#define this_cpu_read_2(pcp)		percpu_from_op("mov", pcp)
#define this_cpu_read_4(pcp)		percpu_from_op("mov", pcp)
#define this_cpu_write_1(pcp, val)	percpu_to_op("mov", (pcp), val)
#define this_cpu_write_2(pcp, val)	percpu_to_op("mov", (pcp), val)
#define this_cpu_write_4(pcp, val)	percpu_to_op("mov", (pcp), val)
#define this_cpu_add_1(pcp, val)	percpu_add_op((pcp), val)
#define this_cpu_add_2(pcp, val)	percpu_add_op((pcp), val)
#define this_cpu_add_4(pcp, val)	percpu_add_op((pcp), val)
#define this_cpu_and_1(pcp, val)	percpu_to_op("and", (pcp), val)
#define this_cpu_and_2(pcp, val)	percpu_to_op("and", (pcp), val)
#define this_cpu_and_4(pcp, val)	percpu_to_op("and", (pcp), val)
#define this_cpu_or_1(pcp, val)		percpu_to_op("or", (pcp), val)
#define this_cpu_or_2(pcp, val)		percpu_to_op("or", (pcp), val)
#define this_cpu_or_4(pcp, val)		percpu_to_op("or", (pcp), val)
#define this_cpu_xchg_1(pcp, nval)	percpu_xchg_op(pcp, nval)
#define this_cpu_xchg_2(pcp, nval)	percpu_xchg_op(pcp, nval)
#define this_cpu_xchg_4(pcp, nval)	percpu_xchg_op(pcp, nval)

#define raw_cpu_add_return_1(pcp, val)		percpu_add_return_op(pcp, val)
#define raw_cpu_add_return_2(pcp, val)		percpu_add_return_op(pcp, val)
#define raw_cpu_add_return_4(pcp, val)		percpu_add_return_op(pcp, val)
#define raw_cpu_cmpxchg_1(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
#define raw_cpu_cmpxchg_2(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
#define raw_cpu_cmpxchg_4(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)

#define this_cpu_add_return_1(pcp, val)		percpu_add_return_op(pcp, val)
#define this_cpu_add_return_2(pcp, val)		percpu_add_return_op(pcp, val)
#define this_cpu_add_return_4(pcp, val)		percpu_add_return_op(pcp, val)
#define this_cpu_cmpxchg_1(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
#define this_cpu_cmpxchg_2(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
#define this_cpu_cmpxchg_4(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)

#ifdef CONFIG_X86_CMPXCHG64
#define percpu_cmpxchg8b_double(pcp1, pcp2, o1, o2, n1, n2)		\
({									\
	bool __ret;							\
	typeof(pcp1) __o1 = (o1), __n1 = (n1);				\
	typeof(pcp2) __o2 = (o2), __n2 = (n2);				\
	asm volatile("cmpxchg8b "__percpu_arg(1)"\n\tsetz %0\n\t"	\
		    : "=a" (__ret), "+m" (pcp1), "+m" (pcp2), "+d" (__o2) \
		    :  "b" (__n1), "c" (__n2), "a" (__o1));		\
	__ret;								\
})

#define raw_cpu_cmpxchg_double_4	percpu_cmpxchg8b_double
#define this_cpu_cmpxchg_double_4	percpu_cmpxchg8b_double
#endif /* CONFIG_X86_CMPXCHG64 */

/*
 * Per cpu atomic 64 bit operations are only available under 64 bit.
 * 32 bit must fall back to generic operations.
 */
#ifdef CONFIG_X86_64
#define raw_cpu_read_8(pcp)			percpu_from_op("mov", pcp)
#define raw_cpu_write_8(pcp, val)		percpu_to_op("mov", (pcp), val)
#define raw_cpu_add_8(pcp, val)			percpu_add_op((pcp), val)
#define raw_cpu_and_8(pcp, val)			percpu_to_op("and", (pcp), val)
#define raw_cpu_or_8(pcp, val)			percpu_to_op("or", (pcp), val)
#define raw_cpu_add_return_8(pcp, val)		percpu_add_return_op(pcp, val)
#define raw_cpu_xchg_8(pcp, nval)		percpu_xchg_op(pcp, nval)
#define raw_cpu_cmpxchg_8(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)

#define this_cpu_read_8(pcp)			percpu_from_op("mov", pcp)
#define this_cpu_write_8(pcp, val)		percpu_to_op("mov", (pcp), val)
#define this_cpu_add_8(pcp, val)		percpu_add_op((pcp), val)
#define this_cpu_and_8(pcp, val)		percpu_to_op("and", (pcp), val)
#define this_cpu_or_8(pcp, val)			percpu_to_op("or", (pcp), val)
#define this_cpu_add_return_8(pcp, val)		percpu_add_return_op(pcp, val)
#define this_cpu_xchg_8(pcp, nval)		percpu_xchg_op(pcp, nval)
#define this_cpu_cmpxchg_8(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)

/*
 * Pretty complex macro to generate cmpxchg16 instruction.  The instruction
 * is not supported on early AMD64 processors so we must be able to emulate
 * it in software.  The address used in the cmpxchg16 instruction must be
 * aligned to a 16 byte boundary.
 */
#define percpu_cmpxchg16b_double(pcp1, pcp2, o1, o2, n1, n2)		\
({									\
	bool __ret;							\
	typeof(pcp1) __o1 = (o1), __n1 = (n1);				\
	typeof(pcp2) __o2 = (o2), __n2 = (n2);				\
	alternative_io("leaq %P1,%%rsi\n\tcall this_cpu_cmpxchg16b_emu\n\t", \
		       "cmpxchg16b " __percpu_arg(1) "\n\tsetz %0\n\t",	\
		       X86_FEATURE_CX16,				\
		       ASM_OUTPUT2("=a" (__ret), "+m" (pcp1),		\
				   "+m" (pcp2), "+d" (__o2)),		\
		       "b" (__n1), "c" (__n2), "a" (__o1) : "rsi");	\
	__ret;								\
})

#define raw_cpu_cmpxchg_double_8	percpu_cmpxchg16b_double
#define this_cpu_cmpxchg_double_8	percpu_cmpxchg16b_double

#endif

static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,
                        const unsigned long __percpu *addr)
{
	unsigned long __percpu *a =
		(unsigned long __percpu *)addr + nr / BITS_PER_LONG;

#ifdef CONFIG_X86_64
	return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_8(*a)) != 0;
#else
	return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_4(*a)) != 0;
#endif
}

static inline bool x86_this_cpu_variable_test_bit(int nr,
                        const unsigned long __percpu *addr)
{
	bool oldbit;

	asm volatile("btl "__percpu_arg(2)",%1"
			CC_SET(c)
			: CC_OUT(c) (oldbit)
			: "m" (*(unsigned long __percpu *)addr), "Ir" (nr));

	return oldbit;
}

#define x86_this_cpu_test_bit(nr, addr)			\
	(__builtin_constant_p((nr))			\
	 ? x86_this_cpu_constant_test_bit((nr), (addr))	\
	 : x86_this_cpu_variable_test_bit((nr), (addr)))


#include <asm-generic/percpu.h>

/* We can use this directly for local CPU (faster). */
DECLARE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off);

#endif /* !__ASSEMBLY__ */

#ifdef CONFIG_SMP

/*
 * Define the "EARLY_PER_CPU" macros.  These are used for some per_cpu
 * variables that are initialized and accessed before there are per_cpu
 * areas allocated.
 */

#define	DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)			\
	DEFINE_PER_CPU(_type, _name) = _initvalue;			\
	__typeof__(_type) _name##_early_map[NR_CPUS] __initdata =	\
				{ [0 ... NR_CPUS-1] = _initvalue };	\
	__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map

#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue)	\
	DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue;		\
	__typeof__(_type) _name##_early_map[NR_CPUS] __initdata =	\
				{ [0 ... NR_CPUS-1] = _initvalue };	\
	__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map

#define EXPORT_EARLY_PER_CPU_SYMBOL(_name)			\
	EXPORT_PER_CPU_SYMBOL(_name)

#define DECLARE_EARLY_PER_CPU(_type, _name)			\
	DECLARE_PER_CPU(_type, _name);				\
	extern __typeof__(_type) *_name##_early_ptr;		\
	extern __typeof__(_type)  _name##_early_map[]

#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name)		\
	DECLARE_PER_CPU_READ_MOSTLY(_type, _name);		\
	extern __typeof__(_type) *_name##_early_ptr;		\
	extern __typeof__(_type)  _name##_early_map[]

#define	early_per_cpu_ptr(_name) (_name##_early_ptr)
#define	early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
#define	early_per_cpu(_name, _cpu) 				\
	*(early_per_cpu_ptr(_name) ?				\
		&early_per_cpu_ptr(_name)[_cpu] :		\
		&per_cpu(_name, _cpu))

#else	/* !CONFIG_SMP */
#define	DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)		\
	DEFINE_PER_CPU(_type, _name) = _initvalue

#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue)	\
	DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue

#define EXPORT_EARLY_PER_CPU_SYMBOL(_name)			\
	EXPORT_PER_CPU_SYMBOL(_name)

#define DECLARE_EARLY_PER_CPU(_type, _name)			\
	DECLARE_PER_CPU(_type, _name)

#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name)		\
	DECLARE_PER_CPU_READ_MOSTLY(_type, _name)

#define	early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
#define	early_per_cpu_ptr(_name) NULL
/* no early_per_cpu_map() */

#endif	/* !CONFIG_SMP */

#endif /* _ASM_X86_PERCPU_H */

Filemanager

Name Type Size Permission Actions
crypto Folder 0755
e820 Folder 0755
fpu Folder 0755
numachip Folder 0755
trace Folder 0755
uv Folder 0755
xen Folder 0755
Kbuild File 294 B 0644
a.out-core.h File 1.89 KB 0644
acenv.h File 1.56 KB 0644
acpi.h File 4.76 KB 0644
agp.h File 1.04 KB 0644
alternative-asm.h File 2.43 KB 0644
alternative.h File 8.28 KB 0644
amd_nb.h File 2.98 KB 0644
apb_timer.h File 1.43 KB 0644
apic.h File 14.53 KB 0644
apic_flat_64.h File 151 B 0644
apicdef.h File 11.26 KB 0644
apm.h File 1.8 KB 0644
arch_hweight.h File 1.28 KB 0644
archrandom.h File 3.03 KB 0644
asm-offsets.h File 35 B 0644
asm-prototypes.h File 946 B 0644
asm.h File 4.97 KB 0644
atomic.h File 6.02 KB 0644
atomic64_32.h File 8.71 KB 0644
atomic64_64.h File 6.31 KB 0644
barrier.h File 3.6 KB 0644
bios_ebda.h File 914 B 0644
bitops.h File 13.78 KB 0644
boot.h File 1.53 KB 0644
bootparam_utils.h File 2.86 KB 0644
bug.h File 2.07 KB 0644
bugs.h File 493 B 0644
cache.h File 641 B 0644
cacheflush.h File 306 B 0644
cacheinfo.h File 209 B 0644
calgary.h File 2.31 KB 0644
ce4100.h File 121 B 0644
checksum.h File 133 B 0644
checksum_32.h File 4.86 KB 0644
checksum_64.h File 5.41 KB 0644
clocksource.h File 488 B 0644
cmdline.h File 302 B 0644
cmpxchg.h File 7.68 KB 0644
cmpxchg_32.h File 3.15 KB 0644
cmpxchg_64.h File 543 B 0644
compat.h File 7.37 KB 0644
cpu.h File 975 B 0644
cpu_device_id.h File 1.38 KB 0644
cpu_entry_area.h File 2.27 KB 0644
cpufeature.h File 7.75 KB 0644
cpufeatures.h File 24.62 KB 0644
cpumask.h File 408 B 0644
crash.h File 320 B 0644
current.h File 443 B 0644
debugreg.h File 2.67 KB 0644
delay.h File 208 B 0644
desc.h File 11.42 KB 0644
desc_defs.h File 3.16 KB 0644
device.h File 568 B 0644
disabled-features.h File 2.31 KB 0644
div64.h File 1.79 KB 0644
dma-mapping.h File 2.4 KB 0644
dma.h File 9.58 KB 0644
dmi.h File 556 B 0644
dwarf2.h File 2.43 KB 0644
edac.h File 474 B 0644
efi.h File 6.9 KB 0644
elf.h File 10.82 KB 0644
emergency-restart.h File 202 B 0644
entry_arch.h File 1.88 KB 0644
espfix.h File 426 B 0644
exec.h File 37 B 0644
export.h File 120 B 0644
extable.h File 1.27 KB 0644
fb.h File 540 B 0644
fixmap.h File 6.04 KB 0644
floppy.h File 6.59 KB 0644
frame.h File 815 B 0644
ftrace.h File 1.8 KB 0644
futex.h File 2.2 KB 0644
gart.h File 2.64 KB 0644
genapic.h File 22 B 0644
geode.h File 842 B 0644
hardirq.h File 2.3 KB 0644
highmem.h File 2.6 KB 0644
hpet.h File 3.38 KB 0644
hugetlb.h File 2.15 KB 0644
hw_breakpoint.h File 1.96 KB 0644
hw_irq.h File 3.85 KB 0644
hypervisor.h File 1.84 KB 0644
i8259.h File 1.93 KB 0644
ia32.h File 1.46 KB 0644
ia32_unistd.h File 313 B 0644
imr.h File 1.81 KB 0644
inat.h File 6.58 KB 0644
inat_types.h File 1013 B 0644
init.h File 632 B 0644
insn-eval.h File 837 B 0644
insn.h File 7.46 KB 0644
inst.h File 5.07 KB 0644
intel-family.h File 3.29 KB 0644
intel-mid.h File 4.91 KB 0644
intel_ds.h File 793 B 0644
intel_mid_vrtc.h File 326 B 0644
intel_pmc_ipc.h File 2.08 KB 0644
intel_pt.h File 292 B 0644
intel_punit_ipc.h File 4.56 KB 0644
intel_rdt_sched.h File 2.59 KB 0644
intel_scu_ipc.h File 2.3 KB 0644
intel_telemetry.h File 3.96 KB 0644
invpcid.h File 1.57 KB 0644
io.h File 12.21 KB 0644
io_apic.h File 5.63 KB 0644
iomap.h File 1.22 KB 0644
iommu.h File 392 B 0644
iommu_table.h File 3.82 KB 0644
iosf_mbi.h File 5.74 KB 0644
ipi.h File 2.84 KB 0644
irq.h File 1.12 KB 0644
irq_regs.h File 679 B 0644
irq_remapping.h File 2.96 KB 0644
irq_vectors.h File 4.12 KB 0644
irq_work.h File 397 B 0644
irqdomain.h File 1.61 KB 0644
irqflags.h File 4.38 KB 0644
ist.h File 735 B 0644
jump_label.h File 2.44 KB 0644
kasan.h File 966 B 0644
kaslr.h File 424 B 0644
kbdleds.h File 454 B 0644
kdebug.h File 752 B 0644
kexec-bzimage64.h File 189 B 0644
kexec.h File 6.69 KB 0644
kgdb.h File 2.09 KB 0644
kmap_types.h File 289 B 0644
kprobes.h File 3.82 KB 0644
kvm_emulate.h File 15.23 KB 0644
kvm_guest.h File 172 B 0644
kvm_host.h File 42.72 KB 0644
kvm_page_track.h File 2.48 KB 0644
kvm_para.h File 3 KB 0644
kvmclock.h File 170 B 0644
linkage.h File 581 B 0644
livepatch.h File 1.12 KB 0644
local.h File 3.83 KB 0644
local64.h File 33 B 0644
mach_timer.h File 1.55 KB 0644
mach_traps.h File 1013 B 0644
math_emu.h File 395 B 0644
mc146818rtc.h File 2.76 KB 0644
mce.h File 12.54 KB 0644
mem_encrypt.h File 2.83 KB 0644
microcode.h File 4.14 KB 0644
microcode_amd.h File 1.41 KB 0644
microcode_intel.h File 2.46 KB 0644
misc.h File 143 B 0644
mmconfig.h File 374 B 0644
mmu.h File 1.57 KB 0644
mmu_context.h File 10.27 KB 0644
mmx.h File 337 B 0644
mmzone.h File 129 B 0644
mmzone_32.h File 1.16 KB 0644
mmzone_64.h File 430 B 0644
module.h File 2.05 KB 0644
mpspec.h File 3.93 KB 0644
mpspec_def.h File 3.93 KB 0644
mpx.h File 2.97 KB 0644
mshyperv.h File 10.69 KB 0644
msi.h File 392 B 0644
msidef.h File 1.77 KB 0644
msr-index.h File 30.36 KB 0644
msr-trace.h File 1.35 KB 0644
msr.h File 10.85 KB 0644
mtrr.h File 4.62 KB 0644
mwait.h File 3.74 KB 0644
nmi.h File 1.39 KB 0644
nops.h File 4.31 KB 0644
nospec-branch.h File 10.87 KB 0644
numa.h File 2.18 KB 0644
numa_32.h File 256 B 0644
olpc.h File 3.16 KB 0644
olpc_ofw.h File 1.1 KB 0644
orc_lookup.h File 1.63 KB 0644
orc_types.h File 3.47 KB 0644
page.h File 2.18 KB 0644
page_32.h File 1.01 KB 0644
page_32_types.h File 1.7 KB 0644
page_64.h File 1.42 KB 0644
page_64_types.h File 2.34 KB 0644
page_types.h File 2.29 KB 0644
paravirt.h File 23.31 KB 0644
paravirt_types.h File 22.15 KB 0644
parport.h File 314 B 0644
pat.h File 768 B 0644
pci-direct.h File 995 B 0644
pci-functions.h File 654 B 0644
pci.h File 3.51 KB 0644
pci_64.h File 684 B 0644
pci_x86.h File 5.71 KB 0644
percpu.h File 18.97 KB 0644
perf_event.h File 8.82 KB 0644
perf_event_p4.h File 26.1 KB 0644
pgalloc.h File 5.57 KB 0644
pgtable-2level.h File 2.75 KB 0644
pgtable-2level_types.h File 867 B 0644
pgtable-3level.h File 10.24 KB 0644
pgtable-3level_types.h File 1.06 KB 0644
pgtable-invert.h File 1.07 KB 0644
pgtable.h File 33.91 KB 0644
pgtable_32.h File 3.1 KB 0644
pgtable_32_types.h File 2.01 KB 0644
pgtable_64.h File 7.37 KB 0644
pgtable_64_types.h File 3.67 KB 0644
pgtable_types.h File 16.75 KB 0644
pkeys.h File 3.17 KB 0644
platform_sst_audio.h File 3.18 KB 0644
pm-trace.h File 611 B 0644
posix_types.h File 144 B 0644
preempt.h File 3.04 KB 0644
probe_roms.h File 273 B 0644
processor-cyrix.h File 879 B 0644
processor-flags.h File 1.71 KB 0644
processor.h File 24.54 KB 0644
prom.h File 1 KB 0644
proto.h File 1003 B 0644
pti.h File 369 B 0644
ptrace.h File 8.52 KB 0644
purgatory.h File 571 B 0644
pvclock-abi.h File 1.49 KB 0644
pvclock.h File 2.64 KB 0644
qrwlock.h File 199 B 0644
qspinlock.h File 2.54 KB 0644
qspinlock_paravirt.h File 1.85 KB 0644
realmode.h File 1.76 KB 0644
reboot.h File 850 B 0644
reboot_fixups.h File 183 B 0644
refcount.h File 2.83 KB 0644
required-features.h File 2.81 KB 0644
rio.h File 2.57 KB 0644
rmwcc.h File 1.62 KB 0644
rwsem.h File 7.02 KB 0644
seccomp.h File 510 B 0644
sections.h File 465 B 0644
segment.h File 9.47 KB 0644
serial.h File 1.11 KB 0644
set_memory.h File 3.86 KB 0644
setup.h File 3.4 KB 0644
setup_arch.h File 77 B 0644
shmparam.h File 193 B 0644
sigcontext.h File 261 B 0644
sigframe.h File 2.25 KB 0644
sighandling.h File 649 B 0644
signal.h File 2.37 KB 0644
simd.h File 287 B 0644
smap.h File 1.71 KB 0644
smp.h File 4.73 KB 0644
sparsemem.h File 1.01 KB 0644
spec-ctrl.h File 2.81 KB 0644
special_insns.h File 5.04 KB 0644
spinlock.h File 1.19 KB 0644
spinlock_types.h File 719 B 0644
sta2x11.h File 352 B 0644
stackprotector.h File 4.13 KB 0644
stacktrace.h File 2.57 KB 0644
string.h File 129 B 0644
string_32.h File 7.74 KB 0644
string_64.h File 3.56 KB 0644
suspend.h File 131 B 0644
suspend_32.h File 822 B 0644
suspend_64.h File 1.79 KB 0644
svm.h File 7.06 KB 0644
swiotlb.h File 991 B 0644
switch_to.h File 2.98 KB 0644
sync_bitops.h File 3.42 KB 0644
sys_ia32.h File 1.8 KB 0644
syscall.h File 5.14 KB 0644
syscalls.h File 1.39 KB 0644
sysfb.h File 2.54 KB 0644
tce.h File 1.68 KB 0644
text-patching.h File 2.3 KB 0644
thread_info.h File 9.33 KB 0644
time.h File 331 B 0644
timer.h File 1 KB 0644
timex.h File 546 B 0644
tlb.h File 1.05 KB 0644
tlbbatch.h File 332 B 0644
tlbflush.h File 17.09 KB 0644
topology.h File 4.84 KB 0644
trace_clock.h File 406 B 0644
traps.h File 5.74 KB 0644
tsc.h File 1.93 KB 0644
uaccess.h File 21.69 KB 0644
uaccess_32.h File 1.54 KB 0644
uaccess_64.h File 5.32 KB 0644
umip.h File 329 B 0644
unaligned.h File 345 B 0644
unistd.h File 1.45 KB 0644
unwind.h File 3.13 KB 0644
unwind_hints.h File 3.01 KB 0644
uprobes.h File 1.57 KB 0644
user.h File 2.2 KB 0644
user32.h File 2.11 KB 0644
user_32.h File 4.92 KB 0644
user_64.h File 5.21 KB 0644
vdso.h File 1.09 KB 0644
vga.h File 740 B 0644
vgtod.h File 2.13 KB 0644
virtext.h File 2.62 KB 0644
vm86.h File 2.16 KB 0644
vmx.h File 23.5 KB 0644
vsyscall.h File 635 B 0644
vvar.h File 1.38 KB 0644
word-at-a-time.h File 2.54 KB 0644
x86_init.h File 9.25 KB 0644
xor.h File 10.26 KB 0644
xor_32.h File 14.4 KB 0644
xor_64.h File 716 B 0644
xor_avx.h File 4.5 KB 0644