diff --git a/arch/alpha/include/asm/bitops.h b/arch/alpha/include/asm/bitops.h index 5adca78830..e1d8483a45 100644 --- a/arch/alpha/include/asm/bitops.h +++ b/arch/alpha/include/asm/bitops.h @@ -430,8 +430,6 @@ static inline unsigned int __arch_hweight8(unsigned int w) #endif /* __KERNEL__ */ -#include - #ifdef __KERNEL__ /* diff --git a/arch/alpha/include/asm/processor.h b/arch/alpha/include/asm/processor.h index 6100431da0..090499c99c 100644 --- a/arch/alpha/include/asm/processor.h +++ b/arch/alpha/include/asm/processor.h @@ -42,7 +42,7 @@ extern void start_thread(struct pt_regs *, unsigned long, unsigned long); struct task_struct; extern void release_thread(struct task_struct *); -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) diff --git a/arch/alpha/include/asm/spinlock_types.h b/arch/alpha/include/asm/spinlock_types.h index 1d5716bc06..2526fd3be5 100644 --- a/arch/alpha/include/asm/spinlock_types.h +++ b/arch/alpha/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef _ALPHA_SPINLOCK_TYPES_H #define _ALPHA_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h index 1dd9baf4a6..284d28755b 100644 --- a/arch/alpha/include/uapi/asm/socket.h +++ b/arch/alpha/include/uapi/asm/socket.h @@ -131,6 +131,8 @@ #define SO_BUF_LOCK 72 +#define SO_RESERVE_MEM 73 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/arc/Kbuild b/arch/arc/Kbuild index 699d8cae9b..b94102fff6 100644 --- a/arch/arc/Kbuild +++ b/arch/arc/Kbuild @@ -1,3 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-y += kernel/ obj-y += mm/ + +# for cleaning +subdir- += boot diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 3a5a80f302..3c2a4753d0 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -20,7 +20,6 @@ config ARC select COMMON_CLK select DMA_DIRECT_REMAP select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC) - select GENERIC_FIND_FIRST_BIT # for now, we don't need GENERIC_IRQ_PROBE, CONFIG_GENERIC_IRQ_CHIP select GENERIC_IRQ_SHOW select GENERIC_PCI_IOMAP @@ -32,7 +31,6 @@ config ARC select HAVE_ARCH_TRANSPARENT_HUGEPAGE if ARC_MMU_V4 select HAVE_DEBUG_STACKOVERFLOW select HAVE_DEBUG_KMEMLEAK - select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_IOREMAP_PROT select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZMA @@ -40,7 +38,6 @@ config ARC select HAVE_KRETPROBES select HAVE_MOD_ARCH_SPECIFIC select HAVE_PERF_EVENTS - select HANDLE_DOMAIN_IRQ select IRQ_DOMAIN select MODULES_USE_ELF_RELA select OF diff --git a/arch/arc/Makefile b/arch/arc/Makefile index 8782a03f24..efc54f3e35 100644 --- a/arch/arc/Makefile +++ b/arch/arc/Makefile @@ -14,10 +14,10 @@ cflags-y += -fno-common -pipe -fno-builtin -mmedium-calls -D__linux__ tune-mcpu-def-$(CONFIG_ISA_ARCOMPACT) := -mcpu=arc700 tune-mcpu-def-$(CONFIG_ISA_ARCV2) := -mcpu=hs38 -ifeq ($(CONFIG_ARC_TUNE_MCPU),"") +ifeq ($(CONFIG_ARC_TUNE_MCPU),) cflags-y += $(tune-mcpu-def-y) else -tune-mcpu := $(shell echo $(CONFIG_ARC_TUNE_MCPU)) +tune-mcpu := $(CONFIG_ARC_TUNE_MCPU) ifneq ($(call cc-option,$(tune-mcpu)),) cflags-y += $(tune-mcpu) else @@ -112,6 +112,3 @@ uImage: $(uimage-default-y) @$(kecho) ' Image $(boot)/uImage is ready' CLEAN_FILES += $(boot)/uImage - -archclean: - $(Q)$(MAKE) $(clean)=$(boot) diff --git a/arch/arc/boot/dts/Makefile b/arch/arc/boot/dts/Makefile index 8483a86c74..4237aa5de3 100644 --- a/arch/arc/boot/dts/Makefile +++ b/arch/arc/boot/dts/Makefile @@ -2,8 +2,8 @@ # Built-in dtb builtindtb-y := nsim_700 -ifneq ($(CONFIG_ARC_BUILTIN_DTB_NAME),"") - builtindtb-y := $(patsubst "%",%,$(CONFIG_ARC_BUILTIN_DTB_NAME)) +ifneq ($(CONFIG_ARC_BUILTIN_DTB_NAME),) + builtindtb-y := $(CONFIG_ARC_BUILTIN_DTB_NAME) endif obj-y += $(builtindtb-y).dtb.o diff --git a/arch/arc/include/asm/bitops.h b/arch/arc/include/asm/bitops.h index a7daaf64ae..bdb7e190a2 100644 --- a/arch/arc/include/asm/bitops.h +++ b/arch/arc/include/asm/bitops.h @@ -189,7 +189,6 @@ static inline __attribute__ ((const)) unsigned long __ffs(unsigned long x) #include #include -#include #include #include diff --git a/arch/arc/include/asm/irqflags-compact.h b/arch/arc/include/asm/irqflags-compact.h index 863d63ad18..0d63e568d6 100644 --- a/arch/arc/include/asm/irqflags-compact.h +++ b/arch/arc/include/asm/irqflags-compact.h @@ -50,8 +50,12 @@ * are redone after IRQs are re-enabled (and gcc doesn't reuse stale register) * * Noted at the time of Abilis Timer List corruption - * Orig Bug + Rejected solution : https://lkml.org/lkml/2013/3/29/67 - * Reasoning : https://lkml.org/lkml/2013/4/8/15 + * + * Orig Bug + Rejected solution: + * https://lore.kernel.org/lkml/1364553218-31255-1-git-send-email-vgupta@synopsys.com + * + * Reasoning: + * https://lore.kernel.org/lkml/CA+55aFyFWjpSVQM6M266tKrG_ZXJzZ-nYejpmXYQXbrr42mGPQ@mail.gmail.com * ******************************************************************/ diff --git a/arch/arc/include/asm/kprobes.h b/arch/arc/include/asm/kprobes.h index 2134721dce..de1566e32c 100644 --- a/arch/arc/include/asm/kprobes.h +++ b/arch/arc/include/asm/kprobes.h @@ -46,7 +46,7 @@ struct kprobe_ctlblk { }; int kprobe_fault_handler(struct pt_regs *regs, unsigned long cause); -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); void trap_is_kprobe(unsigned long address, struct pt_regs *regs); #else #define trap_is_kprobe(address, regs) diff --git a/arch/arc/include/asm/perf_event.h b/arch/arc/include/asm/perf_event.h index e1971d34ef..4c919c0f4b 100644 --- a/arch/arc/include/asm/perf_event.h +++ b/arch/arc/include/asm/perf_event.h @@ -63,166 +63,4 @@ struct arc_reg_cc_build { #define PERF_COUNT_ARC_HW_MAX (PERF_COUNT_HW_MAX + 8) -/* - * Some ARC pct quirks: - * - * PERF_COUNT_HW_STALLED_CYCLES_BACKEND - * PERF_COUNT_HW_STALLED_CYCLES_FRONTEND - * The ARC 700 can either measure stalls per pipeline stage, or all stalls - * combined; for now we assign all stalls to STALLED_CYCLES_BACKEND - * and all pipeline flushes (e.g. caused by mispredicts, etc.) to - * STALLED_CYCLES_FRONTEND. - * - * We could start multiple performance counters and combine everything - * afterwards, but that makes it complicated. - * - * Note that I$ cache misses aren't counted by either of the two! - */ - -/* - * ARC PCT has hardware conditions with fixed "names" but variable "indexes" - * (based on a specific RTL build) - * Below is the static map between perf generic/arc specific event_id and - * h/w condition names. - * At the time of probe, we loop thru each index and find it's name to - * complete the mapping of perf event_id to h/w index as latter is needed - * to program the counter really - */ -static const char * const arc_pmu_ev_hw_map[] = { - /* count cycles */ - [PERF_COUNT_HW_CPU_CYCLES] = "crun", - [PERF_COUNT_HW_REF_CPU_CYCLES] = "crun", - [PERF_COUNT_HW_BUS_CYCLES] = "crun", - - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = "bflush", - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = "bstall", - - /* counts condition */ - [PERF_COUNT_HW_INSTRUCTIONS] = "iall", - /* All jump instructions that are taken */ - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = "ijmptak", -#ifdef CONFIG_ISA_ARCV2 - [PERF_COUNT_HW_BRANCH_MISSES] = "bpmp", -#else - [PERF_COUNT_ARC_BPOK] = "bpok", /* NP-NT, PT-T, PNT-NT */ - [PERF_COUNT_HW_BRANCH_MISSES] = "bpfail", /* NP-T, PT-NT, PNT-T */ -#endif - [PERF_COUNT_ARC_LDC] = "imemrdc", /* Instr: mem read cached */ - [PERF_COUNT_ARC_STC] = "imemwrc", /* Instr: mem write cached */ - - [PERF_COUNT_ARC_DCLM] = "dclm", /* D-cache Load Miss */ - [PERF_COUNT_ARC_DCSM] = "dcsm", /* D-cache Store Miss */ - [PERF_COUNT_ARC_ICM] = "icm", /* I-cache Miss */ - [PERF_COUNT_ARC_EDTLB] = "edtlb", /* D-TLB Miss */ - [PERF_COUNT_ARC_EITLB] = "eitlb", /* I-TLB Miss */ - - [PERF_COUNT_HW_CACHE_REFERENCES] = "imemrdc", /* Instr: mem read cached */ - [PERF_COUNT_HW_CACHE_MISSES] = "dclm", /* D-cache Load Miss */ -}; - -#define C(_x) PERF_COUNT_HW_CACHE_##_x -#define CACHE_OP_UNSUPPORTED 0xffff - -static const unsigned int arc_pmu_cache_map[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { - [C(L1D)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = PERF_COUNT_ARC_LDC, - [C(RESULT_MISS)] = PERF_COUNT_ARC_DCLM, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = PERF_COUNT_ARC_STC, - [C(RESULT_MISS)] = PERF_COUNT_ARC_DCSM, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(L1I)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = PERF_COUNT_HW_INSTRUCTIONS, - [C(RESULT_MISS)] = PERF_COUNT_ARC_ICM, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(LL)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(DTLB)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = PERF_COUNT_ARC_LDC, - [C(RESULT_MISS)] = PERF_COUNT_ARC_EDTLB, - }, - /* DTLB LD/ST Miss not segregated by h/w*/ - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(ITLB)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = PERF_COUNT_ARC_EITLB, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(BPU)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS, - [C(RESULT_MISS)] = PERF_COUNT_HW_BRANCH_MISSES, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(NODE)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, -}; - #endif /* __ASM_PERF_EVENT_H */ diff --git a/arch/arc/include/asm/processor.h b/arch/arc/include/asm/processor.h index f28afcf5c6..54db9d7bb5 100644 --- a/arch/arc/include/asm/processor.h +++ b/arch/arc/include/asm/processor.h @@ -70,7 +70,7 @@ struct task_struct; extern void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long usp); -extern unsigned int get_wchan(struct task_struct *p); +extern unsigned int __get_wchan(struct task_struct *p); #endif /* !__ASSEMBLY__ */ diff --git a/arch/arc/include/asm/ptrace.h b/arch/arc/include/asm/ptrace.h index 4c3c9be5bd..cca8d6583e 100644 --- a/arch/arc/include/asm/ptrace.h +++ b/arch/arc/include/asm/ptrace.h @@ -149,6 +149,11 @@ static inline long regs_return_value(struct pt_regs *regs) return (long)regs->r0; } +static inline void instruction_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + instruction_pointer(regs) = val; +} #endif /* !__ASSEMBLY__ */ #endif /* __ASM_PTRACE_H */ diff --git a/arch/arc/kernel/irq.c b/arch/arc/kernel/irq.c index ef909dd4b4..dd09b58ff8 100644 --- a/arch/arc/kernel/irq.c +++ b/arch/arc/kernel/irq.c @@ -6,6 +6,8 @@ #include #include #include + +#include #include /* @@ -39,5 +41,11 @@ void __init init_IRQ(void) */ void arch_do_IRQ(unsigned int hwirq, struct pt_regs *regs) { - handle_domain_irq(NULL, hwirq, regs); + struct pt_regs *old_regs; + + irq_enter(); + old_regs = set_irq_regs(regs); + generic_handle_domain_irq(NULL, hwirq); + set_irq_regs(old_regs); + irq_exit(); } diff --git a/arch/arc/kernel/kprobes.c b/arch/arc/kernel/kprobes.c index 5f0415fc73..e71d64119d 100644 --- a/arch/arc/kernel/kprobes.c +++ b/arch/arc/kernel/kprobes.c @@ -363,8 +363,9 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, static void __used kretprobe_trampoline_holder(void) { - __asm__ __volatile__(".global kretprobe_trampoline\n" - "kretprobe_trampoline:\n" "nop\n"); + __asm__ __volatile__(".global __kretprobe_trampoline\n" + "__kretprobe_trampoline:\n" + "nop\n"); } void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, @@ -375,13 +376,13 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, ri->fp = NULL; /* Replace the return addr with trampoline addr */ - regs->blink = (unsigned long)&kretprobe_trampoline; + regs->blink = (unsigned long)&__kretprobe_trampoline; } static int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) { - regs->ret = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL); + regs->ret = __kretprobe_trampoline_handler(regs, NULL); /* By returning a non zero value, we are telling the kprobe handler * that we don't want the post_handler to run @@ -390,7 +391,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, } static struct kprobe trampoline_p = { - .addr = (kprobe_opcode_t *) &kretprobe_trampoline, + .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, .pre_handler = trampoline_probe_handler }; @@ -402,7 +403,7 @@ int __init arch_init_kprobes(void) int __kprobes arch_trampoline_kprobe(struct kprobe *p) { - if (p->addr == (kprobe_opcode_t *) &kretprobe_trampoline) + if (p->addr == (kprobe_opcode_t *) &__kretprobe_trampoline) return 1; return 0; diff --git a/arch/arc/kernel/perf_event.c b/arch/arc/kernel/perf_event.c index 145722f80c..adff957962 100644 --- a/arch/arc/kernel/perf_event.c +++ b/arch/arc/kernel/perf_event.c @@ -17,6 +17,168 @@ /* HW holds 8 symbols + one for null terminator */ #define ARCPMU_EVENT_NAME_LEN 9 +/* + * Some ARC pct quirks: + * + * PERF_COUNT_HW_STALLED_CYCLES_BACKEND + * PERF_COUNT_HW_STALLED_CYCLES_FRONTEND + * The ARC 700 can either measure stalls per pipeline stage, or all stalls + * combined; for now we assign all stalls to STALLED_CYCLES_BACKEND + * and all pipeline flushes (e.g. caused by mispredicts, etc.) to + * STALLED_CYCLES_FRONTEND. + * + * We could start multiple performance counters and combine everything + * afterwards, but that makes it complicated. + * + * Note that I$ cache misses aren't counted by either of the two! + */ + +/* + * ARC PCT has hardware conditions with fixed "names" but variable "indexes" + * (based on a specific RTL build) + * Below is the static map between perf generic/arc specific event_id and + * h/w condition names. + * At the time of probe, we loop thru each index and find it's name to + * complete the mapping of perf event_id to h/w index as latter is needed + * to program the counter really + */ +static const char * const arc_pmu_ev_hw_map[] = { + /* count cycles */ + [PERF_COUNT_HW_CPU_CYCLES] = "crun", + [PERF_COUNT_HW_REF_CPU_CYCLES] = "crun", + [PERF_COUNT_HW_BUS_CYCLES] = "crun", + + [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = "bflush", + [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = "bstall", + + /* counts condition */ + [PERF_COUNT_HW_INSTRUCTIONS] = "iall", + /* All jump instructions that are taken */ + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = "ijmptak", +#ifdef CONFIG_ISA_ARCV2 + [PERF_COUNT_HW_BRANCH_MISSES] = "bpmp", +#else + [PERF_COUNT_ARC_BPOK] = "bpok", /* NP-NT, PT-T, PNT-NT */ + [PERF_COUNT_HW_BRANCH_MISSES] = "bpfail", /* NP-T, PT-NT, PNT-T */ +#endif + [PERF_COUNT_ARC_LDC] = "imemrdc", /* Instr: mem read cached */ + [PERF_COUNT_ARC_STC] = "imemwrc", /* Instr: mem write cached */ + + [PERF_COUNT_ARC_DCLM] = "dclm", /* D-cache Load Miss */ + [PERF_COUNT_ARC_DCSM] = "dcsm", /* D-cache Store Miss */ + [PERF_COUNT_ARC_ICM] = "icm", /* I-cache Miss */ + [PERF_COUNT_ARC_EDTLB] = "edtlb", /* D-TLB Miss */ + [PERF_COUNT_ARC_EITLB] = "eitlb", /* I-TLB Miss */ + + [PERF_COUNT_HW_CACHE_REFERENCES] = "imemrdc", /* Instr: mem read cached */ + [PERF_COUNT_HW_CACHE_MISSES] = "dclm", /* D-cache Load Miss */ +}; + +#define C(_x) PERF_COUNT_HW_CACHE_##_x +#define CACHE_OP_UNSUPPORTED 0xffff + +static const unsigned int arc_pmu_cache_map[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { + [C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = PERF_COUNT_ARC_LDC, + [C(RESULT_MISS)] = PERF_COUNT_ARC_DCLM, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = PERF_COUNT_ARC_STC, + [C(RESULT_MISS)] = PERF_COUNT_ARC_DCSM, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = PERF_COUNT_HW_INSTRUCTIONS, + [C(RESULT_MISS)] = PERF_COUNT_ARC_ICM, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = PERF_COUNT_ARC_LDC, + [C(RESULT_MISS)] = PERF_COUNT_ARC_EDTLB, + }, + /* DTLB LD/ST Miss not segregated by h/w*/ + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = PERF_COUNT_ARC_EITLB, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS, + [C(RESULT_MISS)] = PERF_COUNT_HW_BRANCH_MISSES, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + enum arc_pmu_attr_groups { ARCPMU_ATTR_GR_EVENTS, ARCPMU_ATTR_GR_FORMATS, @@ -328,7 +490,7 @@ static void arc_pmu_stop(struct perf_event *event, int flags) } if (!(event->hw.state & PERF_HES_STOPPED)) { - /* stop ARC pmu here */ + /* stop hw counter here */ write_aux_reg(ARC_REG_PCT_INDEX, idx); /* condition code #0 is always "never" */ @@ -361,7 +523,7 @@ static int arc_pmu_add(struct perf_event *event, int flags) { struct arc_pmu_cpu *pmu_cpu = this_cpu_ptr(&arc_pmu_cpu); struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; + int idx; idx = ffz(pmu_cpu->used_mask[0]); if (idx == arc_pmu->n_counters) diff --git a/arch/arc/kernel/stacktrace.c b/arch/arc/kernel/stacktrace.c index c376ff3147..5372dc04e7 100644 --- a/arch/arc/kernel/stacktrace.c +++ b/arch/arc/kernel/stacktrace.c @@ -15,7 +15,7 @@ * = specifics of data structs where trace is saved(CONFIG_STACKTRACE etc) * * vineetg: March 2009 - * -Implemented correct versions of thread_saved_pc() and get_wchan() + * -Implemented correct versions of thread_saved_pc() and __get_wchan() * * rajeshwarr: 2008 * -Initial implementation @@ -248,7 +248,7 @@ void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) * Of course just returning schedule( ) would be pointless so unwind until * the function is not in schedular code */ -unsigned int get_wchan(struct task_struct *tsk) +unsigned int __get_wchan(struct task_struct *tsk) { return arc_unwind_core(tsk, NULL, __get_first_nonsched, NULL); } diff --git a/arch/arc/kernel/unwind.c b/arch/arc/kernel/unwind.c index 9e28058cdb..200270a945 100644 --- a/arch/arc/kernel/unwind.c +++ b/arch/arc/kernel/unwind.c @@ -245,14 +245,9 @@ static void swap_eh_frame_hdr_table_entries(void *p1, void *p2, int size) { struct eh_frame_hdr_table_entry *e1 = p1; struct eh_frame_hdr_table_entry *e2 = p2; - unsigned long v; - v = e1->start; - e1->start = e2->start; - e2->start = v; - v = e1->fde; - e1->fde = e2->fde; - e2->fde = v; + swap(e1->start, e2->start); + swap(e1->fde, e2->fde); } static void init_unwind_hdr(struct unwind_table *table, diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c index 517988e60c..2a7fbbb83b 100644 --- a/arch/arc/mm/dma.c +++ b/arch/arc/mm/dma.c @@ -32,7 +32,7 @@ void arch_dma_prep_coherent(struct page *page, size_t size) /* * Cache operations depending on function and direction argument, inspired by - * https://lkml.org/lkml/2018/5/18/979 + * https://lore.kernel.org/lkml/20180518175004.GF17671@n2100.armlinux.org.uk * "dma_sync_*_for_cpu and direction=TO_DEVICE (was Re: [PATCH 02/20] * dma-mapping: provide a generic dma-noncoherent implementation)" * diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index 5787c261c9..dad27e4d69 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -149,8 +149,7 @@ void do_page_fault(unsigned long address, struct pt_regs *regs) /* * Fault retry nuances, mmap_lock already relinquished by core mm */ - if (unlikely((fault & VM_FAULT_RETRY) && - (flags & FAULT_FLAG_ALLOW_RETRY))) { + if (unlikely(fault & VM_FAULT_RETRY)) { flags |= FAULT_FLAG_TRIED; goto retry; } diff --git a/arch/arc/mm/init.c b/arch/arc/mm/init.c index 699ecf1196..ce4e939a7f 100644 --- a/arch/arc/mm/init.c +++ b/arch/arc/mm/init.c @@ -59,13 +59,13 @@ void __init early_init_dt_add_memory_arch(u64 base, u64 size) low_mem_sz = size; in_use = 1; - memblock_add_node(base, size, 0); + memblock_add_node(base, size, 0, MEMBLOCK_NONE); } else { #ifdef CONFIG_HIGHMEM high_mem_start = base; high_mem_sz = size; in_use = 1; - memblock_add_node(base, size, 1); + memblock_add_node(base, size, 1, MEMBLOCK_NONE); memblock_reserve(base, size); #endif } @@ -173,7 +173,7 @@ static void __init highmem_init(void) #ifdef CONFIG_HIGHMEM unsigned long tmp; - memblock_free(high_mem_start, high_mem_sz); + memblock_phys_free(high_mem_start, high_mem_sz); for (tmp = min_high_pfn; tmp < max_high_pfn; tmp++) free_highmem_page(pfn_to_page(tmp)); #endif diff --git a/arch/arc/plat-axs10x/axs10x.c b/arch/arc/plat-axs10x/axs10x.c index 63ea5a606e..b821df7b00 100644 --- a/arch/arc/plat-axs10x/axs10x.c +++ b/arch/arc/plat-axs10x/axs10x.c @@ -50,7 +50,7 @@ static void __init axs10x_enable_gpio_intc_wire(void) * Current implementation of "irq-dw-apb-ictl" driver doesn't work well * with stacked INTCs. In particular problem happens if its master INTC * not yet instantiated. See discussion here - - * https://lkml.org/lkml/2015/3/4/755 + * https://lore.kernel.org/lkml/54F6FE2C.7020309@synopsys.com * * So setup the first gpio block as a passive pass thru and hide it from * DT hardware topology - connect MB intc directly to cpu intc diff --git a/arch/arc/plat-hsdk/platform.c b/arch/arc/plat-hsdk/platform.c index b3ea1fa11f..c4a875b223 100644 --- a/arch/arc/plat-hsdk/platform.c +++ b/arch/arc/plat-hsdk/platform.c @@ -52,7 +52,7 @@ static void __init hsdk_enable_gpio_intc_wire(void) * Current implementation of "irq-dw-apb-ictl" driver doesn't work well * with stacked INTCs. In particular problem happens if its master INTC * not yet instantiated. See discussion here - - * https://lkml.org/lkml/2015/3/4/755 + * https://lore.kernel.org/lkml/54F6FE2C.7020309@synopsys.com * * So setup the first gpio block as a passive pass thru and hide it from * DT hardware topology - connect intc directly to cpu intc diff --git a/arch/arm64/Kbuild b/arch/arm64/Kbuild index ea7ab4ca81..5bfbf7d79c 100644 --- a/arch/arm64/Kbuild +++ b/arch/arm64/Kbuild @@ -4,3 +4,6 @@ obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_XEN) += xen/ obj-$(subst m,y,$(CONFIG_HYPERV)) += hyperv/ obj-$(CONFIG_CRYPTO) += crypto/ + +# for cleaning +subdir- += boot diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1a18c90457..c842878f81 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -11,6 +11,7 @@ config ARM64 select ACPI_PPTT if ACPI select ARCH_HAS_DEBUG_WX select ARCH_BINFMT_ELF_STATE + select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG select ARCH_ENABLE_MEMORY_HOTREMOVE @@ -119,7 +120,6 @@ config ARM64 select GENERIC_CPU_AUTOPROBE select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP - select GENERIC_FIND_FIRST_BIT select GENERIC_IDLE_POLL_SETUP select GENERIC_IRQ_IPI select GENERIC_IRQ_PROBE @@ -133,7 +133,6 @@ config ARM64 select GENERIC_TIME_VSYSCALL select GENERIC_GETTIMEOFDAY select GENERIC_VDSO_TIME_NS - select HANDLE_DOMAIN_IRQ select HARDIRQS_SW_RESEND select HAVE_MOVE_PMD select HAVE_MOVE_PUD @@ -150,11 +149,12 @@ config ARM64 select HAVE_ARCH_KASAN_VMALLOC if HAVE_ARCH_KASAN select HAVE_ARCH_KASAN_SW_TAGS if HAVE_ARCH_KASAN select HAVE_ARCH_KASAN_HW_TAGS if (HAVE_ARCH_KASAN && ARM64_MTE) + # Some instrumentation may be unsound, hence EXPERT + select HAVE_ARCH_KCSAN if EXPERT select HAVE_ARCH_KFENCE select HAVE_ARCH_KGDB select HAVE_ARCH_MMAP_RND_BITS select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT - select HAVE_ARCH_PFN_VALID select HAVE_ARCH_PREL32_RELOCATIONS select HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET select HAVE_ARCH_SECCOMP_FILTER @@ -186,14 +186,15 @@ config ARM64 select HAVE_GCC_PLUGINS select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_IRQ_TIME_ACCOUNTING + select HAVE_KVM select HAVE_NMI select HAVE_PATA_PLATFORM select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_REGS_AND_STACK_ACCESS_API + select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_FUNCTION_ARG_ACCESS_API - select HAVE_FUTEX_CMPXCHG if FUTEX select MMU_GATHER_RCU_TABLE_FREE select HAVE_RSEQ select HAVE_STACKPROTECTOR @@ -666,6 +667,203 @@ config ARM64_ERRATUM_1508412 If unsure, say Y. +config ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE + bool + +config ARM64_ERRATUM_2051678 + bool "Cortex-A510: 2051678: disable Hardware Update of the page table dirty bit" + default y + help + This options adds the workaround for ARM Cortex-A510 erratum ARM64_ERRATUM_2051678. + Affected Coretex-A510 might not respect the ordering rules for + hardware update of the page table's dirty bit. The workaround + is to not enable the feature on affected CPUs. + + If unsure, say Y. + +config ARM64_ERRATUM_2077057 + bool "Cortex-A510: 2077057: workaround software-step corrupting SPSR_EL2" + help + This option adds the workaround for ARM Cortex-A510 erratum 2077057. + Affected Cortex-A510 may corrupt SPSR_EL2 when the a step exception is + expected, but a Pointer Authentication trap is taken instead. The + erratum causes SPSR_EL1 to be copied to SPSR_EL2, which could allow + EL1 to cause a return to EL2 with a guest controlled ELR_EL2. + + This can only happen when EL2 is stepping EL1. + + When these conditions occur, the SPSR_EL2 value is unchanged from the + previous guest entry, and can be restored from the in-memory copy. + + If unsure, say Y. + +config ARM64_ERRATUM_2119858 + bool "Cortex-A710/X2: 2119858: workaround TRBE overwriting trace data in FILL mode" + default y + depends on CORESIGHT_TRBE + select ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE + help + This option adds the workaround for ARM Cortex-A710/X2 erratum 2119858. + + Affected Cortex-A710/X2 cores could overwrite up to 3 cache lines of trace + data at the base of the buffer (pointed to by TRBASER_EL1) in FILL mode in + the event of a WRAP event. + + Work around the issue by always making sure we move the TRBPTR_EL1 by + 256 bytes before enabling the buffer and filling the first 256 bytes of + the buffer with ETM ignore packets upon disabling. + + If unsure, say Y. + +config ARM64_ERRATUM_2139208 + bool "Neoverse-N2: 2139208: workaround TRBE overwriting trace data in FILL mode" + default y + depends on CORESIGHT_TRBE + select ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE + help + This option adds the workaround for ARM Neoverse-N2 erratum 2139208. + + Affected Neoverse-N2 cores could overwrite up to 3 cache lines of trace + data at the base of the buffer (pointed to by TRBASER_EL1) in FILL mode in + the event of a WRAP event. + + Work around the issue by always making sure we move the TRBPTR_EL1 by + 256 bytes before enabling the buffer and filling the first 256 bytes of + the buffer with ETM ignore packets upon disabling. + + If unsure, say Y. + +config ARM64_WORKAROUND_TSB_FLUSH_FAILURE + bool + +config ARM64_ERRATUM_2054223 + bool "Cortex-A710: 2054223: workaround TSB instruction failing to flush trace" + default y + select ARM64_WORKAROUND_TSB_FLUSH_FAILURE + help + Enable workaround for ARM Cortex-A710 erratum 2054223 + + Affected cores may fail to flush the trace data on a TSB instruction, when + the PE is in trace prohibited state. This will cause losing a few bytes + of the trace cached. + + Workaround is to issue two TSB consecutively on affected cores. + + If unsure, say Y. + +config ARM64_ERRATUM_2067961 + bool "Neoverse-N2: 2067961: workaround TSB instruction failing to flush trace" + default y + select ARM64_WORKAROUND_TSB_FLUSH_FAILURE + help + Enable workaround for ARM Neoverse-N2 erratum 2067961 + + Affected cores may fail to flush the trace data on a TSB instruction, when + the PE is in trace prohibited state. This will cause losing a few bytes + of the trace cached. + + Workaround is to issue two TSB consecutively on affected cores. + + If unsure, say Y. + +config ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE + bool + +config ARM64_ERRATUM_2253138 + bool "Neoverse-N2: 2253138: workaround TRBE writing to address out-of-range" + depends on CORESIGHT_TRBE + default y + select ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE + help + This option adds the workaround for ARM Neoverse-N2 erratum 2253138. + + Affected Neoverse-N2 cores might write to an out-of-range address, not reserved + for TRBE. Under some conditions, the TRBE might generate a write to the next + virtually addressed page following the last page of the TRBE address space + (i.e., the TRBLIMITR_EL1.LIMIT), instead of wrapping around to the base. + + Work around this in the driver by always making sure that there is a + page beyond the TRBLIMITR_EL1.LIMIT, within the space allowed for the TRBE. + + If unsure, say Y. + +config ARM64_ERRATUM_2224489 + bool "Cortex-A710/X2: 2224489: workaround TRBE writing to address out-of-range" + depends on CORESIGHT_TRBE + default y + select ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE + help + This option adds the workaround for ARM Cortex-A710/X2 erratum 2224489. + + Affected Cortex-A710/X2 cores might write to an out-of-range address, not reserved + for TRBE. Under some conditions, the TRBE might generate a write to the next + virtually addressed page following the last page of the TRBE address space + (i.e., the TRBLIMITR_EL1.LIMIT), instead of wrapping around to the base. + + Work around this in the driver by always making sure that there is a + page beyond the TRBLIMITR_EL1.LIMIT, within the space allowed for the TRBE. + + If unsure, say Y. + +config ARM64_ERRATUM_2064142 + bool "Cortex-A510: 2064142: workaround TRBE register writes while disabled" + depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in + default y + help + This option adds the workaround for ARM Cortex-A510 erratum 2064142. + + Affected Cortex-A510 core might fail to write into system registers after the + TRBE has been disabled. Under some conditions after the TRBE has been disabled + writes into TRBE registers TRBLIMITR_EL1, TRBPTR_EL1, TRBBASER_EL1, TRBSR_EL1, + and TRBTRG_EL1 will be ignored and will not be effected. + + Work around this in the driver by executing TSB CSYNC and DSB after collection + is stopped and before performing a system register write to one of the affected + registers. + + If unsure, say Y. + +config ARM64_ERRATUM_2038923 + bool "Cortex-A510: 2038923: workaround TRBE corruption with enable" + depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in + default y + help + This option adds the workaround for ARM Cortex-A510 erratum 2038923. + + Affected Cortex-A510 core might cause an inconsistent view on whether trace is + prohibited within the CPU. As a result, the trace buffer or trace buffer state + might be corrupted. This happens after TRBE buffer has been enabled by setting + TRBLIMITR_EL1.E, followed by just a single context synchronization event before + execution changes from a context, in which trace is prohibited to one where it + isn't, or vice versa. In these mentioned conditions, the view of whether trace + is prohibited is inconsistent between parts of the CPU, and the trace buffer or + the trace buffer state might be corrupted. + + Work around this in the driver by preventing an inconsistent view of whether the + trace is prohibited or not based on TRBLIMITR_EL1.E by immediately following a + change to TRBLIMITR_EL1.E with at least one ISB instruction before an ERET, or + two ISB instructions if no ERET is to take place. + + If unsure, say Y. + +config ARM64_ERRATUM_1902691 + bool "Cortex-A510: 1902691: workaround TRBE trace corruption" + depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in + default y + help + This option adds the workaround for ARM Cortex-A510 erratum 1902691. + + Affected Cortex-A510 core might cause trace data corruption, when being written + into the memory. Effectively TRBE is broken and hence cannot be used to capture + trace data. + + Work around this problem in the driver by just preventing TRBE initialization on + affected cpus. The firmware must have disabled the access to TRBE for the kernel + on such implementations. This will cover the kernel for any firmware that doesn't + do this already. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y @@ -989,6 +1187,15 @@ config SCHED_MC making when dealing with multi-core CPU chips at a cost of slightly increased overhead in some places. If unsure say N here. +config SCHED_CLUSTER + bool "Cluster scheduler support" + help + Cluster scheduler support improves the CPU scheduler's decision + making when dealing with machines that have clusters of CPUs. + Cluster usually means a couple of CPUs which are placed closely + by sharing mid-level caches, last-level cache tags or internal + busses. + config SCHED_SMT bool "SMT scheduler support" help @@ -1014,6 +1221,10 @@ config NUMA select GENERIC_ARCH_NUMA select ACPI_NUMA if ACPI select OF_NUMA + select HAVE_SETUP_PER_CPU_AREA + select NEED_PER_CPU_EMBED_FIRST_CHUNK + select NEED_PER_CPU_PAGE_FIRST_CHUNK + select USE_PERCPU_NUMA_NODE_ID help Enable NUMA (Non-Uniform Memory Access) support. @@ -1030,18 +1241,6 @@ config NODES_SHIFT Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables. -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - -config HAVE_SETUP_PER_CPU_AREA - def_bool y - depends on NUMA - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y - depends on NUMA - source "kernel/Kconfig.hz" config ARCH_SPARSEMEM_ENABLE @@ -1132,7 +1331,7 @@ config CRASH_DUMP config TRANS_TABLE def_bool y - depends on HIBERNATION + depends on HIBERNATION || KEXEC_CORE config XEN_DOM0 def_bool y @@ -1426,6 +1625,12 @@ endmenu menu "ARMv8.2 architectural features" +config AS_HAS_ARMV8_2 + def_bool $(cc-option,-Wa$(comma)-march=armv8.2-a) + +config AS_HAS_SHA3 + def_bool $(as-instr,.arch armv8.2-a+sha3) + config ARM64_PMEM bool "Enable support for persistent memory" select ARCH_HAS_PMEM_API diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms index b0ce18d4cc..21697449d7 100644 --- a/arch/arm64/Kconfig.platforms +++ b/arch/arm64/Kconfig.platforms @@ -44,7 +44,6 @@ config ARCH_BCM2835 select ARM_AMBA select ARM_GIC select ARM_TIMER_SP804 - select BRCMSTB_L2_IRQ help This enables support for the Broadcom BCM2837 and BCM2711 SoC. These SoCs are used in the Raspberry Pi 3 and 4 devices. @@ -82,8 +81,6 @@ config ARCH_BITMAIN config ARCH_BRCMSTB bool "Broadcom Set-Top-Box SoCs" select ARCH_HAS_RESET_CONTROLLER - select BCM7038_L1_IRQ - select BRCMSTB_L2_IRQ select GENERIC_IRQ_CHIP select PINCTRL help @@ -92,10 +89,9 @@ config ARCH_BRCMSTB config ARCH_EXYNOS bool "ARMv8 based Samsung Exynos SoC family" select COMMON_CLK_SAMSUNG - select EXYNOS_CHIPID + select CLKSRC_EXYNOS_MCT select EXYNOS_PM_DOMAINS if PM_GENERIC_DOMAINS select EXYNOS_PMU - select HAVE_S3C_RTC if RTC_CLASS select PINCTRL select PINCTRL_EXYNOS select PM_GENERIC_DOMAINS if PM @@ -166,8 +162,6 @@ config ARCH_MEDIATEK config ARCH_MESON bool "Amlogic Platforms" - select COMMON_CLK - select MESON_IRQ_GPIO help This enables support for the arm64 based Amlogic SoCs such as the s905, S905X/D, S912, A113X/D or S905X/D2 @@ -315,9 +309,6 @@ config ARCH_VISCONTI help This enables support for Toshiba Visconti SoCs Family. -config ARCH_VULCAN - def_bool n - config ARCH_XGENE bool "AppliedMicro X-Gene SOC Family" help diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index c744b1e7b3..2f1de88651 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -58,6 +58,11 @@ stack_protector_prepare: prepare0 include/generated/asm-offsets.h)) endif +ifeq ($(CONFIG_AS_HAS_ARMV8_2), y) +# make sure to pass the newest target architecture to -march. +asm-arch := armv8.2-a +endif + # Ensure that if the compiler supports branch protection we default it # off, this will be overridden if we are using branch protection. branch-prot-flags-y += $(call cc-option,-mbranch-protection=none) @@ -182,13 +187,6 @@ ifeq ($(CONFIG_ARM64_USE_LSE_ATOMICS),y) endif endif - -# We use MRPROPER_FILES and CLEAN_FILES now -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - $(Q)$(MAKE) $(clean)=arch/arm64/kernel/vdso - $(Q)$(MAKE) $(clean)=arch/arm64/kernel/vdso32 - ifeq ($(KBUILD_EXTMOD),) # We need to generate vdso-offsets.h before compiling certain files in kernel/. # In order to do that, we should use the archprepare target, but we can't since diff --git a/arch/arm64/boot/dts/allwinner/Makefile b/arch/arm64/boot/dts/allwinner/Makefile index a96d9d2d8d..8fa5c060a4 100644 --- a/arch/arm64/boot/dts/allwinner/Makefile +++ b/arch/arm64/boot/dts/allwinner/Makefile @@ -37,3 +37,4 @@ dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-orangepi-one-plus.dtb dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-pine-h64.dtb dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-pine-h64-model-b.dtb dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-tanix-tx6.dtb +dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h6-tanix-tx6-mini.dtb diff --git a/arch/arm64/boot/dts/allwinner/axp803.dtsi b/arch/arm64/boot/dts/allwinner/axp803.dtsi index 10e9186a76..578ef368e2 100644 --- a/arch/arm64/boot/dts/allwinner/axp803.dtsi +++ b/arch/arm64/boot/dts/allwinner/axp803.dtsi @@ -10,7 +10,7 @@ &axp803 { interrupt-controller; #interrupt-cells = <1>; - ac_power_supply: ac-power-supply { + ac_power_supply: ac-power { compatible = "x-powers,axp803-ac-power-supply", "x-powers,axp813-ac-power-supply"; status = "disabled"; @@ -26,18 +26,18 @@ axp_gpio: gpio { gpio-controller; #gpio-cells = <2>; - gpio0_ldo: gpio0-ldo { + gpio0_ldo: gpio0-ldo-pin { pins = "GPIO0"; function = "ldo"; }; - gpio1_ldo: gpio1-ldo { + gpio1_ldo: gpio1-ldo-pin { pins = "GPIO1"; function = "ldo"; }; }; - battery_power_supply: battery-power-supply { + battery_power_supply: battery-power { compatible = "x-powers,axp803-battery-power-supply", "x-powers,axp813-battery-power-supply"; status = "disabled"; @@ -147,7 +147,7 @@ reg_drivevbus: drivevbus { }; }; - usb_power_supply: usb-power-supply { + usb_power_supply: usb-power { compatible = "x-powers,axp803-usb-power-supply", "x-powers,axp813-usb-power-supply"; status = "disabled"; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-orangepi-win.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-orangepi-win.dts index 097a551152..c519d9fa69 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a64-orangepi-win.dts +++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-orangepi-win.dts @@ -343,7 +343,7 @@ &sound { &spi0 { status = "okay"; - spi-flash@0 { + flash@0 { compatible = "mxicy,mx25l1606e", "jedec,spi-nor"; reg = <0>; spi-max-frequency = <80000000>; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinebook.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinebook.dts index 34e67f5f82..63571df24d 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinebook.dts +++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinebook.dts @@ -15,6 +15,7 @@ / { model = "Pinebook"; compatible = "pine64,pinebook", "allwinner,sun50i-a64"; + chassis-type = "laptop"; aliases { serial0 = &uart0; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi index 5b44a979f2..87847116ab 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi @@ -12,6 +12,8 @@ #include / { + chassis-type = "handset"; + aliases { ethernet0 = &rtl8723cs; serial0 = &uart0; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinetab.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinetab.dts index 7ef96f9ff7..0a5607f730 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinetab.dts +++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinetab.dts @@ -16,6 +16,7 @@ / { model = "PineTab, Development Sample"; compatible = "pine64,pinetab", "allwinner,sun50i-a64"; + chassis-type = "tablet"; aliases { serial0 = &uart0; @@ -35,6 +36,17 @@ chosen { stdout-path = "serial0:115200n8"; }; + hdmi-connector { + compatible = "hdmi-connector"; + type = "c"; + + port { + hdmi_con_in: endpoint { + remote-endpoint = <&hdmi_out_con>; + }; + }; + }; + i2c-csi { compatible = "i2c-gpio"; sda-gpios = <&pio 4 13 GPIO_ACTIVE_HIGH>; /* PE13 */ @@ -77,7 +89,7 @@ speaker_amp: audio-amplifier { sound-name-prefix = "Speaker Amp"; }; - vdd_bl: regulator@0 { + vdd_bl: regulator { compatible = "regulator-fixed"; regulator-name = "bl-3v3"; regulator-min-microvolt = <3300000>; @@ -410,6 +422,21 @@ ®_rtc_ldo { regulator-name = "vcc-rtc"; }; +&simplefb_hdmi { + vcc-hdmi-supply = <®_dldo1>; +}; + +&hdmi { + hvcc-supply = <®_dldo1>; + status = "okay"; +}; + +&hdmi_out { + hdmi_out_con: endpoint { + remote-endpoint = <&hdmi_con_in>; + }; +}; + &sound { status = "okay"; simple-audio-card,aux-devs = <&codec_analog>, <&speaker_amp>; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-teres-i.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-teres-i.dts index 45e1abdf70..aff0660b89 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a64-teres-i.dts +++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-teres-i.dts @@ -14,6 +14,7 @@ / { model = "Olimex A64 Teres-I"; compatible = "olimex,a64-teres-i", "allwinner,sun50i-a64"; + chassis-type = "laptop"; aliases { serial0 = &uart0; @@ -139,6 +140,8 @@ ports { #size-cells = <0>; port@0 { + reg = <0>; + anx6345_in: endpoint { remote-endpoint = <&tcon0_out_anx6345>; }; @@ -206,7 +209,6 @@ axp803: pmic@3a3 { reg = <0x3a3>; interrupt-parent = <&r_intc>; interrupts = ; - wakeup-source; }; }; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a64.dtsi index 6ddb717f2f..de77c87481 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a64.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-a64.dtsi @@ -102,6 +102,22 @@ de: display-engine { status = "disabled"; }; + gpu_opp_table: opp-table-gpu { + compatible = "operating-points-v2"; + + opp-120000000 { + opp-hz = /bits/ 64 <120000000>; + }; + + opp-312000000 { + opp-hz = /bits/ 64 <312000000>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + }; + }; + osc24M: osc24M_clk { #clock-cells = <0>; compatible = "fixed-clock"; @@ -1103,6 +1119,7 @@ mali: gpu@1c40000 { clocks = <&ccu CLK_BUS_GPU>, <&ccu CLK_GPU>; clock-names = "bus", "core"; resets = <&ccu RST_BUS_GPU>; + operating-points-v2 = <&gpu_opp_table>; }; gic: interrupt-controller@1c81000 { @@ -1129,8 +1146,14 @@ pwm: pwm@1c21400 { mbus: dram-controller@1c62000 { compatible = "allwinner,sun50i-a64-mbus"; - reg = <0x01c62000 0x1000>; - clocks = <&ccu 112>; + reg = <0x01c62000 0x1000>, + <0x01c63000 0x1000>; + reg-names = "mbus", "dram"; + clocks = <&ccu CLK_MBUS>, + <&ccu CLK_DRAM>, + <&ccu CLK_BUS_DRAM>; + clock-names = "mbus", "dram", "bus"; + interrupts = ; #address-cells = <1>; #size-cells = <1>; dma-ranges = <0x00000000 0x40000000 0xc0000000>; @@ -1203,8 +1226,8 @@ hdmi: hdmi@1ee0000 { reg-io-width = <1>; interrupts = ; clocks = <&ccu CLK_BUS_HDMI>, <&ccu CLK_HDMI_DDC>, - <&ccu CLK_HDMI>; - clock-names = "iahb", "isfr", "tmds"; + <&ccu CLK_HDMI>, <&rtc 0>; + clock-names = "iahb", "isfr", "tmds", "cec"; resets = <&ccu RST_BUS_HDMI1>; reset-names = "ctrl"; phys = <&hdmi_phy>; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-r1s-h5.dts b/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-r1s-h5.dts index 55bcdf8d1a..55b369534a 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-r1s-h5.dts +++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-r1s-h5.dts @@ -142,9 +142,16 @@ &i2c0 { status = "okay"; eeprom@51 { - compatible = "microchip,24c02"; + compatible = "microchip,24c02", "atmel,24c02"; reg = <0x51>; pagesize = <16>; + read-only; + #address-cells = <1>; + #size-cells = <1>; + + eth_mac1: mac-address@fa { + reg = <0xfa 0x06>; + }; }; }; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi index 9988e87ea7..a56fae761a 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi @@ -233,6 +233,10 @@ &display_clocks { compatible = "allwinner,sun50i-h5-de2-clk"; }; +&mbus { + compatible = "allwinner,sun50i-h5-mbus"; +}; + &mmc0 { compatible = "allwinner,sun50i-h5-mmc", "allwinner,sun50i-a64-mmc"; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-tanix-tx6.dts b/arch/arm64/boot/dts/allwinner/sun50i-h6-tanix-tx6.dts index 8f2a80f128..9a38ff9b3f 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-h6-tanix-tx6.dts +++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-tanix-tx6.dts @@ -3,145 +3,27 @@ /dts-v1/; -#include "sun50i-h6.dtsi" -#include "sun50i-h6-cpu-opp.dtsi" - -#include +#include "sun50i-h6-tanix.dtsi" / { model = "Tanix TX6"; compatible = "oranth,tanix-tx6", "allwinner,sun50i-h6"; - - aliases { - serial0 = &uart0; - }; - - chosen { - stdout-path = "serial0:115200n8"; - }; - - connector { - compatible = "hdmi-connector"; - ddc-en-gpios = <&pio 7 2 GPIO_ACTIVE_HIGH>; /* PH2 */ - type = "a"; - - port { - hdmi_con_in: endpoint { - remote-endpoint = <&hdmi_out_con>; - }; - }; - }; - - reg_vcc1v8: regulator-vcc1v8 { - compatible = "regulator-fixed"; - regulator-name = "vcc1v8"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - }; - - reg_vcc3v3: regulator-vcc3v3 { - compatible = "regulator-fixed"; - regulator-name = "vcc3v3"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - }; - - reg_vdd_cpu_gpu: regulator-vdd-cpu-gpu { - compatible = "regulator-fixed"; - regulator-name = "vdd-cpu-gpu"; - regulator-min-microvolt = <1135000>; - regulator-max-microvolt = <1135000>; - }; -}; - -&cpu0 { - cpu-supply = <®_vdd_cpu_gpu>; -}; - -&de { - status = "okay"; -}; - -&dwc3 { - status = "okay"; -}; - -&ehci0 { - status = "okay"; -}; - -&ehci3 { - status = "okay"; -}; - -&gpu { - mali-supply = <®_vdd_cpu_gpu>; - status = "okay"; -}; - -&hdmi { - status = "okay"; -}; - -&hdmi_out { - hdmi_out_con: endpoint { - remote-endpoint = <&hdmi_con_in>; - }; -}; - -&mmc0 { - pinctrl-names = "default"; - pinctrl-0 = <&mmc0_pins>; - vmmc-supply = <®_vcc3v3>; - cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>; - bus-width = <4>; - status = "okay"; -}; - -&mmc2 { - vmmc-supply = <®_vcc3v3>; - vqmmc-supply = <®_vcc1v8>; - bus-width = <8>; - non-removable; - cap-mmc-hw-reset; - mmc-hs200-1_8v; - status = "okay"; -}; - -&ohci0 { - status = "okay"; -}; - -&ohci3 { - status = "okay"; -}; - -&pio { - vcc-pc-supply = <®_vcc1v8>; - vcc-pd-supply = <®_vcc3v3>; - vcc-pg-supply = <®_vcc1v8>; }; &r_ir { linux,rc-map-name = "rc-tanix-tx5max"; - status = "okay"; }; -&uart0 { +&uart1 { pinctrl-names = "default"; - pinctrl-0 = <&uart0_ph_pins>; + pinctrl-0 = <&uart1_pins>, <&uart1_rts_cts_pins>; + uart-has-rtscts; status = "okay"; -}; -&usb2otg { - dr_mode = "host"; - status = "okay"; -}; - -&usb2phy { - status = "okay"; -}; - -&usb3phy { - status = "okay"; + bluetooth { + compatible = "realtek,rtl8822cs-bt"; + device-wake-gpios = <&r_pio 1 2 GPIO_ACTIVE_HIGH>; /* PM2 */ + host-wake-gpios = <&r_pio 1 1 GPIO_ACTIVE_HIGH>; /* PM1 */ + enable-gpios = <&r_pio 1 4 GPIO_ACTIVE_HIGH>; /* PM4 */ + }; }; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi index 30d396e8c7..fbe94abbb1 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi @@ -119,10 +119,10 @@ bus@1000000 { display_clocks: clock@0 { compatible = "allwinner,sun50i-h6-de3-clk"; reg = <0x0 0x10000>; - clocks = <&ccu CLK_DE>, - <&ccu CLK_BUS_DE>; - clock-names = "mod", - "bus"; + clocks = <&ccu CLK_BUS_DE>, + <&ccu CLK_DE>; + clock-names = "bus", + "mod"; resets = <&ccu RST_BUS_DE>; #clock-cells = <1>; #reset-cells = <1>; @@ -153,6 +153,15 @@ mixer0_out_tcon_top_mixer0: endpoint { }; }; + video-codec-g2@1c00000 { + compatible = "allwinner,sun50i-h6-vpu-g2"; + reg = <0x01c00000 0x1000>; + interrupts = ; + clocks = <&ccu CLK_BUS_VP9>, <&ccu CLK_VP9>; + clock-names = "bus", "mod"; + resets = <&ccu RST_BUS_VP9>; + }; + video-codec@1c0e000 { compatible = "allwinner,sun50i-h6-video-engine"; reg = <0x01c0e000 0x2000>; diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi index d301ac0d40..3ec301bd08 100644 --- a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi +++ b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi @@ -594,7 +594,7 @@ emac0-tx-ecc@ff8c0400 { }; qspi: spi@ff8d2000 { - compatible = "cdns,qspi-nor"; + compatible = "intel,socfpga-qspi", "cdns,qspi-nor"; #address-cells = <1>; #size-cells = <0>; reg = <0xff8d2000 0x100>, diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_nand.dts b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_nand.dts index f9b4a39683..bbc3db42d6 100644 --- a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_nand.dts +++ b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk_nand.dts @@ -119,28 +119,8 @@ partition@0 { }; partition@200000 { - label = "env"; - reg = <0x200000 0x40000>; - }; - - partition@240000 { - label = "dtb"; - reg = <0x240000 0x40000>; - }; - - partition@280000 { - label = "kernel"; - reg = <0x280000 0x2000000>; - }; - - partition@2280000 { - label = "misc"; - reg = <0x2280000 0x2000000>; - }; - - partition@4280000 { - label = "rootfs"; - reg = <0x4280000 0x3bd80000>; + label = "root"; + reg = <0x200000 0x3fe00000>; }; }; }; diff --git a/arch/arm64/boot/dts/amlogic/Makefile b/arch/arm64/boot/dts/amlogic/Makefile index faa0a79a34..5148cd9e51 100644 --- a/arch/arm64/boot/dts/amlogic/Makefile +++ b/arch/arm64/boot/dts/amlogic/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_MESON) += meson-axg-s400.dtb +dtb-$(CONFIG_ARCH_MESON) += meson-axg-jethome-jethub-j100.dtb +dtb-$(CONFIG_ARCH_MESON) += meson-g12a-radxa-zero.dtb dtb-$(CONFIG_ARCH_MESON) += meson-g12a-sei510.dtb dtb-$(CONFIG_ARCH_MESON) += meson-g12a-u200.dtb dtb-$(CONFIG_ARCH_MESON) += meson-g12a-x96-max.dtb @@ -38,6 +40,7 @@ dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s805x-p241.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905w-p281.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905w-tx3-mini.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905d-libretech-pc.dtb +dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905w-jethome-jethub-j80.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxm-khadas-vim2.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxm-mecool-kiii-pro.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxm-minix-neo-u9h.dtb diff --git a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi index a3a1ea0f21..f84d4b489e 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi @@ -165,61 +165,6 @@ &clkc CLKID_PCIE_COMB status = "disabled"; }; - thermal-zones { - cpu_thermal: cpu-thermal { - polling-delay = <1000>; - polling-delay-passive = <100>; - thermal-sensors = <&cpu_temp>; - - trips { - cpu_passive: cpu-passive { - temperature = <85000>; /* millicelsius */ - hysteresis = <2000>; /* millicelsius */ - type = "passive"; - }; - - cpu_hot: cpu-hot { - temperature = <95000>; /* millicelsius */ - hysteresis = <2000>; /* millicelsius */ - type = "hot"; - }; - - cpu_critical: cpu-critical { - temperature = <110000>; /* millicelsius */ - hysteresis = <2000>; /* millicelsius */ - type = "critical"; - }; - }; - }; - - ddr_thermal: ddr-thermal { - polling-delay = <1000>; - polling-delay-passive = <100>; - thermal-sensors = <&ddr_temp>; - - trips { - ddr_passive: ddr-passive { - temperature = <85000>; /* millicelsius */ - hysteresis = <2000>; /* millicelsius */ - type = "passive"; - }; - - ddr_critical: ddr-critical { - temperature = <110000>; /* millicelsius */ - hysteresis = <2000>; /* millicelsius */ - type = "critical"; - }; - }; - - cooling-maps { - map { - trip = <&ddr_passive>; - cooling-device = <&mali THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; - }; - }; - ethmac: ethernet@ff3f0000 { compatible = "amlogic,meson-g12a-dwmac", "snps,dwmac-3.70a", @@ -2421,6 +2366,61 @@ mali: gpu@ffe40000 { }; }; + thermal-zones { + cpu_thermal: cpu-thermal { + polling-delay = <1000>; + polling-delay-passive = <100>; + thermal-sensors = <&cpu_temp>; + + trips { + cpu_passive: cpu-passive { + temperature = <85000>; /* millicelsius */ + hysteresis = <2000>; /* millicelsius */ + type = "passive"; + }; + + cpu_hot: cpu-hot { + temperature = <95000>; /* millicelsius */ + hysteresis = <2000>; /* millicelsius */ + type = "hot"; + }; + + cpu_critical: cpu-critical { + temperature = <110000>; /* millicelsius */ + hysteresis = <2000>; /* millicelsius */ + type = "critical"; + }; + }; + }; + + ddr_thermal: ddr-thermal { + polling-delay = <1000>; + polling-delay-passive = <100>; + thermal-sensors = <&ddr_temp>; + + trips { + ddr_passive: ddr-passive { + temperature = <85000>; /* millicelsius */ + hysteresis = <2000>; /* millicelsius */ + type = "passive"; + }; + + ddr_critical: ddr-critical { + temperature = <110000>; /* millicelsius */ + hysteresis = <2000>; /* millicelsius */ + type = "critical"; + }; + }; + + cooling-maps { + map { + trip = <&ddr_passive>; + cooling-device = <&mali THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + }; + timer { compatible = "arm,armv8-timer"; interrupts = ; regulator-always-on; vin-supply = <&main_12v>; + gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>; + enable-active-high; }; vcc_1v8: regulator-vcc_1v8 { diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-wetek.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb-wetek.dtsi index a4d34398da..94dafb9553 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxbb-wetek.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-wetek.dtsi @@ -7,6 +7,7 @@ #include "meson-gxbb.dtsi" #include +#include / { aliases { @@ -26,8 +27,10 @@ memory@0 { leds { compatible = "gpio-leds"; - led-system { - label = "wetek-play:system-status"; + led-power { + /* red in suspend or power-off */ + color = ; + function = LED_FUNCTION_POWER; gpios = <&gpio_ao GPIOAO_13 GPIO_ACTIVE_HIGH>; default-state = "on"; panic-indicator; diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s805x-p241.dts b/arch/arm64/boot/dts/amlogic/meson-gxl-s805x-p241.dts index eb7f5a3fef..ff906becd2 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxl-s805x-p241.dts +++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s805x-p241.dts @@ -8,6 +8,7 @@ /dts-v1/; #include +#include #include "meson-gxl-s805x.dtsi" @@ -21,6 +22,13 @@ aliases { ethernet0 = ðmac; }; + au2: analog-amplifier { + compatible = "simple-audio-amplifier"; + sound-name-prefix = "AU2"; + VCC-supply = <&vcc_5v>; + enable-gpios = <&gpio GPIOH_5 GPIO_ACTIVE_HIGH>; + }; + chosen { stdout-path = "serial0:115200n8"; }; @@ -84,6 +92,14 @@ vcc_3v3: regulator-vcc_3v3 { regulator-max-microvolt = <3300000>; }; + vcc_5v: regulator-vcc-5v { + compatible = "regulator-fixed"; + regulator-name = "VCC_5V"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + emmc_pwrseq: emmc-pwrseq { compatible = "mmc-pwrseq-emmc"; reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>; @@ -102,6 +118,68 @@ sdio_pwrseq: sdio-pwrseq { clocks = <&wifi32k>; clock-names = "ext_clock"; }; + + sound { + compatible = "amlogic,gx-sound-card"; + model = "GXL-P241"; + audio-aux-devs = <&au2>; + audio-widgets = "Line", "Lineout"; + audio-routing = "AU2 INL", "ACODEC LOLN", + "AU2 INR", "ACODEC LORN", + "Lineout", "AU2 OUTL", + "Lineout", "AU2 OUTR"; + assigned-clocks = <&clkc CLKID_MPLL0>, + <&clkc CLKID_MPLL1>, + <&clkc CLKID_MPLL2>; + assigned-clock-parents = <0>, <0>, <0>; + assigned-clock-rates = <294912000>, + <270950400>, + <393216000>; + status = "okay"; + + dai-link-0 { + sound-dai = <&aiu AIU_CPU CPU_I2S_FIFO>; + }; + + dai-link-1 { + sound-dai = <&aiu AIU_CPU CPU_I2S_ENCODER>; + dai-format = "i2s"; + mclk-fs = <256>; + + codec-0 { + sound-dai = <&aiu AIU_HDMI CTRL_I2S>; + }; + + codec-1 { + sound-dai = <&aiu AIU_ACODEC CTRL_I2S>; + }; + }; + + dai-link-2 { + sound-dai = <&aiu AIU_HDMI CTRL_OUT>; + + codec-0 { + sound-dai = <&hdmi_tx>; + }; + }; + + dai-link-3 { + sound-dai = <&aiu AIU_ACODEC CTRL_OUT>; + + codec-0 { + sound-dai = <&acodec>; + }; + }; + }; +}; + +&acodec { + AVDD-supply = <&vddio_ao18>; + status = "okay"; +}; + +&aiu { + status = "okay"; }; &cec_AO { @@ -136,6 +214,7 @@ &hdmi_tx { status = "okay"; pinctrl-0 = <&hdmi_hpd_pins>, <&hdmi_i2c_pins>; pinctrl-names = "default"; + hdmi-supply = <&vcc_5v>; }; &hdmi_tx_tmds_port { @@ -220,3 +299,7 @@ &usb { status = "okay"; dr_mode = "host"; }; + +&usb2_phy0 { + phy-supply = <&vcc_5v>; +}; diff --git a/arch/arm64/boot/dts/amlogic/meson-gxm-rbox-pro.dts b/arch/arm64/boot/dts/amlogic/meson-gxm-rbox-pro.dts index dde7cfe12c..50137aafab 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxm-rbox-pro.dts +++ b/arch/arm64/boot/dts/amlogic/meson-gxm-rbox-pro.dts @@ -14,6 +14,7 @@ /dts-v1/; #include "meson-gxm.dtsi" +#include / { compatible = "kingnovel,r-box-pro", "amlogic,s912", "amlogic,meson-gxm"; @@ -33,6 +34,13 @@ memory@0 { reg = <0x0 0x0 0x0 0x80000000>; /* 2 GiB or 3 GiB */ }; + spdif_dit: audio-codec-0 { + #sound-dai-cells = <0>; + compatible = "linux,spdif-dit"; + status = "okay"; + sound-name-prefix = "DIT"; + }; + leds { compatible = "gpio-leds"; @@ -90,6 +98,59 @@ sdio_pwrseq: sdio-pwrseq { clocks = <&wifi32k>; clock-names = "ext_clock"; }; + + sound { + compatible = "amlogic,gx-sound-card"; + model = "RBOX-PRO"; + assigned-clocks = <&clkc CLKID_MPLL0>, + <&clkc CLKID_MPLL1>, + <&clkc CLKID_MPLL2>; + assigned-clock-parents = <0>, <0>, <0>; + assigned-clock-rates = <294912000>, + <270950400>, + <393216000>; + status = "okay"; + + dai-link-0 { + sound-dai = <&aiu AIU_CPU CPU_I2S_FIFO>; + }; + + dai-link-1 { + sound-dai = <&aiu AIU_CPU CPU_SPDIF_FIFO>; + }; + + dai-link-2 { + sound-dai = <&aiu AIU_CPU CPU_I2S_ENCODER>; + dai-format = "i2s"; + mclk-fs = <256>; + + codec-0 { + sound-dai = <&aiu AIU_HDMI CTRL_I2S>; + }; + }; + + dai-link-3 { + sound-dai = <&aiu AIU_CPU CPU_SPDIF_ENCODER>; + + codec-0 { + sound-dai = <&spdif_dit>; + }; + }; + + dai-link-4 { + sound-dai = <&aiu AIU_HDMI CTRL_OUT>; + + codec-0 { + sound-dai = <&hdmi_tx>; + }; + }; + }; +}; + +&aiu { + status = "okay"; + pinctrl-0 = <&spdif_out_h_pins>; + pinctrl-names = "default"; }; ðmac { diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi index 76ad052fbf..ddb1b34539 100644 --- a/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-sm1-odroid.dtsi @@ -225,6 +225,20 @@ &arb { status = "okay"; }; +&cec_AO { + pinctrl-0 = <&cec_ao_a_h_pins>; + pinctrl-names = "default"; + status = "disabled"; + hdmi-phandle = <&hdmi_tx>; +}; + +&cecb_AO { + pinctrl-0 = <&cec_ao_b_h_pins>; + pinctrl-names = "default"; + status = "okay"; + hdmi-phandle = <&hdmi_tx>; +}; + &clkc_audio { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index cbbd701ebf..c0510c25ca 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -1,2 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_APPLE) += t8103-j274.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8103-j293.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8103-j313.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8103-j456.dtb +dtb-$(CONFIG_ARCH_APPLE) += t8103-j457.dtb diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index e0f6775b98..2cd429efba 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -10,36 +10,36 @@ /dts-v1/; #include "t8103.dtsi" +#include "t8103-jxxx.dtsi" / { compatible = "apple,j274", "apple,t8103", "apple,arm-platform"; model = "Apple Mac mini (M1, 2020)"; aliases { - serial0 = &serial0; - }; - - chosen { - #address-cells = <2>; - #size-cells = <2>; - ranges; - - stdout-path = "serial0"; - - framebuffer0: framebuffer@0 { - compatible = "apple,simple-framebuffer", "simple-framebuffer"; - reg = <0 0 0 0>; /* To be filled by loader */ - /* Format properties will be added by loader */ - status = "disabled"; - }; - }; - - memory@800000000 { - device_type = "memory"; - reg = <0x8 0 0x2 0>; /* To be filled by loader */ + ethernet0 = ðernet0; }; }; -&serial0 { +/* + * Force the bus number assignments so that we can declare some of the + * on-board devices and properties that are populated by the bootloader + * (such as MAC addresses). + */ + +&port01 { + bus-range = <2 2>; +}; + +&port02 { + bus-range = <3 3>; + ethernet0: ethernet@0,0 { + reg = <0x30000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; +}; + +&i2c2 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index a1e22a2ea2..19afbc9102 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -7,8 +7,10 @@ * Copyright The Asahi Linux Contributors */ +#include #include #include +#include / { compatible = "apple,t8103", "apple,arm-platform"; @@ -95,11 +97,11 @@ timer { ; }; - clk24: clock-24m { + clkref: clock-ref { compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <24000000>; - clock-output-names = "clk24"; + clock-output-names = "clkref"; }; soc { @@ -110,6 +112,73 @@ soc { ranges; nonposted-mmio; + i2c0: i2c@235010000 { + compatible = "apple,t8103-i2c", "apple,i2c"; + reg = <0x2 0x35010000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c0>; + }; + + i2c1: i2c@235014000 { + compatible = "apple,t8103-i2c", "apple,i2c"; + reg = <0x2 0x35014000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c1_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c1>; + }; + + i2c2: i2c@235018000 { + compatible = "apple,t8103-i2c", "apple,i2c"; + reg = <0x2 0x35018000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c2_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; /* not used in all devices */ + power-domains = <&ps_i2c2>; + }; + + i2c3: i2c@23501c000 { + compatible = "apple,t8103-i2c", "apple,i2c"; + reg = <0x2 0x3501c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c3_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c3>; + }; + + i2c4: i2c@235020000 { + compatible = "apple,t8103-i2c", "apple,i2c"; + reg = <0x2 0x35020000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c4_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c4>; + status = "disabled"; /* only used in J293 */ + }; + serial0: serial@235200000 { compatible = "apple,s5l-uart"; reg = <0x2 0x35200000 0x0 0x1000>; @@ -120,8 +189,21 @@ serial0: serial@235200000 { * TODO: figure out the clocking properly, there may * be a third selectable clock. */ - clocks = <&clk24>, <&clk24>; + clocks = <&clkref>, <&clkref>; clock-names = "uart", "clk_uart_baud0"; + power-domains = <&ps_uart0>; + status = "disabled"; + }; + + serial2: serial@235208000 { + compatible = "apple,s5l-uart"; + reg = <0x2 0x35208000 0x0 0x1000>; + reg-io-width = <4>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>, <&clkref>; + clock-names = "uart", "clk_uart_baud0"; + power-domains = <&ps_uart2>; status = "disabled"; }; @@ -130,6 +212,269 @@ aic: interrupt-controller@23b100000 { #interrupt-cells = <3>; interrupt-controller; reg = <0x2 0x3b100000 0x0 0x8000>; + power-domains = <&ps_aic>; + }; + + pmgr: power-management@23b700000 { + compatible = "apple,t8103-pmgr", "apple,pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x3b700000 0 0x14000>; + }; + + pinctrl_ap: pinctrl@23c100000 { + compatible = "apple,t8103-pinctrl", "apple,pinctrl"; + reg = <0x2 0x3c100000 0x0 0x100000>; + power-domains = <&ps_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 212>; + apple,npins = <212>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + + i2c0_pins: i2c0-pins { + pinmux = , + ; + }; + + i2c1_pins: i2c1-pins { + pinmux = , + ; + }; + + i2c2_pins: i2c2-pins { + pinmux = , + ; + }; + + i2c3_pins: i2c3-pins { + pinmux = , + ; + }; + + i2c4_pins: i2c4-pins { + pinmux = , + ; + }; + + pcie_pins: pcie-pins { + pinmux = , + , + ; + }; + }; + + pinctrl_nub: pinctrl@23d1f0000 { + compatible = "apple,t8103-pinctrl", "apple,pinctrl"; + reg = <0x2 0x3d1f0000 0x0 0x4000>; + power-domains = <&ps_nub_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_nub 0 0 23>; + apple,npins = <23>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pmgr_mini: power-management@23d280000 { + compatible = "apple,t8103-pmgr", "apple,pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x3d280000 0 0x4000>; + }; + + wdt: watchdog@23d2b0000 { + compatible = "apple,t8103-wdt", "apple,wdt"; + reg = <0x2 0x3d2b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + pinctrl_smc: pinctrl@23e820000 { + compatible = "apple,t8103-pinctrl", "apple,pinctrl"; + reg = <0x2 0x3e820000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_smc 0 0 16>; + apple,npins = <16>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pinctrl_aop: pinctrl@24a820000 { + compatible = "apple,t8103-pinctrl", "apple,pinctrl"; + reg = <0x2 0x4a820000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_aop 0 0 42>; + apple,npins = <42>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pcie0_dart_0: dart@681008000 { + compatible = "apple,t8103-dart"; + reg = <0x6 0x81008000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + }; + + pcie0_dart_1: dart@682008000 { + compatible = "apple,t8103-dart"; + reg = <0x6 0x82008000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + }; + + pcie0_dart_2: dart@683008000 { + compatible = "apple,t8103-dart"; + reg = <0x6 0x83008000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + }; + + pcie0: pcie@690000000 { + compatible = "apple,t8103-pcie", "apple,pcie"; + device_type = "pci"; + + reg = <0x6 0x90000000 0x0 0x1000000>, + <0x6 0x80000000 0x0 0x100000>, + <0x6 0x81000000 0x0 0x4000>, + <0x6 0x82000000 0x0 0x4000>, + <0x6 0x83000000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "port1", "port2"; + + interrupt-parent = <&aic>; + interrupts = , + , + ; + + msi-controller; + msi-parent = <&pcie0>; + msi-ranges = <&aic AIC_IRQ 704 IRQ_TYPE_EDGE_RISING 32>; + + + iommu-map = <0x100 &pcie0_dart_0 1 1>, + <0x200 &pcie0_dart_1 1 1>, + <0x300 &pcie0_dart_2 1 1>; + iommu-map-mask = <0xff00>; + + bus-range = <0 3>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x6 0xa0000000 0x6 0xa0000000 0x0 0x20000000>, + <0x02000000 0x0 0xc0000000 0x6 0xc0000000 0x0 0x40000000>; + + power-domains = <&ps_apcie_gp>; + pinctrl-0 = <&pcie_pins>; + pinctrl-names = "default"; + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 152 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + }; + + port01: pci@1,0 { + device_type = "pci"; + reg = <0x800 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 153 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; + }; + + port02: pci@2,0 { + device_type = "pci"; + reg = <0x1000 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 33 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port02 0 0 0 0>, + <0 0 0 2 &port02 0 0 0 1>, + <0 0 0 3 &port02 0 0 0 2>, + <0 0 0 4 &port02 0 0 0 3>; + }; }; }; }; + +#include "t8103-pmgr.dtsi" diff --git a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi index fefd2b5f01..be42932f7e 100644 --- a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi +++ b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi @@ -160,64 +160,75 @@ v2m_sysctl: sysctl@20000 { apbregs@10000 { compatible = "syscon", "simple-mfd"; reg = <0x010000 0x1000>; + ranges = <0x0 0x10000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; - led0 { + led@8,0 { compatible = "register-bit-led"; + reg = <0x08 0x04>; offset = <0x08>; mask = <0x01>; label = "vexpress:0"; linux,default-trigger = "heartbeat"; default-state = "on"; }; - led1 { + led@8,1 { compatible = "register-bit-led"; + reg = <0x08 0x04>; offset = <0x08>; mask = <0x02>; label = "vexpress:1"; linux,default-trigger = "mmc0"; default-state = "off"; }; - led2 { + led@8,2 { compatible = "register-bit-led"; + reg = <0x08 0x04>; offset = <0x08>; mask = <0x04>; label = "vexpress:2"; linux,default-trigger = "cpu0"; default-state = "off"; }; - led3 { + led@8,3 { compatible = "register-bit-led"; + reg = <0x08 0x04>; offset = <0x08>; mask = <0x08>; label = "vexpress:3"; linux,default-trigger = "cpu1"; default-state = "off"; }; - led4 { + led@8,4 { compatible = "register-bit-led"; + reg = <0x08 0x04>; offset = <0x08>; mask = <0x10>; label = "vexpress:4"; linux,default-trigger = "cpu2"; default-state = "off"; }; - led5 { + led@8,5 { compatible = "register-bit-led"; + reg = <0x08 0x04>; offset = <0x08>; mask = <0x20>; label = "vexpress:5"; linux,default-trigger = "cpu3"; default-state = "off"; }; - led6 { + led@8,6 { compatible = "register-bit-led"; + reg = <0x08 0x04>; offset = <0x08>; mask = <0x40>; label = "vexpress:6"; default-state = "off"; }; - led7 { + led@8,7 { compatible = "register-bit-led"; + reg = <0x08 0x04>; offset = <0x08>; mask = <0x80>; label = "vexpress:7"; diff --git a/arch/arm64/boot/dts/broadcom/Makefile b/arch/arm64/boot/dts/broadcom/Makefile index 8eadc747ae..5f782c2cba 100644 --- a/arch/arm64/boot/dts/broadcom/Makefile +++ b/arch/arm64/boot/dts/broadcom/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_BCM2835) += bcm2711-rpi-400.dtb \ bcm2711-rpi-4-b.dtb \ + bcm2711-rpi-cm4-io.dtb \ bcm2837-rpi-3-a-plus.dtb \ bcm2837-rpi-3-b.dtb \ bcm2837-rpi-3-b-plus.dtb \ diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/Makefile b/arch/arm64/boot/dts/broadcom/bcm4908/Makefile index cc75854519..6e364e304d 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/Makefile +++ b/arch/arm64/boot/dts/broadcom/bcm4908/Makefile @@ -2,3 +2,4 @@ dtb-$(CONFIG_ARCH_BCM4908) += bcm4906-netgear-r8000p.dtb dtb-$(CONFIG_ARCH_BCM4908) += bcm4906-tplink-archer-c2300-v1.dtb dtb-$(CONFIG_ARCH_BCM4908) += bcm4908-asus-gt-ac5300.dtb +dtb-$(CONFIG_ARCH_BCM4908) += bcm4908-netgear-raxe500.dtb diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi index 5118816b1e..984c737fa6 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi @@ -296,7 +296,7 @@ uart0: serial@640 { status = "okay"; }; - nand@1800 { + nand-controller@1800 { #address-cells = <1>; #size-cells = <0>; compatible = "brcm,nand-bcm63138", "brcm,brcmnand-v7.1", "brcm,brcmnand"; diff --git a/arch/arm64/boot/dts/exynos/Makefile b/arch/arm64/boot/dts/exynos/Makefile index e0a2facde6..b41e86df0a 100644 --- a/arch/arm64/boot/dts/exynos/Makefile +++ b/arch/arm64/boot/dts/exynos/Makefile @@ -2,4 +2,5 @@ dtb-$(CONFIG_ARCH_EXYNOS) += \ exynos5433-tm2.dtb \ exynos5433-tm2e.dtb \ - exynos7-espresso.dtb + exynos7-espresso.dtb \ + exynosautov9-sadk.dtb diff --git a/arch/arm64/boot/dts/exynos/exynos5433-bus.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-bus.dtsi index 8997f8f2b9..72ccf18eb9 100644 --- a/arch/arm64/boot/dts/exynos/exynos5433-bus.dtsi +++ b/arch/arm64/boot/dts/exynos/exynos5433-bus.dtsi @@ -87,7 +87,7 @@ bus_noc2: bus9 { status = "disabled"; }; - bus_g2d_400_opp_table: opp-table2 { + bus_g2d_400_opp_table: opp-table-2 { compatible = "operating-points-v2"; opp-shared; @@ -117,7 +117,7 @@ opp-100000000 { }; }; - bus_g2d_266_opp_table: opp-table3 { + bus_g2d_266_opp_table: opp-table-3 { compatible = "operating-points-v2"; opp-267000000 { @@ -137,7 +137,7 @@ opp-100000000 { }; }; - bus_gscl_opp_table: opp-table4 { + bus_gscl_opp_table: opp-table-4 { compatible = "operating-points-v2"; opp-333000000 { @@ -151,7 +151,7 @@ opp-166500000 { }; }; - bus_hevc_opp_table: opp-table5 { + bus_hevc_opp_table: opp-table-5 { compatible = "operating-points-v2"; opp-shared; @@ -175,7 +175,7 @@ opp-100000000 { }; }; - bus_noc2_opp_table: opp-table6 { + bus_noc2_opp_table: opp-table-6 { compatible = "operating-points-v2"; opp-400000000 { diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts b/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts index fdd0796b29..aca01709fd 100644 --- a/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts +++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts @@ -13,6 +13,7 @@ / { model = "Samsung TM2 board"; compatible = "samsung,tm2", "samsung,exynos5433"; + chassis-type = "handset"; }; &cmu_disp { diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts b/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts index 089fc7a1af..22d26460f3 100644 --- a/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts +++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts @@ -13,6 +13,7 @@ / { model = "Samsung TM2E board"; compatible = "samsung,tm2e", "samsung,exynos5433"; + chassis-type = "handset"; }; &cmu_disp { diff --git a/arch/arm64/boot/dts/exynos/exynos5433.dtsi b/arch/arm64/boot/dts/exynos/exynos5433.dtsi index 6a6f7dd1d6..bfe4ed8a23 100644 --- a/arch/arm64/boot/dts/exynos/exynos5433.dtsi +++ b/arch/arm64/boot/dts/exynos/exynos5433.dtsi @@ -239,7 +239,7 @@ cluster_a53_l2: l2-cache1 { }; }; - cluster_a53_opp_table: opp-table0 { + cluster_a53_opp_table: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -285,7 +285,7 @@ opp-1300000000 { }; }; - cluster_a57_opp_table: opp-table1 { + cluster_a57_opp_table: opp-table-1 { compatible = "operating-points-v2"; opp-shared; @@ -1132,7 +1132,7 @@ syscon_cam1: syscon@145f0000 { }; syscon_fsys: syscon@156f0000 { - compatible = "syscon"; + compatible = "samsung,exynos5433-sysreg", "syscon"; reg = <0x156f0000 0x1044>; }; @@ -1585,7 +1585,7 @@ pwm: pwm@14dd0000 { status = "disabled"; }; - hsi2c_0: hsi2c@14e40000 { + hsi2c_0: i2c@14e40000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14e40000 0x1000>; interrupts = ; @@ -1598,7 +1598,7 @@ hsi2c_0: hsi2c@14e40000 { status = "disabled"; }; - hsi2c_1: hsi2c@14e50000 { + hsi2c_1: i2c@14e50000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14e50000 0x1000>; interrupts = ; @@ -1611,7 +1611,7 @@ hsi2c_1: hsi2c@14e50000 { status = "disabled"; }; - hsi2c_2: hsi2c@14e60000 { + hsi2c_2: i2c@14e60000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14e60000 0x1000>; interrupts = ; @@ -1624,7 +1624,7 @@ hsi2c_2: hsi2c@14e60000 { status = "disabled"; }; - hsi2c_3: hsi2c@14e70000 { + hsi2c_3: i2c@14e70000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14e70000 0x1000>; interrupts = ; @@ -1637,7 +1637,7 @@ hsi2c_3: hsi2c@14e70000 { status = "disabled"; }; - hsi2c_4: hsi2c@14ec0000 { + hsi2c_4: i2c@14ec0000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14ec0000 0x1000>; interrupts = ; @@ -1650,7 +1650,7 @@ hsi2c_4: hsi2c@14ec0000 { status = "disabled"; }; - hsi2c_5: hsi2c@14ed0000 { + hsi2c_5: i2c@14ed0000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14ed0000 0x1000>; interrupts = ; @@ -1663,7 +1663,7 @@ hsi2c_5: hsi2c@14ed0000 { status = "disabled"; }; - hsi2c_6: hsi2c@14ee0000 { + hsi2c_6: i2c@14ee0000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14ee0000 0x1000>; interrupts = ; @@ -1676,7 +1676,7 @@ hsi2c_6: hsi2c@14ee0000 { status = "disabled"; }; - hsi2c_7: hsi2c@14ef0000 { + hsi2c_7: i2c@14ef0000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14ef0000 0x1000>; interrupts = ; @@ -1689,7 +1689,7 @@ hsi2c_7: hsi2c@14ef0000 { status = "disabled"; }; - hsi2c_8: hsi2c@14d90000 { + hsi2c_8: i2c@14d90000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14d90000 0x1000>; interrupts = ; @@ -1702,7 +1702,7 @@ hsi2c_8: hsi2c@14d90000 { status = "disabled"; }; - hsi2c_9: hsi2c@14da0000 { + hsi2c_9: i2c@14da0000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14da0000 0x1000>; interrupts = ; @@ -1715,7 +1715,7 @@ hsi2c_9: hsi2c@14da0000 { status = "disabled"; }; - hsi2c_10: hsi2c@14de0000 { + hsi2c_10: i2c@14de0000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14de0000 0x1000>; interrupts = ; @@ -1728,7 +1728,7 @@ hsi2c_10: hsi2c@14de0000 { status = "disabled"; }; - hsi2c_11: hsi2c@14df0000 { + hsi2c_11: i2c@14df0000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14df0000 0x1000>; interrupts = ; diff --git a/arch/arm64/boot/dts/exynos/exynos7.dtsi b/arch/arm64/boot/dts/exynos/exynos7.dtsi index c73a597ca6..c3efbc8add 100644 --- a/arch/arm64/boot/dts/exynos/exynos7.dtsi +++ b/arch/arm64/boot/dts/exynos/exynos7.dtsi @@ -359,7 +359,7 @@ pinctrl_bus1: pinctrl@14870000 { interrupts = ; }; - hsi2c_0: hsi2c@13640000 { + hsi2c_0: i2c@13640000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x13640000 0x1000>; interrupts = ; @@ -372,7 +372,7 @@ hsi2c_0: hsi2c@13640000 { status = "disabled"; }; - hsi2c_1: hsi2c@13650000 { + hsi2c_1: i2c@13650000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x13650000 0x1000>; interrupts = ; @@ -385,7 +385,7 @@ hsi2c_1: hsi2c@13650000 { status = "disabled"; }; - hsi2c_2: hsi2c@14e60000 { + hsi2c_2: i2c@14e60000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14e60000 0x1000>; interrupts = ; @@ -398,7 +398,7 @@ hsi2c_2: hsi2c@14e60000 { status = "disabled"; }; - hsi2c_3: hsi2c@14e70000 { + hsi2c_3: i2c@14e70000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14e70000 0x1000>; interrupts = ; @@ -411,7 +411,7 @@ hsi2c_3: hsi2c@14e70000 { status = "disabled"; }; - hsi2c_4: hsi2c@13660000 { + hsi2c_4: i2c@13660000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x13660000 0x1000>; interrupts = ; @@ -424,7 +424,7 @@ hsi2c_4: hsi2c@13660000 { status = "disabled"; }; - hsi2c_5: hsi2c@13670000 { + hsi2c_5: i2c@13670000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x13670000 0x1000>; interrupts = ; @@ -437,7 +437,7 @@ hsi2c_5: hsi2c@13670000 { status = "disabled"; }; - hsi2c_6: hsi2c@14e00000 { + hsi2c_6: i2c@14e00000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14e00000 0x1000>; interrupts = ; @@ -450,7 +450,7 @@ hsi2c_6: hsi2c@14e00000 { status = "disabled"; }; - hsi2c_7: hsi2c@13e10000 { + hsi2c_7: i2c@13e10000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x13e10000 0x1000>; interrupts = ; @@ -463,7 +463,7 @@ hsi2c_7: hsi2c@13e10000 { status = "disabled"; }; - hsi2c_8: hsi2c@14e20000 { + hsi2c_8: i2c@14e20000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x14e20000 0x1000>; interrupts = ; @@ -476,7 +476,7 @@ hsi2c_8: hsi2c@14e20000 { status = "disabled"; }; - hsi2c_9: hsi2c@13680000 { + hsi2c_9: i2c@13680000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x13680000 0x1000>; interrupts = ; @@ -489,7 +489,7 @@ hsi2c_9: hsi2c@13680000 { status = "disabled"; }; - hsi2c_10: hsi2c@13690000 { + hsi2c_10: i2c@13690000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x13690000 0x1000>; interrupts = ; @@ -502,7 +502,7 @@ hsi2c_10: hsi2c@13690000 { status = "disabled"; }; - hsi2c_11: hsi2c@136a0000 { + hsi2c_11: i2c@136a0000 { compatible = "samsung,exynos7-hsi2c"; reg = <0x136a0000 0x1000>; interrupts = ; diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile index db9e36ebe9..6d8f0a5325 100644 --- a/arch/arm64/boot/dts/freescale/Makefile +++ b/arch/arm64/boot/dts/freescale/Makefile @@ -1,4 +1,14 @@ # SPDX-License-Identifier: GPL-2.0 + +# required for overlay support +DTC_FLAGS_fsl-ls1028a-qds := -@ +DTC_FLAGS_fsl-ls1028a-qds-13bb := -@ +DTC_FLAGS_fsl-ls1028a-qds-65bb := -@ +DTC_FLAGS_fsl-ls1028a-qds-7777 := -@ +DTC_FLAGS_fsl-ls1028a-qds-85bb := -@ +DTC_FLAGS_fsl-ls1028a-qds-899b := -@ +DTC_FLAGS_fsl-ls1028a-qds-9999 := -@ + dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1012a-frdm.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1012a-frwy.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1012a-oxalis.dtb @@ -11,6 +21,12 @@ dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-kontron-sl28-var2.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-kontron-sl28-var3-ads2.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-kontron-sl28-var4.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds.dtb +dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-13bb.dtb +dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-65bb.dtb +dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-7777.dtb +dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-85bb.dtb +dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-899b.dtb +dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-9999.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-rdb.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1043a-qds.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1043a-rdb.dtb @@ -25,6 +41,8 @@ dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-rdb.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-simu.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2088a-qds.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2088a-rdb.dtb +dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-lx2160a-bluebox3.dtb +dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-lx2160a-bluebox3-rev-a.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-lx2160a-clearfog-cx.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-lx2160a-honeycomb.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-lx2160a-qds.dtb @@ -38,6 +56,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-icore-mx8mm-ctouch2.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-icore-mx8mm-edimm2.2.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-kontron-n801x-s.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-nitrogen-r2.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8mm-tqma8mqml-mba8mx.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-var-som-symphony.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw71xx-0x.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw72xx-0x.dtb @@ -45,8 +64,11 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw73xx-0x.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7901.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7902.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mn-beacon-kit.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8mn-bsh-smm-s2.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8mn-bsh-smm-s2pro.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mn-evk.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mn-ddr4-evk.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8mn-tqma8mqnl-mba8mx.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mn-var-som-symphony.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mn-venice-gw7902.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk.dtb @@ -58,6 +80,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mq-librem5-devkit.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mq-librem5-r2.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mq-librem5-r3.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mq-librem5-r4.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8mq-tqma8mq-mba8mx.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mq-mnt-reform2.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mq-nitrogen.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mq-phanbell.dtb @@ -69,5 +92,8 @@ dtb-$(CONFIG_ARCH_MXC) += imx8qm-mek.dtb dtb-$(CONFIG_ARCH_MXC) += imx8qxp-ai_ml.dtb dtb-$(CONFIG_ARCH_MXC) += imx8qxp-colibri-eval-v3.dtb dtb-$(CONFIG_ARCH_MXC) += imx8qxp-mek.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8ulp-evk.dtb +dtb-$(CONFIG_ARCH_S32) += s32g274a-evb.dtb +dtb-$(CONFIG_ARCH_S32) += s32g274a-rdb2.dtb dtb-$(CONFIG_ARCH_S32) += s32v234-evb.dtb diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var1.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var1.dts index e8d31279b7..7cd29ab970 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var1.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var1.dts @@ -8,7 +8,7 @@ * None of the four SerDes lanes are used by the module, instead they are * all led out to the carrier for customer use. * - * Copyright (C) 2020 Michael Walle + * Copyright (C) 2021 Michael Walle * */ @@ -21,43 +21,39 @@ / { compatible = "kontron,sl28-var1", "kontron,sl28", "fsl,ls1028a"; }; +&enetc_mdio_pf3 { + /* Delete unused phy node */ + /delete-node/ ethernet-phy@5; + + phy0: ethernet-phy@4 { + reg = <0x4>; + eee-broken-1000t; + eee-broken-100tx; + qca,clk-out-frequency = <125000000>; + qca,clk-out-strength = ; + qca,keep-pll-enabled; + vddio-supply = <&vddio>; + + vddio: vddio-regulator { + regulator-name = "VDDIO"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + vddh: vddh-regulator { + regulator-name = "VDDH"; + }; + }; +}; + &enetc_port0 { status = "disabled"; - /* - * Delete both the phy-handle to the old phy0 label as well as - * the mdio node with the old phy node with the old phy0 label. - */ + /* Delete the phy-handle to the old phy0 label */ /delete-property/ phy-handle; - /delete-node/ mdio; }; &enetc_port1 { phy-handle = <&phy0>; - phy-connection-type = "rgmii-id"; + phy-mode = "rgmii-id"; status = "okay"; - - mdio { - #address-cells = <1>; - #size-cells = <0>; - - phy0: ethernet-phy@4 { - reg = <0x4>; - eee-broken-1000t; - eee-broken-100tx; - qca,clk-out-frequency = <125000000>; - qca,clk-out-strength = ; - qca,keep-pll-enabled; - vddio-supply = <&vddio>; - - vddio: vddio-regulator { - regulator-name = "VDDIO"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - }; - - vddh: vddh-regulator { - regulator-name = "VDDH"; - }; - }; - }; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts index f6a79c8080..330e34f933 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts @@ -5,7 +5,7 @@ * This is for the network variant 2 which has two ethernet ports. These * ports are connected to the internal switch. * - * Copyright (C) 2020 Michael Walle + * Copyright (C) 2021 Michael Walle * */ @@ -18,12 +18,6 @@ / { }; &enetc_mdio_pf3 { - phy0: ethernet-phy@5 { - reg = <0x5>; - eee-broken-1000t; - eee-broken-100tx; - }; - phy1: ethernet-phy@4 { reg = <0x4>; eee-broken-1000t; @@ -34,14 +28,11 @@ phy1: ethernet-phy@4 { &enetc_port0 { status = "disabled"; /* - * In the base device tree the PHY was registered in the mdio - * subnode as it is PHY for this port. On this module this PHY - * is connected to a switch port instead and registered above. - * Therefore, delete the mdio subnode as well as the phy-handle - * property here. + * In the base device tree the PHY at address 5 was assigned for + * this port. On this module this PHY is connected to a switch + * port instead. Therefore, delete the phy-handle property here. */ /delete-property/ phy-handle; - /delete-node/ mdio; }; &enetc_port2 { diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var4.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var4.dts index e65d1c477e..9b5e92fb75 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var4.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var4.dts @@ -5,7 +5,7 @@ * This is for the network variant 4 which has two ethernet ports. It * extends the base and provides one more port connected via RGMII. * - * Copyright (C) 2019 Michael Walle + * Copyright (C) 2021 Michael Walle * */ @@ -18,33 +18,30 @@ / { compatible = "kontron,sl28-var4", "kontron,sl28", "fsl,ls1028a"; }; -&enetc_port1 { - phy-handle = <&phy1>; - phy-connection-type = "rgmii-id"; - status = "okay"; +&enetc_mdio_pf3 { + phy1: ethernet-phy@4 { + reg = <0x4>; + eee-broken-1000t; + eee-broken-100tx; + qca,clk-out-frequency = <125000000>; + qca,clk-out-strength = ; + qca,keep-pll-enabled; + vddio-supply = <&vddio>; - mdio { - #address-cells = <1>; - #size-cells = <0>; + vddio: vddio-regulator { + regulator-name = "VDDIO"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; - phy1: ethernet-phy@4 { - reg = <0x4>; - eee-broken-1000t; - eee-broken-100tx; - qca,clk-out-frequency = <125000000>; - qca,clk-out-strength = ; - qca,keep-pll-enabled; - vddio-supply = <&vddio>; - - vddio: vddio-regulator { - regulator-name = "VDDIO"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - }; - - vddh: vddh-regulator { - regulator-name = "VDDH"; - }; + vddh: vddh-regulator { + regulator-name = "VDDH"; }; }; }; + +&enetc_port1 { + phy-handle = <&phy1>; + phy-mode = "rgmii-id"; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28.dts index a92ecb331c..c03f4e1833 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28.dts @@ -2,7 +2,7 @@ /* * Device Tree file for the Kontron SMARC-sAL28 board. * - * Copyright (C) 2019 Michael Walle + * Copyright (C) 2021 Michael Walle * */ @@ -80,22 +80,19 @@ &duart1 { status = "okay"; }; +&enetc_mdio_pf3 { + phy0: ethernet-phy@5 { + reg = <0x5>; + eee-broken-1000t; + eee-broken-100tx; + }; +}; + &enetc_port0 { phy-handle = <&phy0>; - phy-connection-type = "sgmii"; + phy-mode = "sgmii"; managed = "in-band-status"; status = "okay"; - - mdio { - #address-cells = <1>; - #size-cells = <0>; - - phy0: ethernet-phy@5 { - reg = <0x5>; - eee-broken-1000t; - eee-broken-100tx; - }; - }; }; &esdhc { @@ -160,6 +157,10 @@ partition@3e0000 { }; }; +&ftm_alarm0 { + status = "okay"; +}; + &gpio1 { gpio-line-names = "", "", "", "", "", "", "", "", @@ -309,3 +310,11 @@ eeprom@50 { &lpuart1 { status = "okay"; }; + +&usb0 { + status = "okay"; +}; + +&usb1 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds.dts index 2f92e62eca..177bc1405f 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds.dts @@ -25,7 +25,7 @@ aliases { serial1 = &duart1; mmc0 = &esdhc; mmc1 = &esdhc1; - rtc1 = &ftm_alarm0; + rtc1 = &ftm_alarm1; }; chosen { @@ -211,6 +211,16 @@ &duart1 { status = "okay"; }; +&enetc_port1 { + phy-handle = <&qds_phy1>; + phy-mode = "rgmii-id"; + status = "okay"; +}; + +&enetc_port2 { + status = "okay"; +}; + &esdhc { status = "okay"; }; @@ -234,6 +244,10 @@ mt35xu02g0: flash@0 { }; }; +&ftm_alarm1 { + status = "okay"; +}; + &i2c0 { status = "okay"; @@ -322,13 +336,16 @@ rtc@51 { }; }; -&enetc_port1 { - phy-handle = <&qds_phy1>; - phy-connection-type = "rgmii-id"; +&lpuart0 { status = "okay"; }; -&lpuart0 { +&lpuart1 { + status = "okay"; +}; + +&mscc_felix_port4 { + ethernet = <&enetc_port2>; status = "okay"; }; @@ -339,3 +356,11 @@ &sai1 { &sata { status = "okay"; }; + +&usb0 { + status = "okay"; +}; + +&usb1 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts index d7b5272725..68c31cb8ee 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts @@ -2,7 +2,7 @@ /* * Device Tree file for NXP LS1028A RDB Board. * - * Copyright 2018 NXP + * Copyright 2018-2021 NXP * * Harninder Rai * @@ -21,7 +21,14 @@ aliases { serial1 = &duart1; mmc0 = &esdhc; mmc1 = &esdhc1; - rtc1 = &ftm_alarm0; + rtc1 = &ftm_alarm1; + spi0 = &fspi; + ethernet0 = &enetc_port0; + ethernet1 = &enetc_port2; + ethernet2 = &mscc_felix_port0; + ethernet3 = &mscc_felix_port1; + ethernet4 = &mscc_felix_port2; + ethernet5 = &mscc_felix_port3; }; chosen { @@ -102,6 +109,48 @@ can-transceiver { }; }; +&duart0 { + status = "okay"; +}; + +&duart1 { + status = "okay"; +}; + +&enetc_mdio_pf3 { + sgmii_phy0: ethernet-phy@2 { + reg = <0x2>; + }; + + /* VSC8514 QSGMII quad PHY */ + qsgmii_phy0: ethernet-phy@10 { + reg = <0x10>; + }; + + qsgmii_phy1: ethernet-phy@11 { + reg = <0x11>; + }; + + qsgmii_phy2: ethernet-phy@12 { + reg = <0x12>; + }; + + qsgmii_phy3: ethernet-phy@13 { + reg = <0x13>; + }; +}; + +&enetc_port0 { + phy-handle = <&sgmii_phy0>; + phy-mode = "sgmii"; + managed = "in-band-status"; + status = "okay"; +}; + +&enetc_port2 { + status = "okay"; +}; + &esdhc { sd-uhs-sdr104; sd-uhs-sdr50; @@ -132,6 +181,10 @@ mt35xu02g0: flash@0 { }; }; +&ftm_alarm1 { + status = "okay"; +}; + &i2c0 { status = "okay"; @@ -188,52 +241,6 @@ rtc@51 { }; }; -&duart0 { - status = "okay"; -}; - -&duart1 { - status = "okay"; -}; - -&enetc_mdio_pf3 { - /* VSC8514 QSGMII quad PHY */ - qsgmii_phy0: ethernet-phy@10 { - reg = <0x10>; - }; - - qsgmii_phy1: ethernet-phy@11 { - reg = <0x11>; - }; - - qsgmii_phy2: ethernet-phy@12 { - reg = <0x12>; - }; - - qsgmii_phy3: ethernet-phy@13 { - reg = <0x13>; - }; -}; - -&enetc_port0 { - phy-handle = <&sgmii_phy0>; - phy-connection-type = "sgmii"; - managed = "in-band-status"; - status = "okay"; - - mdio { - #address-cells = <1>; - #size-cells = <0>; - sgmii_phy0: ethernet-phy@2 { - reg = <0x2>; - }; - }; -}; - -&enetc_port2 { - status = "okay"; -}; - &mscc_felix { status = "okay"; }; @@ -279,6 +286,10 @@ &optee { status = "okay"; }; +&pwm0 { + status = "okay"; +}; + &sai4 { status = "okay"; }; @@ -287,6 +298,11 @@ &sata { status = "okay"; }; +&usb0 { + status = "okay"; +}; + &usb1 { dr_mode = "otg"; + status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi index 06b36cc658..5bb8c26e08 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi @@ -66,6 +66,13 @@ CPU_PW20: cpu-pw20 { }; }; + rtc_clk: rtc-clk { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "rtc_clk"; + }; + sysclk: sysclk { compatible = "fixed-clock"; #clock-cells = <0>; @@ -80,13 +87,6 @@ osc_27m: clock-osc-27m { clock-output-names = "phy_27m"; }; - dpclk: clock-controller@f1f0000 { - compatible = "fsl,ls1028a-plldig"; - reg = <0x0 0xf1f0000 0x0 0xffff>; - #clock-cells = <0>; - clocks = <&osc_27m>; - }; - firmware { optee: optee { compatible = "linaro,optee-tz"; @@ -592,6 +592,7 @@ usb0: usb@3100000 { snps,dis_rxdet_inp3_quirk; snps,quirk-frame-length-adjustment = <0x20>; snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + status = "disabled"; }; usb1: usb@3110000 { @@ -602,6 +603,7 @@ usb1: usb@3110000 { snps,dis_rxdet_inp3_quirk; snps,quirk-frame-length-adjustment = <0x20>; snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + status = "disabled"; }; sata: sata@3200000 { @@ -642,6 +644,18 @@ pcie1: pcie@3400000 { status = "disabled"; }; + pcie_ep1: pcie-ep@3400000 { + compatible = "fsl,ls1028a-pcie-ep","fsl,ls-pcie-ep"; + reg = <0x00 0x03400000 0x0 0x00100000 + 0x80 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + interrupts = ; /* PME interrupt */ + interrupt-names = "pme"; + num-ib-windows = <6>; + num-ob-windows = <8>; + status = "disabled"; + }; + pcie2: pcie@3500000 { compatible = "fsl,ls1028a-pcie"; reg = <0x00 0x03500000 0x0 0x00100000>, /* controller registers */ @@ -669,6 +683,18 @@ pcie2: pcie@3500000 { status = "disabled"; }; + pcie_ep2: pcie-ep@3500000 { + compatible = "fsl,ls1028a-pcie-ep","fsl,ls-pcie-ep"; + reg = <0x00 0x03500000 0x0 0x00100000 + 0x88 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + interrupts = ; /* PME interrupt */ + interrupt-names = "pme"; + num-ib-windows = <6>; + num-ob-windows = <8>; + status = "disabled"; + }; + smmu: iommu@5000000 { compatible = "arm,mmu-500"; reg = <0 0x5000000 0 0x800000>; @@ -800,6 +826,38 @@ QORIQ_CLK_PLL_DIV(16)>, clock-names = "wdog_clk", "apb_pclk"; }; + malidp0: display@f080000 { + compatible = "arm,mali-dp500"; + reg = <0x0 0xf080000 0x0 0x10000>; + interrupts = <0 222 IRQ_TYPE_LEVEL_HIGH>, + <0 223 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "DE", "SE"; + clocks = <&dpclk>, + <&clockgen QORIQ_CLK_HWACCEL 2>, + <&clockgen QORIQ_CLK_HWACCEL 2>, + <&clockgen QORIQ_CLK_HWACCEL 2>; + clock-names = "pxlclk", "mclk", "aclk", "pclk"; + arm,malidp-output-port-lines = /bits/ 8 <8 8 8>; + arm,malidp-arqos-value = <0xd000d000>; + + port { + dpi0_out: endpoint { + + }; + }; + }; + + gpu: gpu@f0c0000 { + compatible = "vivante,gc"; + reg = <0x0 0xf0c0000 0x0 0x10000>; + interrupts = ; + clocks = <&clockgen QORIQ_CLK_HWACCEL 2>, + <&clockgen QORIQ_CLK_HWACCEL 2>, + <&clockgen QORIQ_CLK_HWACCEL 2>; + clock-names = "core", "shader", "bus"; + #cooling-cells = <2>; + }; + sai1: audio-controller@f100000 { #sound-dai-cells = <0>; compatible = "fsl,vf610-sai"; @@ -926,6 +984,13 @@ QORIQ_CLK_PLL_DIV(2)>, status = "disabled"; }; + dpclk: clock-controller@f1f0000 { + compatible = "fsl,ls1028a-plldig"; + reg = <0x0 0xf1f0000 0x0 0x10000>; + #clock-cells = <0>; + clocks = <&osc_27m>; + }; + tmu: tmu@1f80000 { compatible = "fsl,qoriq-tmu"; reg = <0x0 0x1f80000 0x0 0x10000>; @@ -1025,6 +1090,7 @@ enetc_port2: ethernet@0,2 { fixed-link { speed = <2500>; full-duplex; + pause; }; }; @@ -1049,7 +1115,7 @@ mscc_felix: ethernet-switch@0,5 { interrupts = ; status = "disabled"; - ports { + mscc_felix_ports: ports { #address-cells = <1>; #size-cells = <0>; @@ -1083,6 +1149,7 @@ mscc_felix_port4: port@4 { fixed-link { speed = <2500>; full-duplex; + pause; }; }; @@ -1094,6 +1161,7 @@ mscc_felix_port5: port@5 { fixed-link { speed = <1000>; full-duplex; + pause; }; }; }; @@ -1108,6 +1176,7 @@ enetc_port3: ethernet@0,6 { fixed-link { speed = <1000>; full-duplex; + pause; }; }; @@ -1124,6 +1193,94 @@ ierb@1f0800000 { reg = <0x01 0xf0800000 0x0 0x10000>; }; + pwm0: pwm@2800000 { + compatible = "fsl,vf610-ftm-pwm"; + #pwm-cells = <3>; + reg = <0x0 0x2800000 0x0 0x10000>; + clock-names = "ftm_sys", "ftm_ext", + "ftm_fix", "ftm_cnt_clk_en"; + clocks = <&clockgen 4 1>, <&clockgen 4 1>, + <&rtc_clk>, <&clockgen 4 1>; + status = "disabled"; + }; + + pwm1: pwm@2810000 { + compatible = "fsl,vf610-ftm-pwm"; + #pwm-cells = <3>; + reg = <0x0 0x2810000 0x0 0x10000>; + clock-names = "ftm_sys", "ftm_ext", + "ftm_fix", "ftm_cnt_clk_en"; + clocks = <&clockgen 4 1>, <&clockgen 4 1>, + <&rtc_clk>, <&clockgen 4 1>; + status = "disabled"; + }; + + pwm2: pwm@2820000 { + compatible = "fsl,vf610-ftm-pwm"; + #pwm-cells = <3>; + reg = <0x0 0x2820000 0x0 0x10000>; + clock-names = "ftm_sys", "ftm_ext", + "ftm_fix", "ftm_cnt_clk_en"; + clocks = <&clockgen 4 1>, <&clockgen 4 1>, + <&rtc_clk>, <&clockgen 4 1>; + status = "disabled"; + }; + + pwm3: pwm@2830000 { + compatible = "fsl,vf610-ftm-pwm"; + #pwm-cells = <3>; + reg = <0x0 0x2830000 0x0 0x10000>; + clock-names = "ftm_sys", "ftm_ext", + "ftm_fix", "ftm_cnt_clk_en"; + clocks = <&clockgen 4 1>, <&clockgen 4 1>, + <&rtc_clk>, <&clockgen 4 1>; + status = "disabled"; + }; + + pwm4: pwm@2840000 { + compatible = "fsl,vf610-ftm-pwm"; + #pwm-cells = <3>; + reg = <0x0 0x2840000 0x0 0x10000>; + clock-names = "ftm_sys", "ftm_ext", + "ftm_fix", "ftm_cnt_clk_en"; + clocks = <&clockgen 4 1>, <&clockgen 4 1>, + <&rtc_clk>, <&clockgen 4 1>; + status = "disabled"; + }; + + pwm5: pwm@2850000 { + compatible = "fsl,vf610-ftm-pwm"; + #pwm-cells = <3>; + reg = <0x0 0x2850000 0x0 0x10000>; + clock-names = "ftm_sys", "ftm_ext", + "ftm_fix", "ftm_cnt_clk_en"; + clocks = <&clockgen 4 1>, <&clockgen 4 1>, + <&rtc_clk>, <&clockgen 4 1>; + status = "disabled"; + }; + + pwm6: pwm@2860000 { + compatible = "fsl,vf610-ftm-pwm"; + #pwm-cells = <3>; + reg = <0x0 0x2860000 0x0 0x10000>; + clock-names = "ftm_sys", "ftm_ext", + "ftm_fix", "ftm_cnt_clk_en"; + clocks = <&clockgen 4 1>, <&clockgen 4 1>, + <&rtc_clk>, <&clockgen 4 1>; + status = "disabled"; + }; + + pwm7: pwm@2870000 { + compatible = "fsl,vf610-ftm-pwm"; + #pwm-cells = <3>; + reg = <0x0 0x2870000 0x0 0x10000>; + clock-names = "ftm_sys", "ftm_ext", + "ftm_fix", "ftm_cnt_clk_en"; + clocks = <&clockgen 4 1>, <&clockgen 4 1>, + <&rtc_clk>, <&clockgen 4 1>; + status = "disabled"; + }; + rcpm: power-controller@1e34040 { compatible = "fsl,ls1028a-rcpm", "fsl,qoriq-rcpm-2.1+"; reg = <0x0 0x1e34040 0x0 0x1c>; @@ -1136,27 +1293,16 @@ ftm_alarm0: timer@2800000 { reg = <0x0 0x2800000 0x0 0x10000>; fsl,rcpm-wakeup = <&rcpm 0x0 0x0 0x0 0x0 0x4000 0x0 0x0>; interrupts = ; + status = "disabled"; + }; + + ftm_alarm1: timer@2810000 { + compatible = "fsl,ls1028a-ftm-alarm"; + reg = <0x0 0x2810000 0x0 0x10000>; + fsl,rcpm-wakeup = <&rcpm 0x0 0x0 0x0 0x0 0x4000 0x0 0x0>; + interrupts = ; + status = "disabled"; }; }; - malidp0: display@f080000 { - compatible = "arm,mali-dp500"; - reg = <0x0 0xf080000 0x0 0x10000>; - interrupts = <0 222 IRQ_TYPE_LEVEL_HIGH>, - <0 223 IRQ_TYPE_LEVEL_HIGH>; - interrupt-names = "DE", "SE"; - clocks = <&dpclk>, - <&clockgen QORIQ_CLK_HWACCEL 2>, - <&clockgen QORIQ_CLK_HWACCEL 2>, - <&clockgen QORIQ_CLK_HWACCEL 2>; - clock-names = "pxlclk", "mclk", "aclk", "pclk"; - arm,malidp-output-port-lines = /bits/ 8 <8 8 8>; - arm,malidp-arqos-value = <0xd000d000>; - - port { - dp0_out: endpoint { - - }; - }; - }; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts index 3516af4726..b290605e92 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts @@ -94,6 +94,8 @@ flash@0 { compatible = "n25q128a13", "jedec,spi-nor"; /* 16MB */ reg = <0>; spi-max-frequency = <1000000>; /* input clock */ + fsl,spi-cs-sck-delay = <100>; + fsl,spi-sck-cs-delay = <100>; }; slic@2 { diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi index 6050723172..18e5291184 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi @@ -206,6 +206,13 @@ sysclk: sysclk { clock-output-names = "sysclk"; }; + reboot { + compatible = "syscon-reboot"; + regmap = <&reset>; + offset = <0x0>; + mask = <0x02>; + }; + soc { compatible = "simple-bus"; #address-cells = <2>; @@ -226,6 +233,11 @@ dcfg: dcfg@1e00000 { little-endian; }; + reset: syscon@1e60000 { + compatible = "fsl,ls1088a-reset", "syscon"; + reg = <0x0 0x1e60000 0x0 0x10000>; + }; + isc: syscon@1f70000 { compatible = "fsl,ls1088a-isc", "syscon"; reg = <0x0 0x1f70000 0x0 0x10000>; @@ -485,6 +497,7 @@ usb1: usb@3110000 { dr_mode = "host"; snps,quirk-frame-length-adjustment = <0x20>; snps,dis_rxdet_inp3_quirk; + snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-lx2160a-qds.dts index d858d9c8b5..2ecfa90f5e 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a-qds.dts +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a-qds.dts @@ -31,6 +31,130 @@ sb_3v3: regulator-sb3v3 { regulator-boot-on; regulator-always-on; }; + + mdio-mux-1 { + compatible = "mdio-mux-multiplexer"; + mux-controls = <&mux 0>; + mdio-parent-bus = <&emdio1>; + #address-cells=<1>; + #size-cells = <0>; + + mdio@0 { /* On-board PHY #1 RGMI1*/ + reg = <0x00>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@8 { /* On-board PHY #2 RGMI2*/ + reg = <0x8>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@18 { /* Slot #1 */ + reg = <0x18>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@19 { /* Slot #2 */ + reg = <0x19>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1a { /* Slot #3 */ + reg = <0x1a>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1b { /* Slot #4 */ + reg = <0x1b>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1c { /* Slot #5 */ + reg = <0x1c>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1d { /* Slot #6 */ + reg = <0x1d>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1e { /* Slot #7 */ + reg = <0x1e>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1f { /* Slot #8 */ + reg = <0x1f>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + + mdio-mux-2 { + compatible = "mdio-mux-multiplexer"; + mux-controls = <&mux 1>; + mdio-parent-bus = <&emdio2>; + #address-cells=<1>; + #size-cells = <0>; + + mdio@0 { /* Slot #1 (secondary EMI) */ + reg = <0x00>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@1 { /* Slot #2 (secondary EMI) */ + reg = <0x01>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@2 { /* Slot #3 (secondary EMI) */ + reg = <0x02>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@3 { /* Slot #4 (secondary EMI) */ + reg = <0x03>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@4 { /* Slot #5 (secondary EMI) */ + reg = <0x04>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@5 { /* Slot #6 (secondary EMI) */ + reg = <0x05>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@6 { /* Slot #7 (secondary EMI) */ + reg = <0x06>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mdio@7 { /* Slot #8 (secondary EMI) */ + reg = <0x07>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; }; &can0 { @@ -81,6 +205,14 @@ dflash2: flash@0 { }; }; +&emdio1 { + status = "okay"; +}; + +&emdio2 { + status = "okay"; +}; + &esdhc0 { status = "okay"; }; @@ -107,6 +239,19 @@ mt35xu512aba0: flash@0 { &i2c0 { status = "okay"; + fpga@66 { + compatible = "fsl,lx2160aqds-fpga", "fsl,fpga-qixis-i2c", + "simple-mfd"; + reg = <0x66>; + + mux: mux-controller { + compatible = "reg-mux"; + #mux-control-cells = <1>; + mux-reg-masks = <0x54 0xf8>, /* 0: reg 0x54, bits 7:3 */ + <0x54 0x07>; /* 1: reg 0x54, bit 2:0 */ + }; + }; + i2c-mux@77 { compatible = "nxp,pca9547"; reg = <0x77>; @@ -156,6 +301,10 @@ rtc@51 { }; }; +&optee { + status = "okay"; +}; + &sata0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-lx2160a-rdb.dts index 028ff8074b..0c44b3cbef 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a-rdb.dts @@ -49,6 +49,14 @@ &dpmac4 { managed = "in-band-status"; }; +&dpmac5 { + phy-handle = <&inphi_phy>; +}; + +&dpmac6 { + phy-handle = <&inphi_phy>; +}; + &dpmac17 { phy-handle = <&rgmii_phy1>; phy-connection-type = "rgmii-id"; @@ -109,6 +117,15 @@ can-transceiver { }; }; +&emdio2 { + status = "okay"; + + inphi_phy: ethernet-phy@0 { + compatible = "ethernet-phy-id0210.7440"; + reg = <0x0>; + }; +}; + &esdhc0 { sd-uhs-sdr104; sd-uhs-sdr50; @@ -202,6 +219,10 @@ rtc@51 { }; }; +&optee { + status = "okay"; +}; + &pcs_mdio3 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi index 51c4f61007..3c611cb4f5 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi @@ -1023,6 +1023,7 @@ usb0: usb@3100000 { interrupts = ; dr_mode = "host"; snps,quirk-frame-length-adjustment = <0x20>; + usb3-lpm-capable; snps,dis_rxdet_inp3_quirk; snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; status = "disabled"; @@ -1034,6 +1035,7 @@ usb1: usb@3110000 { interrupts = ; dr_mode = "host"; snps,quirk-frame-length-adjustment = <0x20>; + usb3-lpm-capable; snps,dis_rxdet_inp3_quirk; snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; status = "disabled"; @@ -1751,4 +1753,12 @@ dpmac18: ethernet@12 { }; }; }; + + firmware { + optee: optee { + compatible = "linaro,optee-tz"; + method = "smc"; + status = "disabled"; + }; + }; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2162a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-lx2162a-qds.dts index e1defee1ad..a1644ceed1 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2162a-qds.dts +++ b/arch/arm64/boot/dts/freescale/fsl-lx2162a-qds.dts @@ -167,6 +167,14 @@ mdio@7 { /* Slot #8 (secondary EMI) */ }; }; +&can0 { + status = "okay"; +}; + +&can1 { + status = "okay"; +}; + &crypto { status = "okay"; }; @@ -226,10 +234,17 @@ &emdio2 { }; &esdhc0 { + sd-uhs-sdr104; + sd-uhs-sdr50; + sd-uhs-sdr25; + sd-uhs-sdr12; status = "okay"; }; &esdhc1 { + mmc-hs200-1_8v; + mmc-hs400-1_8v; + bus-width = <8>; status = "okay"; }; @@ -302,11 +317,17 @@ temperature-sensor@4c { rtc@51 { compatible = "nxp,pcf2129"; reg = <0x51>; + /* IRQ_RTC_B -> IRQ11_B(CPLD) -> IRQ11(CPU), active low */ + interrupts-extended = <&extirq 11 IRQ_TYPE_LEVEL_LOW>; }; }; }; }; +&optee { + status = "okay"; +}; + &sata0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-beacon-baseboard.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-beacon-baseboard.dtsi index 6f5e63696e..0da311898e 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-beacon-baseboard.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-beacon-baseboard.dtsi @@ -43,6 +43,27 @@ reg_audio: regulator-audio { enable-active-high; }; + reg_usbotg1: regulator-usbotg1 { + compatible = "regulator-fixed"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_reg_usb_otg1>; + regulator-name = "usb_otg_vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + gpio = <&gpio4 29 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + reg_camera: regulator-camera { + compatible = "regulator-fixed"; + regulator-name = "mipi_pwr"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + gpio = <&pca6416_1 0 GPIO_ACTIVE_HIGH>; + enable-active-high; + startup-delay-us = <100000>; + }; + reg_usdhc2_vmmc: regulator-usdhc2 { compatible = "regulator-fixed"; regulator-name = "VSD_3V3"; @@ -67,6 +88,10 @@ sound { }; }; +&csi { + status = "okay"; +}; + &ecspi2 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_espi2>; @@ -90,6 +115,30 @@ &i2c2 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c2>; status = "okay"; + + camera@3c { + compatible = "ovti,ov5640"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ov5640>; + reg = <0x3c>; + clocks = <&clk IMX8MM_CLK_CLKO1>; + clock-names = "xclk"; + assigned-clocks = <&clk IMX8MM_CLK_CLKO1>; + assigned-clock-parents = <&clk IMX8MM_CLK_24M>; + assigned-clock-rates = <24000000>; + AVDD-supply = <®_camera>; /* 2.8v */ + powerdown-gpios = <&gpio1 7 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio1 6 GPIO_ACTIVE_LOW>; + + port { + /* MIPI CSI-2 bus endpoint */ + ov5640_to_mipi_csi2: endpoint { + remote-endpoint = <&imx8mm_mipi_csi_in>; + clock-lanes = <0>; + data-lanes = <1 2>; + }; + }; + }; }; &i2c4 { @@ -141,6 +190,18 @@ pca6416_1: gpio@21 { }; }; +&mipi_csi { + status = "okay"; + ports { + port@0 { + imx8mm_mipi_csi_in: endpoint { + remote-endpoint = <&ov5640_to_mipi_csi2>; + data-lanes = <1 2>; + }; + }; + }; +}; + &sai3 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_sai3>; @@ -169,6 +230,24 @@ &uart3 { status = "okay"; }; +&usbotg1 { + vbus-supply = <®_usbotg1>; + disable-over-current; + dr_mode="otg"; + status = "okay"; +}; + +&usbotg2 { + pinctrl-names = "default"; + disable-over-current; + dr_mode="host"; + status = "okay"; +}; + +&usbphynop2 { + reset-gpios = <&pca6416_1 7 GPIO_ACTIVE_HIGH>; +}; + &usdhc2 { pinctrl-names = "default", "state_100mhz", "state_200mhz"; pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>; @@ -209,12 +288,26 @@ MX8MM_IOMUXC_SAI3_RXFS_GPIO4_IO28 0x41 >; }; + pinctrl_ov5640: ov5640grp { + fsl,pins = < + MX8MM_IOMUXC_GPIO1_IO07_GPIO1_IO7 0x19 + MX8MM_IOMUXC_GPIO1_IO06_GPIO1_IO6 0x19 + MX8MM_IOMUXC_GPIO1_IO14_CCMSRCGPCMIX_CLKO1 0x59 + >; + }; + pinctrl_pcal6414: pcal6414-gpiogrp { fsl,pins = < MX8MM_IOMUXC_SAI2_MCLK_GPIO4_IO27 0x19 >; }; + pinctrl_reg_usb_otg1: usbotg1grp { + fsl,pins = < + MX8MM_IOMUXC_SAI3_RXC_GPIO4_IO29 0x19 + >; + }; + pinctrl_sai3: sai3grp { fsl,pins = < MX8MM_IOMUXC_SAI3_TXFS_SAI3_TX_SYNC 0xd6 diff --git a/arch/arm64/boot/dts/freescale/imx8mm-beacon-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-beacon-som.dtsi index 40f5e7a3b0..cf07987ccc 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-beacon-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-beacon-som.dtsi @@ -263,7 +263,6 @@ &usdhc1 { bus-width = <4>; non-removable; cap-power-off-card; - pm-ignore-notify; keep-power-in-suspend; mmc-pwrseq = <&usdhc1_pwrseq>; status = "okay"; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi index e033d0257b..3bac87b7e1 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi @@ -116,6 +116,13 @@ ethphy0: ethernet-phy@0 { reg = <0>; reset-gpios = <&gpio4 22 GPIO_ACTIVE_LOW>; reset-assert-us = <10000>; + qca,disable-smarteee; + vddio-supply = <&vddio>; + + vddio: vddio-regulator { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; }; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm-ctouch2.dts b/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm-ctouch2.dts index 5389d6f2be..5027454028 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm-ctouch2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm-ctouch2.dts @@ -91,7 +91,6 @@ &usdhc1 { max-frequency = <50000000>; bus-width = <4>; no-1-8-v; - pm-ignore-notify; keep-power-in-suspend; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm-edimm2.2.dts b/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm-edimm2.2.dts index a4a2ada148..ddac8bc7ae 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm-edimm2.2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm-edimm2.2.dts @@ -91,7 +91,6 @@ &usdhc1 { max-frequency = <50000000>; bus-width = <4>; no-1-8-v; - pm-ignore-notify; keep-power-in-suspend; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts b/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts index 49d7470812..d40caf14ac 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts @@ -191,8 +191,10 @@ usbnet: usbether@1 { }; &usdhc2 { - pinctrl-names = "default"; + pinctrl-names = "default", "state_100mhz", "state_200mhz"; pinctrl-0 = <&pinctrl_usdhc2>; + pinctrl-1 = <&pinctrl_usdhc2_100mhz>; + pinctrl-2 = <&pinctrl_usdhc2_200mhz>; vmmc-supply = <®_vdd_3v3>; vqmmc-supply = <®_nvcc_sd>; cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>; @@ -321,4 +323,28 @@ MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d0 MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x019 >; }; + + pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK 0x194 + MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD 0x1d4 + MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0 0x1d4 + MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1 0x1d4 + MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d4 + MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d4 + MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x019 + >; + }; + + pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK 0x196 + MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD 0x1d6 + MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0 0x1d6 + MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1 0x1d6 + MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d6 + MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d6 + MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x019 + >; + }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-som.dtsi index 42bbbb3f53..22a5ef771d 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-som.dtsi @@ -63,7 +63,7 @@ opp-750M { &ecspi1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ecspi1>; - cs-gpios = <&gpio5 9 GPIO_ACTIVE_HIGH>; + cs-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>; status = "okay"; spi-flash@0 { @@ -86,6 +86,7 @@ pca9450: pmic@25 { pinctrl-0 = <&pinctrl_pmic>; interrupt-parent = <&gpio1>; interrupts = <0 IRQ_TYPE_LEVEL_LOW>; + sd-vsel-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; regulators { reg_vdd_soc: BUCK1 { @@ -228,6 +229,7 @@ MX8MM_IOMUXC_I2C1_SDA_I2C1_SDA 0x400001c3 pinctrl_pmic: pmicgrp { fsl,pins = < MX8MM_IOMUXC_GPIO1_IO00_GPIO1_IO0 0x141 + MX8MM_IOMUXC_GPIO1_IO04_GPIO1_IO4 0x141 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw71xx.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw71xx.dtsi index 8e4a0ce997..28012279f6 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw71xx.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw71xx.dtsi @@ -57,7 +57,7 @@ reg_usb_otg1_vbus: regulator-usb-otg1 { &ecspi2 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi2>; - cs-gpios = <&gpio5 13 GPIO_ACTIVE_HIGH>; + cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi index b7c91bdc21..27afa46a25 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx.dtsi @@ -76,7 +76,7 @@ reg_usb_otg2_vbus: regulator-usb-otg2 { &ecspi2 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi2>; - cs-gpios = <&gpio5 13 GPIO_ACTIVE_HIGH>; + cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw73xx.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw73xx.dtsi index d2ffd62a3b..a59e849c7b 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw73xx.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw73xx.dtsi @@ -96,7 +96,7 @@ reg_wifi_en: regulator-wifi-en { &ecspi2 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi2>; - cs-gpios = <&gpio5 13 GPIO_ACTIVE_HIGH>; + cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7901.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7901.dts index bafd5c8ea4..21c546c462 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7901.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7901.dts @@ -255,6 +255,10 @@ opp-750M { }; }; +&disp_blk_ctrl { + status = "disabled"; +}; + &ecspi1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi1>; @@ -282,6 +286,14 @@ fixed-link { }; }; +&gpu_2d { + status = "disabled"; +}; + +&gpu_3d { + status = "disabled"; +}; + &i2c1 { clock-frequency = <100000>; pinctrl-names = "default"; @@ -632,6 +644,18 @@ &i2c4 { status = "okay"; }; +&pgc_gpu { + status = "disabled"; +}; + +&pgc_gpumix { + status = "disabled"; +}; + +&pgc_mipi { + status = "disabled"; +}; + &uart1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>, <&pinctrl_uart1_gpio>; diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi index 2f632e8ca3..0c7a72c51a 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include "imx8mm-pinfunc.h" @@ -63,6 +65,12 @@ A53_0: cpu@0 { clock-latency = <61036>; /* two CLK32 periods */ clocks = <&clk IMX8MM_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; nvmem-cells = <&cpu_speed_grade>; @@ -78,6 +86,12 @@ A53_1: cpu@1 { clock-latency = <61036>; /* two CLK32 periods */ clocks = <&clk IMX8MM_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; cpu-idle-states = <&cpu_pd_wait>; @@ -91,6 +105,12 @@ A53_2: cpu@2 { clock-latency = <61036>; /* two CLK32 periods */ clocks = <&clk IMX8MM_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; cpu-idle-states = <&cpu_pd_wait>; @@ -104,6 +124,12 @@ A53_3: cpu@3 { clock-latency = <61036>; /* two CLK32 periods */ clocks = <&clk IMX8MM_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; cpu-idle-states = <&cpu_pd_wait>; @@ -112,6 +138,10 @@ A53_3: cpu@3 { A53_L2: l2-cache0 { compatible = "cache"; + cache-level = <2>; + cache-size = <0x80000>; + cache-line-size = <64>; + cache-sets = <512>; }; }; @@ -589,8 +619,7 @@ clk: clock-controller@30380000 { <&clk IMX8MM_CLK_IPG_AUDIO_ROOT>, <&clk IMX8MM_SYS_PLL3>, <&clk IMX8MM_VIDEO_PLL1>, - <&clk IMX8MM_AUDIO_PLL1>, - <&clk IMX8MM_AUDIO_PLL2>; + <&clk IMX8MM_AUDIO_PLL1>; assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_800M>, <&clk IMX8MM_ARM_PLL_OUT>, <&clk IMX8MM_SYS_PLL3_OUT>, @@ -600,8 +629,7 @@ clk: clock-controller@30380000 { <400000000>, <750000000>, <594000000>, - <393216000>, - <361267200>; + <393216000>; }; src: reset-controller@30390000 { @@ -610,6 +638,110 @@ src: reset-controller@30390000 { interrupts = ; #reset-cells = <1>; }; + + gpc: gpc@303a0000 { + compatible = "fsl,imx8mm-gpc"; + reg = <0x303a0000 0x10000>; + interrupts = ; + interrupt-parent = <&gic>; + interrupt-controller; + #interrupt-cells = <3>; + + pgc { + #address-cells = <1>; + #size-cells = <0>; + + pgc_hsiomix: power-domain@0 { + #power-domain-cells = <0>; + reg = ; + clocks = <&clk IMX8MM_CLK_USB_BUS>; + assigned-clocks = <&clk IMX8MM_CLK_USB_BUS>; + assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_500M>; + }; + + pgc_pcie: power-domain@1 { + #power-domain-cells = <0>; + reg = ; + power-domains = <&pgc_hsiomix>; + clocks = <&clk IMX8MM_CLK_PCIE1_ROOT>; + }; + + pgc_otg1: power-domain@2 { + #power-domain-cells = <0>; + reg = ; + power-domains = <&pgc_hsiomix>; + }; + + pgc_otg2: power-domain@3 { + #power-domain-cells = <0>; + reg = ; + power-domains = <&pgc_hsiomix>; + }; + + pgc_gpumix: power-domain@4 { + #power-domain-cells = <0>; + reg = ; + clocks = <&clk IMX8MM_CLK_GPU_BUS_ROOT>, + <&clk IMX8MM_CLK_GPU_AHB>; + assigned-clocks = <&clk IMX8MM_CLK_GPU_AXI>, + <&clk IMX8MM_CLK_GPU_AHB>; + assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_800M>, + <&clk IMX8MM_SYS_PLL1_800M>; + assigned-clock-rates = <800000000>, <400000000>; + }; + + pgc_gpu: power-domain@5 { + #power-domain-cells = <0>; + reg = ; + clocks = <&clk IMX8MM_CLK_GPU_AHB>, + <&clk IMX8MM_CLK_GPU_BUS_ROOT>, + <&clk IMX8MM_CLK_GPU2D_ROOT>, + <&clk IMX8MM_CLK_GPU3D_ROOT>; + resets = <&src IMX8MQ_RESET_GPU_RESET>; + power-domains = <&pgc_gpumix>; + }; + + pgc_vpumix: power-domain@6 { + #power-domain-cells = <0>; + reg = ; + clocks = <&clk IMX8MM_CLK_VPU_DEC_ROOT>; + assigned-clocks = <&clk IMX8MM_CLK_VPU_BUS>; + assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_800M>; + }; + + pgc_vpu_g1: power-domain@7 { + #power-domain-cells = <0>; + reg = ; + }; + + pgc_vpu_g2: power-domain@8 { + #power-domain-cells = <0>; + reg = ; + }; + + pgc_vpu_h1: power-domain@9 { + #power-domain-cells = <0>; + reg = ; + }; + + pgc_dispmix: power-domain@10 { + #power-domain-cells = <0>; + reg = ; + clocks = <&clk IMX8MM_CLK_DISP_APB_ROOT>, + <&clk IMX8MM_CLK_DISP_AXI_ROOT>; + assigned-clocks = <&clk IMX8MM_CLK_DISP_AXI>, + <&clk IMX8MM_CLK_DISP_APB>; + assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_1000M>, + <&clk IMX8MM_SYS_PLL1_800M>; + assigned-clock-rates = <500000000>, <200000000>; + }; + + pgc_mipi: power-domain@11 { + #power-domain-cells = <0>; + reg = ; + }; + }; + }; }; aips2: bus@30400000 { @@ -947,7 +1079,6 @@ fec1: ethernet@30be0000 { fsl,num-rx-queues = <3>; nvmem-cells = <&fec_mac_address>; nvmem-cell-names = "mac-address"; - nvmem_macaddr_swap; fsl,stop-mode = <&gpr 0x10 3>; status = "disabled"; }; @@ -961,6 +1092,84 @@ aips4: bus@32c00000 { #size-cells = <1>; ranges = <0x32c00000 0x32c00000 0x400000>; + csi: csi@32e20000 { + compatible = "fsl,imx8mm-csi", "fsl,imx7-csi"; + reg = <0x32e20000 0x1000>; + interrupts = ; + clocks = <&clk IMX8MM_CLK_CSI1_ROOT>; + clock-names = "mclk"; + power-domains = <&disp_blk_ctrl IMX8MM_DISPBLK_PD_CSI_BRIDGE>; + status = "disabled"; + + port { + csi_in: endpoint { + remote-endpoint = <&imx8mm_mipi_csi_out>; + }; + }; + }; + + disp_blk_ctrl: blk-ctrl@32e28000 { + compatible = "fsl,imx8mm-disp-blk-ctrl", "syscon"; + reg = <0x32e28000 0x100>; + power-domains = <&pgc_dispmix>, <&pgc_dispmix>, + <&pgc_dispmix>, <&pgc_mipi>, + <&pgc_mipi>; + power-domain-names = "bus", "csi-bridge", + "lcdif", "mipi-dsi", + "mipi-csi"; + clocks = <&clk IMX8MM_CLK_DISP_AXI_ROOT>, + <&clk IMX8MM_CLK_DISP_APB_ROOT>, + <&clk IMX8MM_CLK_CSI1_ROOT>, + <&clk IMX8MM_CLK_DISP_AXI_ROOT>, + <&clk IMX8MM_CLK_DISP_APB_ROOT>, + <&clk IMX8MM_CLK_DISP_ROOT>, + <&clk IMX8MM_CLK_DSI_CORE>, + <&clk IMX8MM_CLK_DSI_PHY_REF>, + <&clk IMX8MM_CLK_CSI1_CORE>, + <&clk IMX8MM_CLK_CSI1_PHY_REF>; + clock-names = "csi-bridge-axi","csi-bridge-apb", + "csi-bridge-core", "lcdif-axi", + "lcdif-apb", "lcdif-pix", + "dsi-pclk", "dsi-ref", + "csi-aclk", "csi-pclk"; + #power-domain-cells = <1>; + }; + + mipi_csi: mipi-csi@32e30000 { + compatible = "fsl,imx8mm-mipi-csi2"; + reg = <0x32e30000 0x1000>; + interrupts = ; + assigned-clocks = <&clk IMX8MM_CLK_CSI1_CORE>, + <&clk IMX8MM_CLK_CSI1_PHY_REF>; + assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_1000M>, + <&clk IMX8MM_SYS_PLL2_1000M>; + clock-frequency = <333000000>; + clocks = <&clk IMX8MM_CLK_DISP_APB_ROOT>, + <&clk IMX8MM_CLK_CSI1_ROOT>, + <&clk IMX8MM_CLK_CSI1_PHY_REF>, + <&clk IMX8MM_CLK_DISP_AXI_ROOT>; + clock-names = "pclk", "wrap", "phy", "axi"; + power-domains = <&disp_blk_ctrl IMX8MM_DISPBLK_PD_MIPI_CSI>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + + imx8mm_mipi_csi_out: endpoint { + remote-endpoint = <&csi_in>; + }; + }; + }; + }; + usbotg1: usb@32e40000 { compatible = "fsl,imx8mm-usb", "fsl,imx7d-usb"; reg = <0x32e40000 0x200>; @@ -971,6 +1180,7 @@ usbotg1: usb@32e40000 { assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_500M>; phys = <&usbphynop1>; fsl,usbmisc = <&usbmisc1 0>; + power-domains = <&pgc_otg1>; status = "disabled"; }; @@ -990,6 +1200,7 @@ usbotg2: usb@32e50000 { assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_500M>; phys = <&usbphynop2>; fsl,usbmisc = <&usbmisc2 0>; + power-domains = <&pgc_otg2>; status = "disabled"; }; @@ -1030,6 +1241,50 @@ gpmi: nand-controller@33002000{ status = "disabled"; }; + gpu_3d: gpu@38000000 { + compatible = "vivante,gc"; + reg = <0x38000000 0x8000>; + interrupts = ; + clocks = <&clk IMX8MM_CLK_GPU_AHB>, + <&clk IMX8MM_CLK_GPU_BUS_ROOT>, + <&clk IMX8MM_CLK_GPU3D_ROOT>, + <&clk IMX8MM_CLK_GPU3D_ROOT>; + clock-names = "reg", "bus", "core", "shader"; + assigned-clocks = <&clk IMX8MM_CLK_GPU3D_CORE>, + <&clk IMX8MM_GPU_PLL_OUT>; + assigned-clock-parents = <&clk IMX8MM_GPU_PLL_OUT>; + assigned-clock-rates = <0>, <1000000000>; + power-domains = <&pgc_gpu>; + }; + + gpu_2d: gpu@38008000 { + compatible = "vivante,gc"; + reg = <0x38008000 0x8000>; + interrupts = ; + clocks = <&clk IMX8MM_CLK_GPU_AHB>, + <&clk IMX8MM_CLK_GPU_BUS_ROOT>, + <&clk IMX8MM_CLK_GPU2D_ROOT>; + clock-names = "reg", "bus", "core"; + assigned-clocks = <&clk IMX8MM_CLK_GPU2D_CORE>, + <&clk IMX8MM_GPU_PLL_OUT>; + assigned-clock-parents = <&clk IMX8MM_GPU_PLL_OUT>; + assigned-clock-rates = <0>, <1000000000>; + power-domains = <&pgc_gpu>; + }; + + vpu_blk_ctrl: blk-ctrl@38330000 { + compatible = "fsl,imx8mm-vpu-blk-ctrl", "syscon"; + reg = <0x38330000 0x100>; + power-domains = <&pgc_vpumix>, <&pgc_vpu_g1>, + <&pgc_vpu_g2>, <&pgc_vpu_h1>; + power-domain-names = "bus", "g1", "g2", "h1"; + clocks = <&clk IMX8MM_CLK_VPU_G1_ROOT>, + <&clk IMX8MM_CLK_VPU_G2_ROOT>, + <&clk IMX8MM_CLK_VPU_H1_ROOT>; + clock-names = "g1", "g2", "h1"; + #power-domain-cells = <1>; + }; + gic: interrupt-controller@38800000 { compatible = "arm,gic-v3"; reg = <0x38800000 0x10000>, /* GIC Dist */ diff --git a/arch/arm64/boot/dts/freescale/imx8mn-beacon-baseboard.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-beacon-baseboard.dtsi index 376ca8ff72..0f40b43ac0 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-beacon-baseboard.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn-beacon-baseboard.dtsi @@ -126,7 +126,6 @@ wm8962: audio-codec@1a { compatible = "wlf,wm8962"; reg = <0x1a>; clocks = <&clk IMX8MN_CLK_SAI3_ROOT>; - clock-names = "xclk"; DCVDD-supply = <®_audio>; DBVDD-supply = <®_audio>; AVDD-supply = <®_audio>; diff --git a/arch/arm64/boot/dts/freescale/imx8mn-beacon-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-beacon-som.dtsi index 3b2d627a03..1133cded9b 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-beacon-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn-beacon-som.dtsi @@ -274,7 +274,6 @@ &usdhc1 { bus-width = <4>; non-removable; cap-power-off-card; - pm-ignore-notify; keep-power-in-suspend; mmc-pwrseq = <&usdhc1_pwrseq>; status = "okay"; diff --git a/arch/arm64/boot/dts/freescale/imx8mn-evk.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-evk.dtsi index 85e65f8719..c3f15192b7 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-evk.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn-evk.dtsi @@ -97,6 +97,15 @@ mdio { ethphy0: ethernet-phy@0 { compatible = "ethernet-phy-ieee802.3-c22"; reg = <0>; + reset-gpios = <&gpio4 22 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + qca,disable-smarteee; + vddio-supply = <&vddio>; + + vddio: vddio-regulator { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; }; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mn.dtsi b/arch/arm64/boot/dts/freescale/imx8mn.dtsi index da6c942fb7..b8d49d5f26 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn.dtsi @@ -63,6 +63,12 @@ A53_0: cpu@0 { clock-latency = <61036>; clocks = <&clk IMX8MN_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; nvmem-cells = <&cpu_speed_grade>; @@ -78,6 +84,12 @@ A53_1: cpu@1 { clock-latency = <61036>; clocks = <&clk IMX8MN_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; cpu-idle-states = <&cpu_pd_wait>; @@ -91,6 +103,12 @@ A53_2: cpu@2 { clock-latency = <61036>; clocks = <&clk IMX8MN_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; cpu-idle-states = <&cpu_pd_wait>; @@ -104,6 +122,12 @@ A53_3: cpu@3 { clock-latency = <61036>; clocks = <&clk IMX8MN_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; cpu-idle-states = <&cpu_pd_wait>; @@ -112,6 +136,10 @@ A53_3: cpu@3 { A53_L2: l2-cache0 { compatible = "cache"; + cache-level = <2>; + cache-size = <0x80000>; + cache-line-size = <64>; + cache-sets = <512>; }; }; @@ -948,7 +976,6 @@ fec1: ethernet@30be0000 { fsl,num-rx-queues = <3>; nvmem-cells = <&fec_mac_address>; nvmem-cell-names = "mac-address"; - nvmem_macaddr_swap; fsl,stop-mode = <&gpr 0x10 3>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts index 7b99fad6e4..2eb9432106 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts @@ -86,6 +86,9 @@ &eqos { pinctrl-0 = <&pinctrl_eqos>; phy-mode = "rgmii-id"; phy-handle = <ðphy0>; + snps,force_thresh_dma_mode; + snps,mtl-tx-config = <&mtl_tx_setup>; + snps,mtl-rx-config = <&mtl_rx_setup>; status = "okay"; mdio { @@ -97,6 +100,75 @@ ethphy0: ethernet-phy@1 { compatible = "ethernet-phy-ieee802.3-c22"; reg = <1>; eee-broken-1000t; + reset-gpios = <&gpio4 22 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + reset-deassert-us = <80000>; + realtek,clkout-disable; + }; + }; + + mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <5>; + snps,tx-sched-sp; + + queue0 { + snps,dcb-algorithm; + snps,priority = <0x1>; + }; + + queue1 { + snps,dcb-algorithm; + snps,priority = <0x2>; + }; + + queue2 { + snps,dcb-algorithm; + snps,priority = <0x4>; + }; + + queue3 { + snps,dcb-algorithm; + snps,priority = <0x8>; + }; + + queue4 { + snps,dcb-algorithm; + snps,priority = <0xf0>; + }; + }; + + mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <5>; + snps,rx-sched-sp; + + queue0 { + snps,dcb-algorithm; + snps,priority = <0x1>; + snps,map-to-dma-channel = <0>; + }; + + queue1 { + snps,dcb-algorithm; + snps,priority = <0x2>; + snps,map-to-dma-channel = <1>; + }; + + queue2 { + snps,dcb-algorithm; + snps,priority = <0x4>; + snps,map-to-dma-channel = <2>; + }; + + queue3 { + snps,dcb-algorithm; + snps,priority = <0x8>; + snps,map-to-dma-channel = <3>; + }; + + queue4 { + snps,dcb-algorithm; + snps,priority = <0xf0>; + snps,map-to-dma-channel = <4>; }; }; }; @@ -120,6 +192,7 @@ ethphy1: ethernet-phy@1 { reset-gpios = <&gpio4 2 GPIO_ACTIVE_LOW>; reset-assert-us = <10000>; reset-deassert-us = <80000>; + realtek,clkout-disable; }; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp.dtsi index 9b07b26230..6b840c05dd 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp.dtsi @@ -51,6 +51,12 @@ A53_0: cpu@0 { clock-latency = <61036>; clocks = <&clk IMX8MP_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; #cooling-cells = <2>; }; @@ -62,6 +68,12 @@ A53_1: cpu@1 { clock-latency = <61036>; clocks = <&clk IMX8MP_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; #cooling-cells = <2>; }; @@ -73,6 +85,12 @@ A53_2: cpu@2 { clock-latency = <61036>; clocks = <&clk IMX8MP_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; #cooling-cells = <2>; }; @@ -84,12 +102,22 @@ A53_3: cpu@3 { clock-latency = <61036>; clocks = <&clk IMX8MP_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; #cooling-cells = <2>; }; A53_L2: l2-cache0 { compatible = "cache"; + cache-level = <2>; + cache-size = <0x80000>; + cache-line-size = <64>; + cache-sets = <512>; }; }; @@ -369,6 +397,10 @@ cpu_speed_grade: speed-grade@10 { eth_mac1: mac-address@90 { reg = <0x90 6>; }; + + eth_mac2: mac-address@96 { + reg = <0x96 6>; + }; }; anatop: anatop@30360000 { @@ -785,7 +817,7 @@ flexspi: spi@30bb0000 { interrupts = ; clocks = <&clk IMX8MP_CLK_QSPI_ROOT>, <&clk IMX8MP_CLK_QSPI_ROOT>; - clock-names = "fspi", "fspi_en"; + clock-names = "fspi_en", "fspi"; assigned-clock-rates = <80000000>; assigned-clocks = <&clk IMX8MP_CLK_QSPI>; #address-cells = <1>; @@ -832,7 +864,6 @@ fec: ethernet@30be0000 { nvmem-cells = <ð_mac1>; nvmem-cell-names = "mac-address"; fsl,stop-mode = <&gpr 0x10 3>; - nvmem_macaddr_swap; status = "disabled"; }; @@ -854,6 +885,8 @@ eqos: ethernet@30bf0000 { <&clk IMX8MP_SYS_PLL2_100M>, <&clk IMX8MP_SYS_PLL2_125M>; assigned-clock-rates = <0>, <100000000>, <125000000>; + nvmem-cells = <ð_mac2>; + nvmem-cell-names = "mac-address"; intf_mode = <&gpr 0x4>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-evk.dts b/arch/arm64/boot/dts/freescale/imx8mq-evk.dts index b83df77195..a1b7582f3e 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx8mq-evk.dts @@ -169,6 +169,11 @@ ethphy0: ethernet-phy@0 { reg = <0>; reset-gpios = <&gpio1 9 GPIO_ACTIVE_LOW>; reset-assert-us = <10000>; + qca,disable-smarteee; + vddio-supply = <&vddh>; + + vddh: vddh-regulator { + }; }; }; }; @@ -326,6 +331,10 @@ &pgc_gpu { power-supply = <&sw1a_reg>; }; +&pgc_vpu { + power-supply = <&sw1c_reg>; +}; + &qspi0 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_qspi>; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts index cd3c3edd48..4533a84fb0 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r3.dts @@ -1,14 +1,9 @@ // SPDX-License-Identifier: (GPL-2.0+ OR MIT) -// Copyright (C) 2020 Purism SPC +// Copyright (C) 2021 Purism SPC /dts-v1/; -#include "imx8mq-librem5.dtsi" - -/ { - model = "Purism Librem 5r3"; - compatible = "purism,librem5r3", "purism,librem5", "fsl,imx8mq"; -}; +#include "imx8mq-librem5-r3.dtsi" &a53_opp_table { opp-1000000000 { @@ -16,22 +11,6 @@ opp-1000000000 { }; }; -&accel_gyro { - mount-matrix = "1", "0", "0", - "0", "1", "0", - "0", "0", "-1"; -}; - -&bq25895 { - ti,battery-regulation-voltage = <4200000>; /* uV */ - ti,charge-current = <1500000>; /* uA */ - ti,termination-current = <144000>; /* uA */ -}; - &buck3_reg { regulator-always-on; }; - -&proximity { - proximity-near-level = <25>; -}; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r4.dts b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r4.dts index cbfb49aa25..30d65bef69 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5-r4.dts +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5-r4.dts @@ -1,31 +1,19 @@ // SPDX-License-Identifier: (GPL-2.0+ OR MIT) -// Copyright (C) 2020 Purism SPC +// Copyright (C) 2021 Purism SPC /dts-v1/; -#include "imx8mq-librem5.dtsi" +#include "imx8mq-librem5-r3.dtsi" / { model = "Purism Librem 5r4"; compatible = "purism,librem5r4", "purism,librem5", "fsl,imx8mq"; }; -&accel_gyro { - mount-matrix = "1", "0", "0", - "0", "1", "0", - "0", "0", "-1"; -}; - &bat { maxim,rsns-microohm = <1667>; }; -&bq25895 { - ti,battery-regulation-voltage = <4200000>; /* uV */ - ti,charge-current = <1500000>; /* uA */ - ti,termination-current = <144000>; /* uA */ -}; - &led_backlight { led-max-microamp = <25000>; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi index 460ef0d865..2d4a472af6 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi @@ -14,6 +14,7 @@ / { model = "Purism Librem 5"; compatible = "purism,librem5", "fsl,imx8mq"; + chassis-type = "handset"; backlight_dsi: backlight-dsi { compatible = "led-backlight"; @@ -40,12 +41,14 @@ vol-down { label = "VOL_DOWN"; gpios = <&gpio1 17 GPIO_ACTIVE_LOW>; linux,code = ; + debounce-interval = <50>; }; vol-up { label = "VOL_UP"; gpios = <&gpio1 16 GPIO_ACTIVE_LOW>; linux,code = ; + debounce-interval = <50>; }; }; @@ -60,6 +63,40 @@ reg_aud_1v8: regulator-audio-1v8 { enable-active-high; }; + /* + * the pinctrl for reg_csi_1v8 and reg_vcam_1v8 is added to the PMIC + * since we can't have it twice in the 2 different regulator nodes. + */ + reg_csi_1v8: regulator-csi-1v8 { + compatible = "regulator-fixed"; + regulator-name = "CAMERA_VDDIO_1V8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <®_vdd_3v3>; + gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + /* controlled by the CAMERA_POWER_KEY HKS */ + reg_vcam_1v2: regulator-vcam-1v2 { + compatible = "regulator-fixed"; + regulator-name = "CAMERA_VDDD_1V2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + vin-supply = <®_vdd_1v8>; + enable-active-high; + }; + + reg_vcam_2v8: regulator-vcam-2v8 { + compatible = "regulator-fixed"; + regulator-name = "CAMERA_VDDA_2V8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + vin-supply = <®_vdd_3v3>; + gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + reg_gnss: regulator-gnss { compatible = "regulator-fixed"; pinctrl-names = "default"; @@ -138,9 +175,14 @@ reg_vsys_3v4: regulator-vsys-3v4 { reg_wifi_3v3: regulator-wifi-3v3 { compatible = "regulator-fixed"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_wifi_pwr>; regulator-name = "3V3_WIFI"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; + gpio = <&gpio3 10 GPIO_ACTIVE_HIGH>; + enable-active-high; + vin-supply = <®_vdd_3v3>; }; sound { @@ -193,6 +235,14 @@ simple-audio-card,codec { }; }; + usdhc2_pwrseq: pwrseq { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_bt>, <&pinctrl_wifi_disable>; + compatible = "mmc-pwrseq-simple"; + reset-gpios = <&gpio3 25 GPIO_ACTIVE_HIGH>, + <&gpio4 29 GPIO_ACTIVE_HIGH>; + }; + bm818_codec: sound-wwan-codec { compatible = "broadmobi,bm818", "option,gtm601"; #sound-dai-cells = <0>; @@ -222,10 +272,14 @@ &A53_3 { cpu-supply = <&buck2_reg>; }; +&csi1 { + status = "okay"; +}; + &ddrc { operating-points-v2 = <&ddrc_opp_table>; - ddrc_opp_table: ddrc-opp-table { + ddrc_opp_table: opp-table { compatible = "operating-points-v2"; opp-25M { @@ -307,6 +361,27 @@ MX8MQ_IOMUXC_NAND_DQS_GPIO3_IO14 0x83 >; }; + pinctrl_bt: btgrp { + fsl,pins = < + /* BT_REG_ON */ + MX8MQ_IOMUXC_SAI5_MCLK_GPIO3_IO25 0x83 + >; + }; + + pinctrl_camera_pwr: camerapwrgrp { + fsl,pins = < + /* CAMERA_PWR_EN_3V3 */ + MX8MQ_IOMUXC_GPIO1_IO00_GPIO1_IO0 0x83 + >; + }; + + pinctrl_csi1: csi1grp { + fsl,pins = < + /* CSI1_NRST */ + MX8MQ_IOMUXC_ENET_RXC_GPIO1_IO25 0x83 + >; + }; + pinctrl_charger_in: chargeringrp { fsl,pins = < /* CHRG_INT */ @@ -638,6 +713,20 @@ MX8MQ_IOMUXC_SD2_RESET_B_USDHC2_RESET_B 0xc1 >; }; + pinctrl_wifi_disable: wifidisablegrp { + fsl,pins = < + /* WIFI_REG_ON */ + MX8MQ_IOMUXC_SAI3_RXC_GPIO4_IO29 0x83 + >; + }; + + pinctrl_wifi_pwr: wifipwrgrp { + fsl,pins = < + /* WIFI3V3_EN */ + MX8MQ_IOMUXC_NAND_DATA04_GPIO3_IO10 0x83 + >; + }; + pinctrl_wdog: wdoggrp { fsl,pins = < /* nWDOG */ @@ -689,7 +778,7 @@ pmic: pmic@4b { compatible = "rohm,bd71837"; reg = <0x4b>; pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_pmic>; + pinctrl-0 = <&pinctrl_pmic>, <&pinctrl_camera_pwr>; clocks = <&pmic_osc>; clock-names = "osc"; clock-output-names = "pmic_clk"; @@ -922,6 +1011,31 @@ codec: audio-codec@1a { >; }; + camera_front: camera@20 { + compatible = "hynix,hi846"; + reg = <0x20>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_csi1>; + clocks = <&clk IMX8MQ_CLK_CLKO2>; + assigned-clocks = <&clk IMX8MQ_CLK_CLKO2>; + assigned-clock-rates = <25000000>; + reset-gpios = <&gpio1 25 GPIO_ACTIVE_LOW>; + vdda-supply = <®_vcam_2v8>; + vddd-supply = <®_vcam_1v2>; + vddio-supply = <®_csi_1v8>; + rotation = <90>; + orientation = <0>; + + port { + camera1_ep: endpoint { + data-lanes = <1 2>; + link-frequencies = /bits/ 64 + <80000000 200000000 300000000>; + remote-endpoint = <&mipi1_sensor_ep>; + }; + }; + }; + backlight@36 { compatible = "ti,lm36922"; reg = <0x36>; @@ -995,6 +1109,23 @@ &lcdif { status = "okay"; }; +&mipi_csi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + ports { + port@0 { + reg = <0>; + + mipi1_sensor_ep: endpoint { + remote-endpoint = <&camera1_ep>; + data-lanes = <1 2>; + }; + }; + }; +}; + &mipi_dsi { #address-cells = <1>; #size-cells = <0>; @@ -1010,6 +1141,7 @@ lcd_panel: panel@0 { vddi-supply = <®_lcd_1v8>; backlight = <&backlight_dsi>; reset-gpios = <&gpio1 29 GPIO_ACTIVE_LOW>; + mantix,tp-rstn-gpios = <&gpio1 24 GPIO_ACTIVE_LOW>; port { panel_in: endpoint { @@ -1055,13 +1187,13 @@ &pwm2 { &pwm3 { pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_led_g>; + pinctrl-0 = <&pinctrl_led_r>; status = "okay"; }; &pwm4 { pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_led_r>; + pinctrl-0 = <&pinctrl_led_g>; status = "okay"; }; @@ -1199,7 +1331,10 @@ &usdhc2 { pinctrl-2 = <&pinctrl_usdhc2_200mhz>; bus-width = <4>; vmmc-supply = <®_wifi_3v3>; + mmc-pwrseq = <&usdhc2_pwrseq>; + post-power-on-delay-ms = <1000>; cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>; + max-frequency = <50000000>; disable-wp; cap-sdio-irq; keep-power-in-suspend; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts b/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts index 2535268f09..fa721a13de 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts @@ -12,6 +12,7 @@ / { model = "MNT Reform 2"; compatible = "mntre,reform2", "boundary,imx8mq-nitrogen8m-som", "fsl,imx8mq"; + chassis-type = "laptop"; pcie1_refclk: clock-pcie1-refclk { compatible = "fixed-clock"; @@ -202,6 +203,7 @@ MX8MQ_IOMUXC_UART2_TXD_UART2_DCE_TX 0x45 pinctrl_usdhc2: usdhc2grp { fsl,pins = < + MX8MQ_IOMUXC_SD2_CD_B_USDHC2_CD_B 0x0 MX8MQ_IOMUXC_SD2_CLK_USDHC2_CLK 0x83 MX8MQ_IOMUXC_SD2_CMD_USDHC2_CMD 0xc3 MX8MQ_IOMUXC_SD2_DATA0_USDHC2_DATA0 0xc3 diff --git a/arch/arm64/boot/dts/freescale/imx8mq-nitrogen-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-nitrogen-som.dtsi index 36fc428ebe..395f77b5ac 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-nitrogen-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq-nitrogen-som.dtsi @@ -69,6 +69,9 @@ ethphy0: ethernet-phy@4 { reg = <4>; interrupt-parent = <&gpio1>; interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&gpio1 9 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + reset-deassert-us = <300>; }; }; }; @@ -191,20 +194,20 @@ MX8MQ_IOMUXC_ENET_TD1_ENET1_RGMII_TD1 0x1f MX8MQ_IOMUXC_ENET_TD2_ENET1_RGMII_TD2 0x1f MX8MQ_IOMUXC_ENET_TD3_ENET1_RGMII_TD3 0x1f MX8MQ_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL 0x91 - MX8MQ_IOMUXC_ENET_RXC_ENET1_RGMII_RXC 0x91 + MX8MQ_IOMUXC_ENET_RXC_ENET1_RGMII_RXC 0xd1 MX8MQ_IOMUXC_ENET_RD0_ENET1_RGMII_RD0 0x91 MX8MQ_IOMUXC_ENET_RD1_ENET1_RGMII_RD1 0x91 MX8MQ_IOMUXC_ENET_RD2_ENET1_RGMII_RD2 0x91 - MX8MQ_IOMUXC_ENET_RD3_ENET1_RGMII_RD3 0x91 - MX8MQ_IOMUXC_GPIO1_IO09_GPIO1_IO9 0x19 - MX8MQ_IOMUXC_GPIO1_IO11_GPIO1_IO11 0x59 + MX8MQ_IOMUXC_ENET_RD3_ENET1_RGMII_RD3 0xd1 + MX8MQ_IOMUXC_GPIO1_IO09_GPIO1_IO9 0x1 + MX8MQ_IOMUXC_GPIO1_IO11_GPIO1_IO11 0x41 >; }; pinctrl_i2c1: i2c1grp { fsl,pins = < - MX8MQ_IOMUXC_I2C1_SCL_I2C1_SCL 0x4000007f - MX8MQ_IOMUXC_I2C1_SDA_I2C1_SDA 0x4000007f + MX8MQ_IOMUXC_I2C1_SCL_I2C1_SCL 0x40000022 + MX8MQ_IOMUXC_I2C1_SDA_I2C1_SDA 0x40000022 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-zii-ultra.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-zii-ultra.dtsi index a08a568c31..2222ef7b3e 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-zii-ultra.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq-zii-ultra.dtsi @@ -555,6 +555,7 @@ &pcie0 { <&clk IMX8MQ_CLK_PCIE1_PHY>, <&pcie0_refclk>; clock-names = "pcie", "pcie_aux", "pcie_phy", "pcie_bus"; + vph-supply = <&vgen5_reg>; status = "okay"; }; @@ -567,6 +568,7 @@ &pcie1 { <&clk IMX8MQ_CLK_PCIE2_PHY>, <&pcie1_refclk>; clock-names = "pcie", "pcie_aux", "pcie_phy", "pcie_bus"; + vph-supply = <&vgen5_reg>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi index fd38092bb2..e92ebb6147 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi @@ -102,6 +102,12 @@ A53_0: cpu@0 { clock-latency = <61036>; /* two CLK32 periods */ clocks = <&clk IMX8MQ_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; #cooling-cells = <2>; @@ -116,6 +122,12 @@ A53_1: cpu@1 { clock-latency = <61036>; /* two CLK32 periods */ clocks = <&clk IMX8MQ_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; #cooling-cells = <2>; @@ -128,6 +140,12 @@ A53_2: cpu@2 { clock-latency = <61036>; /* two CLK32 periods */ clocks = <&clk IMX8MQ_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; #cooling-cells = <2>; @@ -140,6 +158,12 @@ A53_3: cpu@3 { clock-latency = <61036>; /* two CLK32 periods */ clocks = <&clk IMX8MQ_CLK_ARM>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; operating-points-v2 = <&a53_opp_table>; #cooling-cells = <2>; @@ -147,6 +171,10 @@ A53_3: cpu@3 { A53_L2: l2-cache0 { compatible = "cache"; + cache-level = <2>; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; }; }; @@ -429,49 +457,49 @@ tmu: tmu@30260000 { clocks = <&clk IMX8MQ_CLK_TMU_ROOT>; little-endian; fsl,tmu-range = <0xb0000 0xa0026 0x80048 0x70061>; - fsl,tmu-calibration = <0x00000000 0x00000023 - 0x00000001 0x00000029 - 0x00000002 0x0000002f - 0x00000003 0x00000035 - 0x00000004 0x0000003d - 0x00000005 0x00000043 - 0x00000006 0x0000004b - 0x00000007 0x00000051 - 0x00000008 0x00000057 - 0x00000009 0x0000005f - 0x0000000a 0x00000067 - 0x0000000b 0x0000006f + fsl,tmu-calibration = <0x00000000 0x00000023>, + <0x00000001 0x00000029>, + <0x00000002 0x0000002f>, + <0x00000003 0x00000035>, + <0x00000004 0x0000003d>, + <0x00000005 0x00000043>, + <0x00000006 0x0000004b>, + <0x00000007 0x00000051>, + <0x00000008 0x00000057>, + <0x00000009 0x0000005f>, + <0x0000000a 0x00000067>, + <0x0000000b 0x0000006f>, - 0x00010000 0x0000001b - 0x00010001 0x00000023 - 0x00010002 0x0000002b - 0x00010003 0x00000033 - 0x00010004 0x0000003b - 0x00010005 0x00000043 - 0x00010006 0x0000004b - 0x00010007 0x00000055 - 0x00010008 0x0000005d - 0x00010009 0x00000067 - 0x0001000a 0x00000070 + <0x00010000 0x0000001b>, + <0x00010001 0x00000023>, + <0x00010002 0x0000002b>, + <0x00010003 0x00000033>, + <0x00010004 0x0000003b>, + <0x00010005 0x00000043>, + <0x00010006 0x0000004b>, + <0x00010007 0x00000055>, + <0x00010008 0x0000005d>, + <0x00010009 0x00000067>, + <0x0001000a 0x00000070>, - 0x00020000 0x00000017 - 0x00020001 0x00000023 - 0x00020002 0x0000002d - 0x00020003 0x00000037 - 0x00020004 0x00000041 - 0x00020005 0x0000004b - 0x00020006 0x00000057 - 0x00020007 0x00000063 - 0x00020008 0x0000006f + <0x00020000 0x00000017>, + <0x00020001 0x00000023>, + <0x00020002 0x0000002d>, + <0x00020003 0x00000037>, + <0x00020004 0x00000041>, + <0x00020005 0x0000004b>, + <0x00020006 0x00000057>, + <0x00020007 0x00000063>, + <0x00020008 0x0000006f>, - 0x00030000 0x00000015 - 0x00030001 0x00000021 - 0x00030002 0x0000002d - 0x00030003 0x00000039 - 0x00030004 0x00000045 - 0x00030005 0x00000053 - 0x00030006 0x0000005f - 0x00030007 0x00000071>; + <0x00030000 0x00000015>, + <0x00030001 0x00000021>, + <0x00030002 0x0000002d>, + <0x00030003 0x00000039>, + <0x00030004 0x00000045>, + <0x00030005 0x00000053>, + <0x00030006 0x0000005f>, + <0x00030007 0x00000071>; #thermal-sensor-cells = <1>; }; @@ -1290,7 +1318,6 @@ fec1: ethernet@30be0000 { fsl,num-rx-queues = <3>; nvmem-cells = <&fec_mac_address>; nvmem-cell-names = "mac-address"; - nvmem_macaddr_swap; fsl,stop-mode = <&iomuxc_gpr 0x10 3>; status = "disabled"; }; @@ -1465,10 +1492,9 @@ pcie0: pcie@33800000 { #size-cells = <2>; device_type = "pci"; bus-range = <0x00 0xff>; - ranges = <0x81000000 0 0x00000000 0x1ff80000 0 0x00010000 /* downstream I/O 64KB */ - 0x82000000 0 0x18000000 0x18000000 0 0x07f00000>; /* non-prefetchable memory */ + ranges = <0x81000000 0 0x00000000 0x1ff80000 0 0x00010000>, /* downstream I/O 64KB */ + <0x82000000 0 0x18000000 0x18000000 0 0x07f00000>; /* non-prefetchable memory */ num-lanes = <1>; - num-viewport = <4>; interrupts = ; interrupt-names = "msi"; #interrupt-cells = <1>; @@ -1503,10 +1529,9 @@ pcie1: pcie@33c00000 { #address-cells = <3>; #size-cells = <2>; device_type = "pci"; - ranges = <0x81000000 0 0x00000000 0x27f80000 0 0x00010000 /* downstream I/O 64KB */ - 0x82000000 0 0x20000000 0x20000000 0 0x07f00000>; /* non-prefetchable memory */ + ranges = <0x81000000 0 0x00000000 0x27f80000 0 0x00010000>, /* downstream I/O 64KB */ + <0x82000000 0 0x20000000 0x20000000 0 0x07f00000>; /* non-prefetchable memory */ num-lanes = <1>; - num-viewport = <4>; interrupts = ; interrupt-names = "msi"; #interrupt-cells = <1>; diff --git a/arch/arm64/boot/dts/freescale/imx8qm.dtsi b/arch/arm64/boot/dts/freescale/imx8qm.dtsi index aebbe2b84a..4a7c017b5f 100644 --- a/arch/arm64/boot/dts/freescale/imx8qm.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8qm.dtsi @@ -57,6 +57,12 @@ A53_0: cpu@0 { compatible = "arm,cortex-a53", "arm,armv8"; reg = <0x0 0x0>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; }; @@ -65,6 +71,12 @@ A53_1: cpu@1 { compatible = "arm,cortex-a53", "arm,armv8"; reg = <0x0 0x1>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; }; @@ -73,6 +85,12 @@ A53_2: cpu@2 { compatible = "arm,cortex-a53", "arm,armv8"; reg = <0x0 0x2>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; }; @@ -81,6 +99,12 @@ A53_3: cpu@3 { compatible = "arm,cortex-a53", "arm,armv8"; reg = <0x0 0x3>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A53_L2>; }; @@ -89,6 +113,12 @@ A72_0: cpu@100 { compatible = "arm,cortex-a72", "arm,armv8"; reg = <0x0 0x100>; enable-method = "psci"; + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; next-level-cache = <&A72_L2>; }; @@ -102,10 +132,18 @@ A72_1: cpu@101 { A53_L2: l2-cache0 { compatible = "cache"; + cache-level = <2>; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; }; A72_L2: l2-cache1 { compatible = "cache"; + cache-level = <2>; + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8qxp.dtsi b/arch/arm64/boot/dts/freescale/imx8qxp.dtsi index 617618edf7..dbec7c106e 100644 --- a/arch/arm64/boot/dts/freescale/imx8qxp.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8qxp.dtsi @@ -58,6 +58,12 @@ A35_0: cpu@0 { compatible = "arm,cortex-a35"; reg = <0x0 0x0>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A35_L2>; clocks = <&clk IMX_SC_R_A35 IMX_SC_PM_CLK_CPU>; operating-points-v2 = <&a35_opp_table>; @@ -69,6 +75,12 @@ A35_1: cpu@1 { compatible = "arm,cortex-a35"; reg = <0x0 0x1>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A35_L2>; clocks = <&clk IMX_SC_R_A35 IMX_SC_PM_CLK_CPU>; operating-points-v2 = <&a35_opp_table>; @@ -80,6 +92,12 @@ A35_2: cpu@2 { compatible = "arm,cortex-a35"; reg = <0x0 0x2>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A35_L2>; clocks = <&clk IMX_SC_R_A35 IMX_SC_PM_CLK_CPU>; operating-points-v2 = <&a35_opp_table>; @@ -91,6 +109,12 @@ A35_3: cpu@3 { compatible = "arm,cortex-a35"; reg = <0x0 0x3>; enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; next-level-cache = <&A35_L2>; clocks = <&clk IMX_SC_R_A35 IMX_SC_PM_CLK_CPU>; operating-points-v2 = <&a35_opp_table>; @@ -99,6 +123,10 @@ A35_3: cpu@3 { A35_L2: l2-cache0 { compatible = "cache"; + cache-level = <2>; + cache-size = <0x80000>; + cache-line-size = <64>; + cache-sets = <1024>; }; }; diff --git a/arch/arm64/boot/dts/hisilicon/hi3660.dtsi b/arch/arm64/boot/dts/hisilicon/hi3660.dtsi index 6eabec2602..8bd6d7e8a4 100644 --- a/arch/arm64/boot/dts/hisilicon/hi3660.dtsi +++ b/arch/arm64/boot/dts/hisilicon/hi3660.dtsi @@ -210,7 +210,7 @@ A73_L2: l2-cache1 { }; }; - cluster0_opp: opp_table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -245,7 +245,7 @@ opp04 { }; }; - cluster1_opp: opp_table1 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; diff --git a/arch/arm64/boot/dts/hisilicon/hi3670-hikey970.dts b/arch/arm64/boot/dts/hisilicon/hi3670-hikey970.dts index d8abf442ee..7c32f5fd5c 100644 --- a/arch/arm64/boot/dts/hisilicon/hi3670-hikey970.dts +++ b/arch/arm64/boot/dts/hisilicon/hi3670-hikey970.dts @@ -12,6 +12,7 @@ #include "hi3670.dtsi" #include "hikey970-pinctrl.dtsi" +#include "hikey970-pmic.dtsi" / { model = "HiKey970"; @@ -39,23 +40,6 @@ memory@0 { reg = <0x0 0x0 0x0 0x0>; }; - sd_1v8: regulator-1v8 { - compatible = "regulator-fixed"; - regulator-name = "fixed-1.8V"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-always-on; - }; - - sd_3v3: regulator-3v3 { - compatible = "regulator-fixed"; - regulator-name = "fixed-3.3V"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - regulator-boot-on; - regulator-always-on; - }; - wlan_en: wlan-en-1-8v { compatible = "regulator-fixed"; regulator-name = "wlan-en-regulator"; @@ -402,8 +386,8 @@ &dwmmc1 { pinctrl-0 = <&sd_pmx_func &sd_clk_cfg_func &sd_cfg_func>; - vmmc-supply = <&sd_3v3>; - vqmmc-supply = <&sd_1v8>; + vmmc-supply = <&ldo16>; + vqmmc-supply = <&ldo9>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/hisilicon/hi3670.dtsi b/arch/arm64/boot/dts/hisilicon/hi3670.dtsi index 20698cfd06..636c8817df 100644 --- a/arch/arm64/boot/dts/hisilicon/hi3670.dtsi +++ b/arch/arm64/boot/dts/hisilicon/hi3670.dtsi @@ -669,7 +669,7 @@ ufs: ufs@ff3c0000 { interrupt-parent = <&gic>; interrupts = ; clocks = <&crg_ctrl HI3670_CLK_GATE_UFSIO_REF>, - <&crg_ctrl HI3670_CLK_GATE_UFS_SUBSYS>; + <&crg_ctrl HI3670_CLK_GATE_UFS_SUBSYS>; clock-names = "ref_clk", "phy_clk"; freq-table-hz = <0 0 0 0>; diff --git a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi index e4860b8a63..ae0a7cfeeb 100644 --- a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi +++ b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi @@ -193,7 +193,7 @@ CLUSTER1_L2: l2-cache1 { }; }; - cpu_opp_table: cpu_opp_table { + cpu_opp_table: opp-table-0 { compatible = "operating-points-v2"; opp-shared; diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi index de1e98c99e..f4270cf189 100644 --- a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi +++ b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi @@ -628,7 +628,7 @@ sdmmca-ecc@ff8c8c00 { }; qspi: spi@ff8d2000 { - compatible = "cdns,qspi-nor"; + compatible = "intel,socfpga-qspi", "cdns,qspi-nor"; #address-cells = <1>; #size-cells = <0>; reg = <0xff8d2000 0x100>, diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex_socdk_nand.dts b/arch/arm64/boot/dts/intel/socfpga_agilex_socdk_nand.dts index cc2dcabf34..57f83481f5 100644 --- a/arch/arm64/boot/dts/intel/socfpga_agilex_socdk_nand.dts +++ b/arch/arm64/boot/dts/intel/socfpga_agilex_socdk_nand.dts @@ -91,24 +91,8 @@ partition@0 { reg = <0 0x200000>; }; partition@200000 { - label = "env"; - reg = <0x200000 0x40000>; - }; - partition@240000 { - label = "dtb"; - reg = <0x240000 0x40000>; - }; - partition@280000 { - label = "kernel"; - reg = <0x280000 0x2000000>; - }; - partition@2280000 { - label = "misc"; - reg = <0x2280000 0x2000000>; - }; - partition@4280000 { - label = "rootfs"; - reg = <0x4280000 0x3bd80000>; + label = "root"; + reg = <0x200000 0x3fe00000>; }; }; }; diff --git a/arch/arm64/boot/dts/intel/socfpga_n5x_socdk.dts b/arch/arm64/boot/dts/intel/socfpga_n5x_socdk.dts index 01f1307ce4..f3c1310dae 100644 --- a/arch/arm64/boot/dts/intel/socfpga_n5x_socdk.dts +++ b/arch/arm64/boot/dts/intel/socfpga_n5x_socdk.dts @@ -29,6 +29,36 @@ &clkmgr { compatible = "intel,easic-n5x-clkmgr"; }; +&gmac0 { + status = "okay"; + phy-mode = "rgmii"; + phy-handle = <&phy0>; + + max-frame-size = <9000>; + + mdio0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dwmac-mdio"; + phy0: ethernet-phy@0 { + reg = <4>; + + txd0-skew-ps = <0>; /* -420ps */ + txd1-skew-ps = <0>; /* -420ps */ + txd2-skew-ps = <0>; /* -420ps */ + txd3-skew-ps = <0>; /* -420ps */ + rxd0-skew-ps = <420>; /* 0ps */ + rxd1-skew-ps = <420>; /* 0ps */ + rxd2-skew-ps = <420>; /* 0ps */ + rxd3-skew-ps = <420>; /* 0ps */ + txen-skew-ps = <0>; /* -420ps */ + txc-skew-ps = <900>; /* 0ps */ + rxdv-skew-ps = <420>; /* 0ps */ + rxc-skew-ps = <1680>; /* 780ps */ + }; + }; +}; + &mmc { status = "okay"; cap-sd-highspeed; @@ -40,10 +70,51 @@ &osc1 { clock-frequency = <25000000>; }; +&qspi { + status = "okay"; + flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "micron,mt25qu02g", "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <100000000>; + + m25p,fast-read; + cdns,page-size = <256>; + cdns,block-size = <16>; + cdns,read-delay = <2>; + cdns,tshsl-ns = <50>; + cdns,tsd2d-ns = <50>; + cdns,tchsh-ns = <4>; + cdns,tslch-ns = <4>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + qspi_boot: partition@0 { + label = "Boot and fpga data"; + reg = <0x0 0x03FE0000>; + }; + + qspi_rootfs: partition@3FE0000 { + label = "Root Filesystem - JFFS2"; + reg = <0x03FE0000 0x0C020000>; + }; + }; + }; +}; + &uart0 { status = "okay"; }; +&usb0 { + status = "okay"; + disable-over-current; +}; + &watchdog0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/marvell/Makefile b/arch/arm64/boot/dts/marvell/Makefile index c686a8dd3c..1c794cdcb8 100644 --- a/arch/arm64/boot/dts/marvell/Makefile +++ b/arch/arm64/boot/dts/marvell/Makefile @@ -9,6 +9,7 @@ dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-espressobin-v7-emmc.dtb dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-turris-mox.dtb dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-uDPU.dtb dtb-$(CONFIG_ARCH_MVEBU) += armada-7040-db.dtb +dtb-$(CONFIG_ARCH_MVEBU) += armada-7040-mochabin.dtb dtb-$(CONFIG_ARCH_MVEBU) += armada-8040-clearfog-gt-8k.dtb dtb-$(CONFIG_ARCH_MVEBU) += armada-8040-db.dtb dtb-$(CONFIG_ARCH_MVEBU) += armada-8040-mcbin.dtb diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi index 9acc5d2b5a..fb78ef613b 100644 --- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi +++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi @@ -265,6 +265,8 @@ comphy: phy@18300 { "lane2_sata_usb3"; #address-cells = <1>; #size-cells = <0>; + clocks = <&xtalclk>; + clock-names = "xtal"; comphy0: phy@0 { reg = <0>; @@ -497,7 +499,7 @@ pcie0: pcie@d0070000 { * (totaling 127 MiB) for MEM. */ ranges = <0x82000000 0 0xe8000000 0 0xe8000000 0 0x07f00000 /* Port 0 MEM */ - 0x81000000 0 0xefff0000 0 0xefff0000 0 0x00010000>; /* Port 0 IO */ + 0x81000000 0 0x00000000 0 0xefff0000 0 0x00010000>; /* Port 0 IO */ interrupt-map-mask = <0 0 0 7>; interrupt-map = <0 0 0 1 &pcie_intc 0>, <0 0 0 2 &pcie_intc 1>, diff --git a/arch/arm64/boot/dts/marvell/cn9130-crb.dtsi b/arch/arm64/boot/dts/marvell/cn9130-crb.dtsi index 505ae69289..d9f9f2c197 100644 --- a/arch/arm64/boot/dts/marvell/cn9130-crb.dtsi +++ b/arch/arm64/boot/dts/marvell/cn9130-crb.dtsi @@ -17,6 +17,8 @@ aliases { ethernet0 = &cp0_eth0; ethernet1 = &cp0_eth1; ethernet2 = &cp0_eth2; + gpio1 = &cp0_gpio1; + gpio2 = &cp0_gpio2; }; memory@0 { @@ -71,6 +73,17 @@ cp0_reg_sd_vcc: cp0_sd_vcc@0 { enable-active-high; regulator-always-on; }; + + sfp: sfp { + compatible = "sff,sfp"; + i2c-bus = <&cp0_i2c1>; + mod-def0-gpios = <&expander0 3 GPIO_ACTIVE_LOW>; + los-gpio = <&expander0 15 GPIO_ACTIVE_HIGH>; + tx-disable-gpio = <&expander0 2 GPIO_ACTIVE_HIGH>; + tx-fault-gpio = <&cp0_gpio1 24 GPIO_ACTIVE_HIGH>; + maximum-power-milliwatt = <3000>; + status = "okay"; + }; }; &uart0 { @@ -114,6 +127,14 @@ cp0_spi0_pins: cp0-spi-pins-0 { }; }; +&cp0_gpio1 { + status = "okay"; +}; + +&cp0_gpio2 { + status = "okay"; +}; + &cp0_i2c0 { pinctrl-names = "default"; pinctrl-0 = <&cp0_i2c0_pins>; @@ -185,6 +206,125 @@ &cp0_mdio { phy0: ethernet-phy@0 { reg = <0>; }; + + switch6: switch0@6 { + /* Actual device is MV88E6393X */ + compatible = "marvell,mv88e6190"; + #address-cells = <1>; + #size-cells = <0>; + reg = <6>; + interrupt-parent = <&cp0_gpio1>; + interrupts = <28 IRQ_TYPE_LEVEL_LOW>; + interrupt-controller; + #interrupt-cells = <2>; + + dsa,member = <0 0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + label = "p1"; + phy-handle = <&switch0phy1>; + }; + + port@2 { + reg = <2>; + label = "p2"; + phy-handle = <&switch0phy2>; + }; + + port@3 { + reg = <3>; + label = "p3"; + phy-handle = <&switch0phy3>; + }; + + port@4 { + reg = <4>; + label = "p4"; + phy-handle = <&switch0phy4>; + }; + + port@5 { + reg = <5>; + label = "p5"; + phy-handle = <&switch0phy5>; + }; + + port@6 { + reg = <6>; + label = "p6"; + phy-handle = <&switch0phy6>; + }; + + port@7 { + reg = <7>; + label = "p7"; + phy-handle = <&switch0phy7>; + }; + + port@8 { + reg = <8>; + label = "p8"; + phy-handle = <&switch0phy8>; + }; + + port@9 { + reg = <9>; + label = "p9"; + phy-mode = "10gbase-r"; + sfp = <&sfp>; + managed = "in-band-status"; + }; + + port@a { + reg = <10>; + label = "cpu"; + ethernet = <&cp0_eth0>; + }; + + }; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + switch0phy1: switch0phy1@1 { + reg = <0x1>; + }; + + switch0phy2: switch0phy2@2 { + reg = <0x2>; + }; + + switch0phy3: switch0phy3@3 { + reg = <0x3>; + }; + + switch0phy4: switch0phy4@4 { + reg = <0x4>; + }; + + switch0phy5: switch0phy5@5 { + reg = <0x5>; + }; + + switch0phy6: switch0phy6@6 { + reg = <0x6>; + }; + + switch0phy7: switch0phy7@7 { + reg = <0x7>; + }; + + switch0phy8: switch0phy8@8 { + reg = <0x8>; + }; + }; + }; }; &cp0_xmdio { diff --git a/arch/arm64/boot/dts/mediatek/Makefile b/arch/arm64/boot/dts/mediatek/Makefile index 4f68ebed2e..8c1e18032f 100644 --- a/arch/arm64/boot/dts/mediatek/Makefile +++ b/arch/arm64/boot/dts/mediatek/Makefile @@ -7,6 +7,8 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += mt6797-evb.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt6797-x20-dev.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt7622-rfb1.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt7622-bananapi-bpi-r64.dtb +dtb-$(CONFIG_ARCH_MEDIATEK) += mt7986a-rfb.dtb +dtb-$(CONFIG_ARCH_MEDIATEK) += mt7986b-rfb.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8167-pumpkin.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8173-elm.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8173-elm-hana.dtb @@ -14,16 +16,20 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += mt8173-elm-hana-rev7.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8173-evb.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-evb.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-burnet.dtb +dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-cozmo.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-damu.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-fennel-sku1.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-fennel-sku6.dtb +dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-fennel-sku7.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-fennel14.dtb +dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-fennel14-sku2.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-juniper-sku16.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-kappa.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-kenzo.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-willow-sku0.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-jacuzzi-willow-sku1.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-kakadu.dtb +dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-kakadu-sku22.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-kodama-sku16.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-kodama-sku272.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-kodama-sku288.dtb diff --git a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi index a9cca9c146..de16c0d80c 100644 --- a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi @@ -915,64 +915,67 @@ u3port1: usb-phy@8700 { }; }; - pcie: pcie@11700000 { + pcie1: pcie@112ff000 { compatible = "mediatek,mt2712-pcie"; device_type = "pci"; - reg = <0 0x11700000 0 0x1000>, - <0 0x112ff000 0 0x1000>; - reg-names = "port0", "port1"; + reg = <0 0x112ff000 0 0x1000>; + reg-names = "port1"; + linux,pci-domain = <1>; #address-cells = <3>; #size-cells = <2>; - interrupts = , - ; - clocks = <&topckgen CLK_TOP_PE2_MAC_P0_SEL>, - <&topckgen CLK_TOP_PE2_MAC_P1_SEL>, - <&pericfg CLK_PERI_PCIE0>, + interrupts = ; + interrupt-names = "pcie_irq"; + clocks = <&topckgen CLK_TOP_PE2_MAC_P1_SEL>, <&pericfg CLK_PERI_PCIE1>; - clock-names = "sys_ck0", "sys_ck1", "ahb_ck0", "ahb_ck1"; - phys = <&u3port0 PHY_TYPE_PCIE>, <&u3port1 PHY_TYPE_PCIE>; - phy-names = "pcie-phy0", "pcie-phy1"; + clock-names = "sys_ck1", "ahb_ck1"; + phys = <&u3port1 PHY_TYPE_PCIE>; + phy-names = "pcie-phy1"; bus-range = <0x00 0xff>; - ranges = <0x82000000 0 0x20000000 0x0 0x20000000 0 0x10000000>; + ranges = <0x82000000 0 0x11400000 0x0 0x11400000 0 0x300000>; + status = "disabled"; - pcie0: pcie@0,0 { - device_type = "pci"; - status = "disabled"; - reg = <0x0000 0 0 0 0>; - #address-cells = <3>; - #size-cells = <2>; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc1 0>, + <0 0 0 2 &pcie_intc1 1>, + <0 0 0 3 &pcie_intc1 2>, + <0 0 0 4 &pcie_intc1 3>; + pcie_intc1: interrupt-controller { + interrupt-controller; + #address-cells = <0>; #interrupt-cells = <1>; - ranges; - interrupt-map-mask = <0 0 0 7>; - interrupt-map = <0 0 0 1 &pcie_intc0 0>, - <0 0 0 2 &pcie_intc0 1>, - <0 0 0 3 &pcie_intc0 2>, - <0 0 0 4 &pcie_intc0 3>; - pcie_intc0: interrupt-controller { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <1>; - }; }; + }; - pcie1: pcie@1,0 { - device_type = "pci"; - status = "disabled"; - reg = <0x0800 0 0 0 0>; - #address-cells = <3>; - #size-cells = <2>; + pcie0: pcie@11700000 { + compatible = "mediatek,mt2712-pcie"; + device_type = "pci"; + reg = <0 0x11700000 0 0x1000>; + reg-names = "port0"; + linux,pci-domain = <0>; + #address-cells = <3>; + #size-cells = <2>; + interrupts = ; + interrupt-names = "pcie_irq"; + clocks = <&topckgen CLK_TOP_PE2_MAC_P0_SEL>, + <&pericfg CLK_PERI_PCIE0>; + clock-names = "sys_ck0", "ahb_ck0"; + phys = <&u3port0 PHY_TYPE_PCIE>; + phy-names = "pcie-phy0"; + bus-range = <0x00 0xff>; + ranges = <0x82000000 0 0x20000000 0x0 0x20000000 0 0x10000000>; + status = "disabled"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc0 0>, + <0 0 0 2 &pcie_intc0 1>, + <0 0 0 3 &pcie_intc0 2>, + <0 0 0 4 &pcie_intc0 3>; + pcie_intc0: interrupt-controller { + interrupt-controller; + #address-cells = <0>; #interrupt-cells = <1>; - ranges; - interrupt-map-mask = <0 0 0 7>; - interrupt-map = <0 0 0 1 &pcie_intc1 0>, - <0 0 0 2 &pcie_intc1 1>, - <0 0 0 3 &pcie_intc1 2>, - <0 0 0 4 &pcie_intc1 3>; - pcie_intc1: interrupt-controller { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <1>; - }; }; }; diff --git a/arch/arm64/boot/dts/mediatek/mt6358.dtsi b/arch/arm64/boot/dts/mediatek/mt6358.dtsi index fa159b2037..95145076b7 100644 --- a/arch/arm64/boot/dts/mediatek/mt6358.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt6358.dtsi @@ -13,6 +13,7 @@ pmic: mt6358 { mt6358codec: mt6358codec { compatible = "mediatek,mt6358-sound"; + mediatek,dmic-mode = <0>; /* two-wires */ }; mt6358regulator: mt6358regulator { diff --git a/arch/arm64/boot/dts/mediatek/mt7622-bananapi-bpi-r64.dts b/arch/arm64/boot/dts/mediatek/mt7622-bananapi-bpi-r64.dts index 2f77dc40b9..2b9bf8dd14 100644 --- a/arch/arm64/boot/dts/mediatek/mt7622-bananapi-bpi-r64.dts +++ b/arch/arm64/boot/dts/mediatek/mt7622-bananapi-bpi-r64.dts @@ -257,18 +257,16 @@ flash@0 { }; }; -&pcie { +&pcie0 { pinctrl-names = "default"; - pinctrl-0 = <&pcie0_pins>, <&pcie1_pins>; + pinctrl-0 = <&pcie0_pins>; status = "okay"; +}; - pcie@0,0 { - status = "okay"; - }; - - pcie@1,0 { - status = "okay"; - }; +&pcie1 { + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_pins>; + status = "okay"; }; &pio { diff --git a/arch/arm64/boot/dts/mediatek/mt7622-rfb1.dts b/arch/arm64/boot/dts/mediatek/mt7622-rfb1.dts index f2dc850010..596c073d8b 100644 --- a/arch/arm64/boot/dts/mediatek/mt7622-rfb1.dts +++ b/arch/arm64/boot/dts/mediatek/mt7622-rfb1.dts @@ -234,14 +234,10 @@ flash@0 { }; }; -&pcie { +&pcie0 { pinctrl-names = "default"; pinctrl-0 = <&pcie0_pins>; status = "okay"; - - pcie@0,0 { - status = "okay"; - }; }; &pio { diff --git a/arch/arm64/boot/dts/mediatek/mt7622.dtsi b/arch/arm64/boot/dts/mediatek/mt7622.dtsi index 890a942ec6..6f8cb3ad1e 100644 --- a/arch/arm64/boot/dts/mediatek/mt7622.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt7622.dtsi @@ -781,75 +781,83 @@ pciesys: pciesys@1a100800 { #reset-cells = <1>; }; - pcie: pcie@1a140000 { + pciecfg: pciecfg@1a140000 { + compatible = "mediatek,generic-pciecfg", "syscon"; + reg = <0 0x1a140000 0 0x1000>; + }; + + pcie0: pcie@1a143000 { compatible = "mediatek,mt7622-pcie"; device_type = "pci"; - reg = <0 0x1a140000 0 0x1000>, - <0 0x1a143000 0 0x1000>, - <0 0x1a145000 0 0x1000>; - reg-names = "subsys", "port0", "port1"; + reg = <0 0x1a143000 0 0x1000>; + reg-names = "port0"; + linux,pci-domain = <0>; #address-cells = <3>; #size-cells = <2>; - interrupts = , - ; + interrupts = ; + interrupt-names = "pcie_irq"; clocks = <&pciesys CLK_PCIE_P0_MAC_EN>, - <&pciesys CLK_PCIE_P1_MAC_EN>, - <&pciesys CLK_PCIE_P0_AHB_EN>, <&pciesys CLK_PCIE_P0_AHB_EN>, <&pciesys CLK_PCIE_P0_AUX_EN>, - <&pciesys CLK_PCIE_P1_AUX_EN>, <&pciesys CLK_PCIE_P0_AXI_EN>, - <&pciesys CLK_PCIE_P1_AXI_EN>, <&pciesys CLK_PCIE_P0_OBFF_EN>, - <&pciesys CLK_PCIE_P1_OBFF_EN>, - <&pciesys CLK_PCIE_P0_PIPE_EN>, - <&pciesys CLK_PCIE_P1_PIPE_EN>; - clock-names = "sys_ck0", "sys_ck1", "ahb_ck0", "ahb_ck1", - "aux_ck0", "aux_ck1", "axi_ck0", "axi_ck1", - "obff_ck0", "obff_ck1", "pipe_ck0", "pipe_ck1"; + <&pciesys CLK_PCIE_P0_PIPE_EN>; + clock-names = "sys_ck0", "ahb_ck0", "aux_ck0", + "axi_ck0", "obff_ck0", "pipe_ck0"; + power-domains = <&scpsys MT7622_POWER_DOMAIN_HIF0>; bus-range = <0x00 0xff>; - ranges = <0x82000000 0 0x20000000 0x0 0x20000000 0 0x10000000>; + ranges = <0x82000000 0 0x20000000 0x0 0x20000000 0 0x8000000>; status = "disabled"; - pcie0: pcie@0,0 { - reg = <0x0000 0 0 0 0>; - #address-cells = <3>; - #size-cells = <2>; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc0 0>, + <0 0 0 2 &pcie_intc0 1>, + <0 0 0 3 &pcie_intc0 2>, + <0 0 0 4 &pcie_intc0 3>; + pcie_intc0: interrupt-controller { + interrupt-controller; + #address-cells = <0>; #interrupt-cells = <1>; - ranges; - status = "disabled"; - - interrupt-map-mask = <0 0 0 7>; - interrupt-map = <0 0 0 1 &pcie_intc0 0>, - <0 0 0 2 &pcie_intc0 1>, - <0 0 0 3 &pcie_intc0 2>, - <0 0 0 4 &pcie_intc0 3>; - pcie_intc0: interrupt-controller { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <1>; - }; }; + }; - pcie1: pcie@1,0 { - reg = <0x0800 0 0 0 0>; - #address-cells = <3>; - #size-cells = <2>; + pcie1: pcie@1a145000 { + compatible = "mediatek,mt7622-pcie"; + device_type = "pci"; + reg = <0 0x1a145000 0 0x1000>; + reg-names = "port1"; + linux,pci-domain = <1>; + #address-cells = <3>; + #size-cells = <2>; + interrupts = ; + interrupt-names = "pcie_irq"; + clocks = <&pciesys CLK_PCIE_P1_MAC_EN>, + /* designer has connect RC1 with p0_ahb clock */ + <&pciesys CLK_PCIE_P0_AHB_EN>, + <&pciesys CLK_PCIE_P1_AUX_EN>, + <&pciesys CLK_PCIE_P1_AXI_EN>, + <&pciesys CLK_PCIE_P1_OBFF_EN>, + <&pciesys CLK_PCIE_P1_PIPE_EN>; + clock-names = "sys_ck1", "ahb_ck1", "aux_ck1", + "axi_ck1", "obff_ck1", "pipe_ck1"; + + power-domains = <&scpsys MT7622_POWER_DOMAIN_HIF0>; + bus-range = <0x00 0xff>; + ranges = <0x82000000 0 0x28000000 0x0 0x28000000 0 0x8000000>; + status = "disabled"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc1 0>, + <0 0 0 2 &pcie_intc1 1>, + <0 0 0 3 &pcie_intc1 2>, + <0 0 0 4 &pcie_intc1 3>; + pcie_intc1: interrupt-controller { + interrupt-controller; + #address-cells = <0>; #interrupt-cells = <1>; - ranges; - status = "disabled"; - - interrupt-map-mask = <0 0 0 7>; - interrupt-map = <0 0 0 1 &pcie_intc1 0>, - <0 0 0 2 &pcie_intc1 1>, - <0 0 0 3 &pcie_intc1 2>, - <0 0 0 4 &pcie_intc1 3>; - pcie_intc1: interrupt-controller { - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <1>; - }; }; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8173-elm.dtsi b/arch/arm64/boot/dts/mediatek/mt8173-elm.dtsi index e666ebb289..9c75fbb31f 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173-elm.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8173-elm.dtsi @@ -28,7 +28,7 @@ backlight: backlight { enable-gpios = <&pio 95 GPIO_ACTIVE_HIGH>; pinctrl-names = "default"; - pinctrl-0 = <&disp_pwm0_pins>; + pinctrl-0 = <&panel_backlight_en_pins>; status = "okay"; }; @@ -827,6 +827,12 @@ pins_clk { }; }; + panel_backlight_en_pins: panel_backlight_en_pins { + pins1 { + pinmux = ; + }; + }; + panel_fixed_pins: panel_fixed_pins { pins1 { pinmux = ; @@ -901,6 +907,8 @@ pins1 { }; &pwm0 { + pinctrl-names = "default"; + pinctrl-0 = <&disp_pwm0_pins>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi index d9e005ae5b..2b7d331a45 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi @@ -996,6 +996,7 @@ mmsys: syscon@14000000 { assigned-clocks = <&topckgen CLK_TOP_MM_SEL>; assigned-clock-rates = <400000000>; #clock-cells = <1>; + #reset-cells = <1>; mboxes = <&gce 0 CMDQ_THR_PRIO_HIGHEST>, <&gce 1 CMDQ_THR_PRIO_HIGHEST>; mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0 0x1000>; @@ -1211,6 +1212,7 @@ ufoe@1401a000 { interrupts = ; power-domains = <&spm MT8173_POWER_DOMAIN_MM>; clocks = <&mmsys CLK_MM_DISP_UFOE>; + mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0xa000 0x1000>; }; dsi0: dsi@1401b000 { @@ -1222,6 +1224,7 @@ dsi0: dsi@1401b000 { <&mmsys CLK_MM_DSI0_DIGITAL>, <&mipi_tx0>; clock-names = "engine", "digital", "hs"; + resets = <&mmsys MT8173_MMSYS_SW0_RST_B_DISP_DSI0>; phys = <&mipi_tx0>; phy-names = "dphy"; status = "disabled"; @@ -1314,6 +1317,7 @@ od@14023000 { compatible = "mediatek,mt8173-disp-od"; reg = <0 0x14023000 0 0x1000>; clocks = <&mmsys CLK_MM_DISP_OD>; + mediatek,gce-client-reg = <&gce SUBSYS_1402XXXX 0x3000 0x1000>; }; hdmi0: hdmi@14025000 { diff --git a/arch/arm64/boot/dts/mediatek/mt8183-evb.dts b/arch/arm64/boot/dts/mediatek/mt8183-evb.dts index 7bc0a6a7fa..f3fd3cca23 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-evb.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-evb.dts @@ -36,6 +36,14 @@ scp_mem_reserved: scp_mem_region { no-map; }; }; + + ntc@0 { + compatible = "murata,ncp03wf104"; + pullup-uv = <1800000>; + pullup-ohm = <390000>; + pulldown-ohm = <0>; + io-channels = <&auxadc 0>; + }; }; &auxadc { diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-burnet.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-burnet.dts index a8d6f32ade..1a2ec0787d 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-burnet.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-burnet.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi.dtsi" +#include "mt8183-kukui-audio-ts3a227e-max98357a.dtsi" / { model = "Google burnet board"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-damu.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-damu.dts index 42ba9c0086..0eca3ff867 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-damu.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-damu.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi.dtsi" +#include "mt8183-kukui-audio-da7219-max98357a.dtsi" / { model = "Google damu board"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel-sku1.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel-sku1.dts index ef6257c9a2..dec11a4eb5 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel-sku1.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel-sku1.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi-fennel.dtsi" +#include "mt8183-kukui-audio-da7219-rt1015p.dtsi" / { model = "Google fennel sku1 board"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel-sku6.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel-sku6.dts index 899c2e4238..37e6e58f63 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel-sku6.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel-sku6.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi-fennel.dtsi" +#include "mt8183-kukui-audio-da7219-rt1015p.dtsi" / { model = "Google fennel sku6 board"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel14.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel14.dts index e8c41f6b4b..23ad0b91e9 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel14.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-fennel14.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi-fennel.dtsi" +#include "mt8183-kukui-audio-da7219-rt1015p.dtsi" / { model = "Google fennel14 sku0 board"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-juniper-sku16.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-juniper-sku16.dts index 36d2c3b3ca..bc2c57f0a8 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-juniper-sku16.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-juniper-sku16.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi-juniper.dtsi" +#include "mt8183-kukui-audio-ts3a227e-max98357a.dtsi" / { model = "Google juniper sku16 board"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-kappa.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-kappa.dts index b3f46c16e5..e5bd9191e4 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-kappa.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-kappa.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi.dtsi" +#include "mt8183-kukui-audio-ts3a227e-max98357a.dtsi" / { model = "Google kappa board"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-kenzo.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-kenzo.dts index 6f1aa69275..8fa89db03e 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-kenzo.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-kenzo.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi-juniper.dtsi" +#include "mt8183-kukui-audio-ts3a227e-max98357a.dtsi" / { model = "Google kenzo sku17 board"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-willow-sku0.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-willow-sku0.dts index 281265f082..89208b843b 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-willow-sku0.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-willow-sku0.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi-willow.dtsi" +#include "mt8183-kukui-audio-da7219-max98357a.dtsi" / { model = "Google willow board sku0"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-willow-sku1.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-willow-sku1.dts index 22e56bdc1e..c7b20441d0 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-willow-sku1.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-willow-sku1.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-jacuzzi-willow.dtsi" +#include "mt8183-kukui-audio-ts3a227e-max98357a.dtsi" / { model = "Google willow board sku1"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi.dtsi b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi.dtsi index d8826c82bc..8f7bf33f60 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi.dtsi @@ -9,7 +9,6 @@ / { panel: panel { compatible = "auo,b116xw03"; power-supply = <&pp3300_panel>; - ddc-i2c-bus = <&i2c4>; backlight = <&backlight_lcd0>; port { diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-kakadu.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-kakadu.dts index 20eb0dc68f..89a139a0ee 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-kakadu.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-kakadu.dts @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui-kakadu.dtsi" +#include "mt8183-kukui-audio-da7219-rt1015p.dtsi" / { model = "MediaTek kakadu board"; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-kodama.dtsi b/arch/arm64/boot/dts/mediatek/mt8183-kukui-kodama.dtsi index 3aa79403c0..06f8c80bf5 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-kodama.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-kodama.dtsi @@ -5,6 +5,7 @@ /dts-v1/; #include "mt8183-kukui.dtsi" +#include "mt8183-kukui-audio-ts3a227e-max98357a.dtsi" / { ppvarn_lcd: ppvarn-lcd { diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-krane.dtsi b/arch/arm64/boot/dts/mediatek/mt8183-kukui-krane.dtsi index 30c183c96a..a7b0cb3ff7 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-krane.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-krane.dtsi @@ -4,6 +4,7 @@ */ #include "mt8183-kukui.dtsi" +#include "mt8183-kukui-audio-max98357a.dtsi" / { ppvarn_lcd: ppvarn-lcd { @@ -345,3 +346,7 @@ rst_pin { &qca_wifi { qcom,ath10k-calibration-variant = "LE_Krane"; }; + +&sound { + compatible = "mediatek,mt8183_mt6358_ts3a227_max98357"; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi b/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi index 8e9cf36a9a..0f9480f912 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi @@ -115,12 +115,18 @@ scp_mem_reserved: scp_mem_region { }; }; - max98357a: codec0 { - compatible = "maxim,max98357a"; - sdmode-gpios = <&pio 175 0>; + sound: mt8183-sound { + mediatek,platform = <&afe>; + pinctrl-names = "default", + "aud_tdm_out_on", + "aud_tdm_out_off"; + pinctrl-0 = <&aud_pins_default>; + pinctrl-1 = <&aud_pins_tdm_out_on>; + pinctrl-2 = <&aud_pins_tdm_out_off>; + status = "okay"; }; - btsco: codec1 { + btsco: bt-sco { compatible = "linux,bt-sco"; }; @@ -151,33 +157,33 @@ tboard_thermistor1: thermal-sensor1 { #thermal-sensor-cells = <0>; io-channels = <&auxadc 0>; io-channel-names = "sensor-channel"; - temperature-lookup-table = < (-5000) 4241 - 0 4063 - 5000 3856 - 10000 3621 - 15000 3364 - 20000 3091 - 25000 2810 - 30000 2526 - 35000 2247 - 40000 1982 - 45000 1734 - 50000 1507 - 55000 1305 - 60000 1122 - 65000 964 - 70000 827 - 75000 710 - 80000 606 - 85000 519 - 90000 445 - 95000 382 - 100000 330 - 105000 284 - 110000 245 - 115000 213 - 120000 183 - 125000 161>; + temperature-lookup-table = < (-5000) 1553 + 0 1488 + 5000 1412 + 10000 1326 + 15000 1232 + 20000 1132 + 25000 1029 + 30000 925 + 35000 823 + 40000 726 + 45000 635 + 50000 552 + 55000 478 + 60000 411 + 65000 353 + 70000 303 + 75000 260 + 80000 222 + 85000 190 + 90000 163 + 95000 140 + 100000 121 + 105000 104 + 110000 90 + 115000 78 + 120000 67 + 125000 59>; }; tboard_thermistor2: thermal-sensor2 { @@ -185,36 +191,41 @@ tboard_thermistor2: thermal-sensor2 { #thermal-sensor-cells = <0>; io-channels = <&auxadc 1>; io-channel-names = "sensor-channel"; - temperature-lookup-table = < (-5000) 4241 - 0 4063 - 5000 3856 - 10000 3621 - 15000 3364 - 20000 3091 - 25000 2810 - 30000 2526 - 35000 2247 - 40000 1982 - 45000 1734 - 50000 1507 - 55000 1305 - 60000 1122 - 65000 964 - 70000 827 - 75000 710 - 80000 606 - 85000 519 - 90000 445 - 95000 382 - 100000 330 - 105000 284 - 110000 245 - 115000 213 - 120000 183 - 125000 161>; + temperature-lookup-table = < (-5000) 1553 + 0 1488 + 5000 1412 + 10000 1326 + 15000 1232 + 20000 1132 + 25000 1029 + 30000 925 + 35000 823 + 40000 726 + 45000 635 + 50000 552 + 55000 478 + 60000 411 + 65000 353 + 70000 303 + 75000 260 + 80000 222 + 85000 190 + 90000 163 + 95000 140 + 100000 121 + 105000 104 + 110000 90 + 115000 78 + 120000 67 + 125000 59>; }; }; +&afe { + i2s3-share = "I2S2"; + i2s0-share = "I2S5"; +}; + &auxadc { status = "okay"; }; @@ -402,6 +413,54 @@ &mt6358_vsim2_reg { }; &pio { + aud_pins_default: audiopins { + pins_bus { + pinmux = , + , + , + , + , /*i2s to da7219/max98357*/ + , + , + , + , /*i2s to wifi/bt*/ + , + , + , + , + , + , + , + ; /*mtkaif3.0*/ + }; + }; + + aud_pins_tdm_out_on: audiotdmouton { + pins_bus { + pinmux = , + , + , + , + , + ; /*8ch-i2s to it6505*/ + drive-strength = ; + }; + }; + + aud_pins_tdm_out_off: audiotdmoutoff { + pins_bus { + pinmux = , + , + , + , + , + ; + input-enable; + bias-pull-down; + drive-strength = ; + }; + }; + bt_pins: bt-pins { pins_bt_en { pinmux = ; @@ -754,7 +813,7 @@ &scp { cros_ec { compatible = "google,cros-ec-rpmsg"; - mtk,rpmsg-name = "cros-ec-rpmsg"; + mediatek,rpmsg-name = "cros-ec-rpmsg"; }; }; @@ -827,6 +886,20 @@ usbc_extcon: extcon0 { cbas { compatible = "google,cros-cbas"; }; + + typec { + compatible = "google,cros-ec-typec"; + #address-cells = <1>; + #size-cells = <0>; + + usb_c0: connector@0 { + compatible = "usb-c-connector"; + reg = <0>; + power-role = "dual"; + data-role = "host"; + try-power-role = "sink"; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8183.dtsi b/arch/arm64/boot/dts/mediatek/mt8183.dtsi index 409cf82797..00f2ddd245 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183.dtsi @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include @@ -367,6 +367,70 @@ sysirq: interrupt-controller@c530a80 { reg = <0 0x0c530a80 0 0x50>; }; + cpu_debug0: cpu-debug@d410000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0x0 0xd410000 0x0 0x1000>; + clocks = <&infracfg CLK_INFRA_DEBUGSYS>; + clock-names = "apb_pclk"; + cpu = <&cpu0>; + }; + + cpu_debug1: cpu-debug@d510000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0x0 0xd510000 0x0 0x1000>; + clocks = <&infracfg CLK_INFRA_DEBUGSYS>; + clock-names = "apb_pclk"; + cpu = <&cpu1>; + }; + + cpu_debug2: cpu-debug@d610000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0x0 0xd610000 0x0 0x1000>; + clocks = <&infracfg CLK_INFRA_DEBUGSYS>; + clock-names = "apb_pclk"; + cpu = <&cpu2>; + }; + + cpu_debug3: cpu-debug@d710000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0x0 0xd710000 0x0 0x1000>; + clocks = <&infracfg CLK_INFRA_DEBUGSYS>; + clock-names = "apb_pclk"; + cpu = <&cpu3>; + }; + + cpu_debug4: cpu-debug@d810000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0x0 0xd810000 0x0 0x1000>; + clocks = <&infracfg CLK_INFRA_DEBUGSYS>; + clock-names = "apb_pclk"; + cpu = <&cpu4>; + }; + + cpu_debug5: cpu-debug@d910000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0x0 0xd910000 0x0 0x1000>; + clocks = <&infracfg CLK_INFRA_DEBUGSYS>; + clock-names = "apb_pclk"; + cpu = <&cpu5>; + }; + + cpu_debug6: cpu-debug@da10000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0x0 0xda10000 0x0 0x1000>; + clocks = <&infracfg CLK_INFRA_DEBUGSYS>; + clock-names = "apb_pclk"; + cpu = <&cpu6>; + }; + + cpu_debug7: cpu-debug@db10000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0x0 0xdb10000 0x0 0x1000>; + clocks = <&infracfg CLK_INFRA_DEBUGSYS>; + clock-names = "apb_pclk"; + cpu = <&cpu7>; + }; + topckgen: syscon@10000000 { compatible = "mediatek,mt8183-topckgen", "syscon"; reg = <0 0x10000000 0 0x1000>; @@ -1115,10 +1179,102 @@ usb_host: usb@11200000 { }; }; - audiosys: syscon@11220000 { + audiosys: audio-controller@11220000 { compatible = "mediatek,mt8183-audiosys", "syscon"; reg = <0 0x11220000 0 0x1000>; #clock-cells = <1>; + afe: mt8183-afe-pcm { + compatible = "mediatek,mt8183-audio"; + interrupts = ; + resets = <&watchdog MT8183_TOPRGU_AUDIO_SW_RST>; + reset-names = "audiosys"; + power-domains = + <&spm MT8183_POWER_DOMAIN_AUDIO>; + clocks = <&audiosys CLK_AUDIO_AFE>, + <&audiosys CLK_AUDIO_DAC>, + <&audiosys CLK_AUDIO_DAC_PREDIS>, + <&audiosys CLK_AUDIO_ADC>, + <&audiosys CLK_AUDIO_PDN_ADDA6_ADC>, + <&audiosys CLK_AUDIO_22M>, + <&audiosys CLK_AUDIO_24M>, + <&audiosys CLK_AUDIO_APLL_TUNER>, + <&audiosys CLK_AUDIO_APLL2_TUNER>, + <&audiosys CLK_AUDIO_I2S1>, + <&audiosys CLK_AUDIO_I2S2>, + <&audiosys CLK_AUDIO_I2S3>, + <&audiosys CLK_AUDIO_I2S4>, + <&audiosys CLK_AUDIO_TDM>, + <&audiosys CLK_AUDIO_TML>, + <&infracfg CLK_INFRA_AUDIO>, + <&infracfg CLK_INFRA_AUDIO_26M_BCLK>, + <&topckgen CLK_TOP_MUX_AUDIO>, + <&topckgen CLK_TOP_MUX_AUD_INTBUS>, + <&topckgen CLK_TOP_SYSPLL_D2_D4>, + <&topckgen CLK_TOP_MUX_AUD_1>, + <&topckgen CLK_TOP_APLL1_CK>, + <&topckgen CLK_TOP_MUX_AUD_2>, + <&topckgen CLK_TOP_APLL2_CK>, + <&topckgen CLK_TOP_MUX_AUD_ENG1>, + <&topckgen CLK_TOP_APLL1_D8>, + <&topckgen CLK_TOP_MUX_AUD_ENG2>, + <&topckgen CLK_TOP_APLL2_D8>, + <&topckgen CLK_TOP_MUX_APLL_I2S0>, + <&topckgen CLK_TOP_MUX_APLL_I2S1>, + <&topckgen CLK_TOP_MUX_APLL_I2S2>, + <&topckgen CLK_TOP_MUX_APLL_I2S3>, + <&topckgen CLK_TOP_MUX_APLL_I2S4>, + <&topckgen CLK_TOP_MUX_APLL_I2S5>, + <&topckgen CLK_TOP_APLL12_DIV0>, + <&topckgen CLK_TOP_APLL12_DIV1>, + <&topckgen CLK_TOP_APLL12_DIV2>, + <&topckgen CLK_TOP_APLL12_DIV3>, + <&topckgen CLK_TOP_APLL12_DIV4>, + <&topckgen CLK_TOP_APLL12_DIVB>, + /*<&topckgen CLK_TOP_APLL12_DIV5>,*/ + <&clk26m>; + clock-names = "aud_afe_clk", + "aud_dac_clk", + "aud_dac_predis_clk", + "aud_adc_clk", + "aud_adc_adda6_clk", + "aud_apll22m_clk", + "aud_apll24m_clk", + "aud_apll1_tuner_clk", + "aud_apll2_tuner_clk", + "aud_i2s1_bclk_sw", + "aud_i2s2_bclk_sw", + "aud_i2s3_bclk_sw", + "aud_i2s4_bclk_sw", + "aud_tdm_clk", + "aud_tml_clk", + "aud_infra_clk", + "mtkaif_26m_clk", + "top_mux_audio", + "top_mux_aud_intbus", + "top_syspll_d2_d4", + "top_mux_aud_1", + "top_apll1_ck", + "top_mux_aud_2", + "top_apll2_ck", + "top_mux_aud_eng1", + "top_apll1_d8", + "top_mux_aud_eng2", + "top_apll2_d8", + "top_i2s0_m_sel", + "top_i2s1_m_sel", + "top_i2s2_m_sel", + "top_i2s3_m_sel", + "top_i2s4_m_sel", + "top_i2s5_m_sel", + "top_apll12_div0", + "top_apll12_div1", + "top_apll12_div2", + "top_apll12_div3", + "top_apll12_div4", + "top_apll12_divb", + /*"top_apll12_div5",*/ + "top_clk26m_clk"; + }; }; mmc0: mmc@11230000 { @@ -1227,6 +1383,7 @@ mmsys: syscon@14000000 { compatible = "mediatek,mt8183-mmsys", "syscon"; reg = <0 0x14000000 0 0x1000>; #clock-cells = <1>; + #reset-cells = <1>; mboxes = <&gce 0 CMDQ_THR_PRIO_HIGHEST>, <&gce 1 CMDQ_THR_PRIO_HIGHEST>; mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0 0x1000>; @@ -1341,11 +1498,11 @@ dsi0: dsi@14014000 { reg = <0 0x14014000 0 0x1000>; interrupts = ; power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; - mediatek,syscon-dsi = <&mmsys 0x140>; clocks = <&mmsys CLK_MM_DSI0_MM>, <&mmsys CLK_MM_DSI0_IF>, <&mipi_tx0>; clock-names = "engine", "digital", "hs"; + resets = <&mmsys MT8183_MMSYS_SW0_RST_B_DISP_DSI0>; phys = <&mipi_tx0>; phy-names = "dphy"; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8192.dtsi b/arch/arm64/boot/dts/mediatek/mt8192.dtsi index 9757138a8b..53d790c335 100644 --- a/arch/arm64/boot/dts/mediatek/mt8192.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8192.dtsi @@ -5,6 +5,7 @@ */ /dts-v1/; +#include #include #include #include @@ -257,6 +258,24 @@ ppi_cluster1: interrupt-partition-1 { }; }; + topckgen: syscon@10000000 { + compatible = "mediatek,mt8192-topckgen", "syscon"; + reg = <0 0x10000000 0 0x1000>; + #clock-cells = <1>; + }; + + infracfg: syscon@10001000 { + compatible = "mediatek,mt8192-infracfg", "syscon"; + reg = <0 0x10001000 0 0x1000>; + #clock-cells = <1>; + }; + + pericfg: syscon@10003000 { + compatible = "mediatek,mt8192-pericfg", "syscon"; + reg = <0 0x10003000 0 0x1000>; + #clock-cells = <1>; + }; + pio: pinctrl@10005000 { compatible = "mediatek,mt8192-pinctrl"; reg = <0 0x10005000 0 0x1000>, @@ -282,6 +301,12 @@ pio: pinctrl@10005000 { #interrupt-cells = <2>; }; + apmixedsys: syscon@1000c000 { + compatible = "mediatek,mt8192-apmixedsys", "syscon"; + reg = <0 0x1000c000 0 0x1000>; + #clock-cells = <1>; + }; + systimer: timer@10017000 { compatible = "mediatek,mt8192-timer", "mediatek,mt6765-timer"; @@ -291,6 +316,12 @@ systimer: timer@10017000 { clock-names = "clk13m"; }; + scp_adsp: clock-controller@10720000 { + compatible = "mediatek,mt8192-scp_adsp"; + reg = <0 0x10720000 0 0x1000>; + #clock-cells = <1>; + }; + uart0: serial@11002000 { compatible = "mediatek,mt8192-uart", "mediatek,mt6577-uart"; @@ -311,6 +342,12 @@ uart1: serial@11003000 { status = "disabled"; }; + imp_iic_wrap_c: clock-controller@11007000 { + compatible = "mediatek,mt8192-imp_iic_wrap_c"; + reg = <0 0x11007000 0 0x1000>; + #clock-cells = <1>; + }; + spi0: spi@1100a000 { compatible = "mediatek,mt8192-spi", "mediatek,mt6765-spi"; @@ -436,7 +473,13 @@ nor_flash: spi@11234000 { status = "disable"; }; - i2c3: i2c3@11cb0000 { + audsys: clock-controller@11210000 { + compatible = "mediatek,mt8192-audsys", "syscon"; + reg = <0 0x11210000 0 0x1000>; + #clock-cells = <1>; + }; + + i2c3: i2c@11cb0000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11cb0000 0 0x1000>, <0 0x10217300 0 0x80>; @@ -449,7 +492,13 @@ i2c3: i2c3@11cb0000 { status = "disabled"; }; - i2c7: i2c7@11d00000 { + imp_iic_wrap_e: clock-controller@11cb1000 { + compatible = "mediatek,mt8192-imp_iic_wrap_e"; + reg = <0 0x11cb1000 0 0x1000>; + #clock-cells = <1>; + }; + + i2c7: i2c@11d00000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11d00000 0 0x1000>, <0 0x10217600 0 0x180>; @@ -462,7 +511,7 @@ i2c7: i2c7@11d00000 { status = "disabled"; }; - i2c8: i2c8@11d01000 { + i2c8: i2c@11d01000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11d01000 0 0x1000>, <0 0x10217780 0 0x180>; @@ -475,7 +524,7 @@ i2c8: i2c8@11d01000 { status = "disabled"; }; - i2c9: i2c9@11d02000 { + i2c9: i2c@11d02000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11d02000 0 0x1000>, <0 0x10217900 0 0x180>; @@ -488,7 +537,13 @@ i2c9: i2c9@11d02000 { status = "disabled"; }; - i2c1: i2c1@11d20000 { + imp_iic_wrap_s: clock-controller@11d03000 { + compatible = "mediatek,mt8192-imp_iic_wrap_s"; + reg = <0 0x11d03000 0 0x1000>; + #clock-cells = <1>; + }; + + i2c1: i2c@11d20000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11d20000 0 0x1000>, <0 0x10217100 0 0x80>; @@ -501,7 +556,7 @@ i2c1: i2c1@11d20000 { status = "disabled"; }; - i2c2: i2c2@11d21000 { + i2c2: i2c@11d21000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11d21000 0 0x1000>, <0 0x10217180 0 0x180>; @@ -514,7 +569,7 @@ i2c2: i2c2@11d21000 { status = "disabled"; }; - i2c4: i2c4@11d22000 { + i2c4: i2c@11d22000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11d22000 0 0x1000>, <0 0x10217380 0 0x180>; @@ -527,7 +582,13 @@ i2c4: i2c4@11d22000 { status = "disabled"; }; - i2c5: i2c5@11e00000 { + imp_iic_wrap_ws: clock-controller@11d23000 { + compatible = "mediatek,mt8192-imp_iic_wrap_ws"; + reg = <0 0x11d23000 0 0x1000>; + #clock-cells = <1>; + }; + + i2c5: i2c@11e00000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11e00000 0 0x1000>, <0 0x10217500 0 0x80>; @@ -540,7 +601,13 @@ i2c5: i2c5@11e00000 { status = "disabled"; }; - i2c0: i2c0@11f00000 { + imp_iic_wrap_w: clock-controller@11e01000 { + compatible = "mediatek,mt8192-imp_iic_wrap_w"; + reg = <0 0x11e01000 0 0x1000>; + #clock-cells = <1>; + }; + + i2c0: i2c@11f00000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11f00000 0 0x1000>, <0 0x10217080 0 0x80>; @@ -553,7 +620,7 @@ i2c0: i2c0@11f00000 { status = "disabled"; }; - i2c6: i2c6@11f01000 { + i2c6: i2c@11f01000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11f01000 0 0x1000>, <0 0x10217580 0 0x80>; @@ -565,5 +632,101 @@ i2c6: i2c6@11f01000 { #size-cells = <0>; status = "disabled"; }; + + imp_iic_wrap_n: clock-controller@11f02000 { + compatible = "mediatek,mt8192-imp_iic_wrap_n"; + reg = <0 0x11f02000 0 0x1000>; + #clock-cells = <1>; + }; + + msdc_top: clock-controller@11f10000 { + compatible = "mediatek,mt8192-msdc_top"; + reg = <0 0x11f10000 0 0x1000>; + #clock-cells = <1>; + }; + + msdc: clock-controller@11f60000 { + compatible = "mediatek,mt8192-msdc"; + reg = <0 0x11f60000 0 0x1000>; + #clock-cells = <1>; + }; + + mfgcfg: clock-controller@13fbf000 { + compatible = "mediatek,mt8192-mfgcfg"; + reg = <0 0x13fbf000 0 0x1000>; + #clock-cells = <1>; + }; + + mmsys: syscon@14000000 { + compatible = "mediatek,mt8192-mmsys", "syscon"; + reg = <0 0x14000000 0 0x1000>; + #clock-cells = <1>; + }; + + imgsys: clock-controller@15020000 { + compatible = "mediatek,mt8192-imgsys"; + reg = <0 0x15020000 0 0x1000>; + #clock-cells = <1>; + }; + + imgsys2: clock-controller@15820000 { + compatible = "mediatek,mt8192-imgsys2"; + reg = <0 0x15820000 0 0x1000>; + #clock-cells = <1>; + }; + + vdecsys_soc: clock-controller@1600f000 { + compatible = "mediatek,mt8192-vdecsys_soc"; + reg = <0 0x1600f000 0 0x1000>; + #clock-cells = <1>; + }; + + vdecsys: clock-controller@1602f000 { + compatible = "mediatek,mt8192-vdecsys"; + reg = <0 0x1602f000 0 0x1000>; + #clock-cells = <1>; + }; + + vencsys: clock-controller@17000000 { + compatible = "mediatek,mt8192-vencsys"; + reg = <0 0x17000000 0 0x1000>; + #clock-cells = <1>; + }; + + camsys: clock-controller@1a000000 { + compatible = "mediatek,mt8192-camsys"; + reg = <0 0x1a000000 0 0x1000>; + #clock-cells = <1>; + }; + + camsys_rawa: clock-controller@1a04f000 { + compatible = "mediatek,mt8192-camsys_rawa"; + reg = <0 0x1a04f000 0 0x1000>; + #clock-cells = <1>; + }; + + camsys_rawb: clock-controller@1a06f000 { + compatible = "mediatek,mt8192-camsys_rawb"; + reg = <0 0x1a06f000 0 0x1000>; + #clock-cells = <1>; + }; + + camsys_rawc: clock-controller@1a08f000 { + compatible = "mediatek,mt8192-camsys_rawc"; + reg = <0 0x1a08f000 0 0x1000>; + #clock-cells = <1>; + }; + + ipesys: clock-controller@1b000000 { + compatible = "mediatek,mt8192-ipesys"; + reg = <0 0x1b000000 0 0x1000>; + #clock-cells = <1>; + }; + + mdpsys: clock-controller@1f000000 { + compatible = "mediatek,mt8192-mdpsys"; + reg = <0 0x1f000000 0 0x1000>; + #clock-cells = <1>; + }; }; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8516.dtsi b/arch/arm64/boot/dts/mediatek/mt8516.dtsi index bbe5a1419e..d1b67c82d7 100644 --- a/arch/arm64/boot/dts/mediatek/mt8516.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8516.dtsi @@ -345,14 +345,9 @@ i2c0: i2c@11009000 { reg = <0 0x11009000 0 0x90>, <0 0x11000180 0 0x80>; interrupts = ; - clocks = <&topckgen CLK_TOP_AHB_INFRA_D2>, - <&infracfg CLK_IFR_I2C0_SEL>, - <&topckgen CLK_TOP_I2C0>, + clocks = <&topckgen CLK_TOP_I2C0>, <&topckgen CLK_TOP_APDMA>; - clock-names = "main-source", - "main-sel", - "main", - "dma"; + clock-names = "main", "dma"; #address-cells = <1>; #size-cells = <0>; status = "disabled"; @@ -364,14 +359,9 @@ i2c1: i2c@1100a000 { reg = <0 0x1100a000 0 0x90>, <0 0x11000200 0 0x80>; interrupts = ; - clocks = <&topckgen CLK_TOP_AHB_INFRA_D2>, - <&infracfg CLK_IFR_I2C1_SEL>, - <&topckgen CLK_TOP_I2C1>, + clocks = <&topckgen CLK_TOP_I2C1>, <&topckgen CLK_TOP_APDMA>; - clock-names = "main-source", - "main-sel", - "main", - "dma"; + clock-names = "main", "dma"; #address-cells = <1>; #size-cells = <0>; status = "disabled"; @@ -383,14 +373,9 @@ i2c2: i2c@1100b000 { reg = <0 0x1100b000 0 0x90>, <0 0x11000280 0 0x80>; interrupts = ; - clocks = <&topckgen CLK_TOP_AHB_INFRA_D2>, - <&infracfg CLK_IFR_I2C2_SEL>, - <&topckgen CLK_TOP_I2C2>, + clocks = <&topckgen CLK_TOP_I2C2>, <&topckgen CLK_TOP_APDMA>; - clock-names = "main-source", - "main-sel", - "main", - "dma"; + clock-names = "main", "dma"; #address-cells = <1>; #size-cells = <0>; status = "disabled"; diff --git a/arch/arm64/boot/dts/nvidia/Makefile b/arch/arm64/boot/dts/nvidia/Makefile index c80f7dc293..ea3f338fd0 100644 --- a/arch/arm64/boot/dts/nvidia/Makefile +++ b/arch/arm64/boot/dts/nvidia/Makefile @@ -12,3 +12,4 @@ dtb-$(CONFIG_ARCH_TEGRA_194_SOC) += tegra194-p2972-0000.dtb dtb-$(CONFIG_ARCH_TEGRA_194_SOC) += tegra194-p3509-0000+p3668-0000.dtb dtb-$(CONFIG_ARCH_TEGRA_194_SOC) += tegra194-p3509-0000+p3668-0001.dtb dtb-$(CONFIG_ARCH_TEGRA_234_SOC) += tegra234-sim-vdk.dtb +dtb-$(CONFIG_ARCH_TEGRA_234_SOC) += tegra234-p3737-0000+p3701-0000.dtb diff --git a/arch/arm64/boot/dts/nvidia/tegra132-norrin.dts b/arch/arm64/boot/dts/nvidia/tegra132-norrin.dts index 6e5f846566..f16acb4cab 100644 --- a/arch/arm64/boot/dts/nvidia/tegra132-norrin.dts +++ b/arch/arm64/boot/dts/nvidia/tegra132-norrin.dts @@ -564,7 +564,7 @@ as3722: pmic@40 { pinctrl-names = "default"; pinctrl-0 = <&as3722_default>; - as3722_default: pinmux@0 { + as3722_default: pinmux { gpio0 { pins = "gpio0"; function = "gpio"; @@ -770,7 +770,7 @@ i2c_20: i2c-tunnel { google,remote-bus = <0>; - charger: bq24735 { + charger: bq24735@9 { compatible = "ti,bq24735"; reg = <0x9>; interrupt-parent = <&gpio>; @@ -781,7 +781,7 @@ TEGRA_GPIO(J, 0) GPIO_ACTIVE_HIGH>; }; - battery: smart-battery { + battery: smart-battery@b { compatible = "sbs,sbs-battery"; reg = <0xb>; sbs,i2c-retry-count = <2>; @@ -885,14 +885,12 @@ MATRIX_KEY(0x07, 0x0b, KEY_UP) pmc@7000e400 { nvidia,invert-interrupt; nvidia,suspend-mode = <0>; - #wake-cells = <3>; nvidia,cpu-pwr-good-time = <500>; nvidia,cpu-pwr-off-time = <300>; nvidia,core-pwr-good-time = <641 3845>; nvidia,core-pwr-off-time = <61036>; nvidia,core-power-req-active-high; nvidia,sys-clock-req-active-high; - nvidia,reset-gpio = <&gpio TEGRA_GPIO(I, 5) GPIO_ACTIVE_LOW>; }; usb@70090000 { @@ -1021,11 +1019,9 @@ backlight: backlight { brightness-levels = <0 4 8 16 32 64 128 255>; default-brightness-level = <6>; - - backlight-boot-off; }; - clk32k_in: clock@0 { + clk32k_in: clock-32k { compatible = "fixed-clock"; clock-frequency = <32768>; #clock-cells = <0>; @@ -1059,7 +1055,7 @@ panel: panel { ddc-i2c-bus = <&dpaux>; }; - vdd_mux: regulator@0 { + vdd_mux: regulator-vdd-mux { compatible = "regulator-fixed"; regulator-name = "+VDD_MUX"; regulator-min-microvolt = <19000000>; @@ -1068,7 +1064,7 @@ vdd_mux: regulator@0 { regulator-boot-on; }; - vdd_5v0_sys: regulator@1 { + vdd_5v0_sys: regulator-vdd-5v0-sys { compatible = "regulator-fixed"; regulator-name = "+5V_SYS"; regulator-min-microvolt = <5000000>; @@ -1078,7 +1074,7 @@ vdd_5v0_sys: regulator@1 { vin-supply = <&vdd_mux>; }; - vdd_3v3_sys: regulator@2 { + vdd_3v3_sys: regulator-vdd-3v3-sys { compatible = "regulator-fixed"; regulator-name = "+3.3V_SYS"; regulator-min-microvolt = <3300000>; @@ -1088,7 +1084,7 @@ vdd_3v3_sys: regulator@2 { vin-supply = <&vdd_mux>; }; - vdd_3v3_run: regulator@3 { + vdd_3v3_run: regulator-vdd-3v3-run { compatible = "regulator-fixed"; regulator-name = "+3.3V_RUN"; regulator-min-microvolt = <3300000>; @@ -1100,7 +1096,7 @@ vdd_3v3_run: regulator@3 { vin-supply = <&vdd_3v3_sys>; }; - vdd_3v3_hdmi: regulator@4 { + vdd_3v3_hdmi: regulator-vdd-3v3-hdmi { compatible = "regulator-fixed"; regulator-name = "+3.3V_AVDD_HDMI_AP_GATED"; regulator-min-microvolt = <3300000>; @@ -1108,7 +1104,7 @@ vdd_3v3_hdmi: regulator@4 { vin-supply = <&vdd_3v3_run>; }; - vdd_led: regulator@5 { + vdd_led: regulator-vdd-led { compatible = "regulator-fixed"; regulator-name = "+VDD_LED"; regulator-min-microvolt = <3300000>; @@ -1118,7 +1114,7 @@ vdd_led: regulator@5 { vin-supply = <&vdd_mux>; }; - vdd_usb1_vbus: regulator@6 { + vdd_usb1_vbus: regulator-vdd-usb1-vbus { compatible = "regulator-fixed"; regulator-name = "+5V_USB_HS"; regulator-min-microvolt = <5000000>; @@ -1129,7 +1125,7 @@ vdd_usb1_vbus: regulator@6 { vin-supply = <&vdd_5v0_sys>; }; - vdd_usb3_vbus: regulator@7 { + vdd_usb3_vbus: regulator-vdd-usb3-vbus { compatible = "regulator-fixed"; regulator-name = "+5V_USB_SS"; regulator-min-microvolt = <5000000>; @@ -1140,7 +1136,7 @@ vdd_usb3_vbus: regulator@7 { vin-supply = <&vdd_5v0_sys>; }; - vdd_3v3_panel: regulator@8 { + vdd_3v3_panel: regulator-vdd-3v3-panel { compatible = "regulator-fixed"; regulator-name = "+3.3V_PANEL"; regulator-min-microvolt = <3300000>; @@ -1150,7 +1146,7 @@ vdd_3v3_panel: regulator@8 { vin-supply = <&vdd_3v3_sys>; }; - vdd_hdmi_pll: regulator@9 { + vdd_hdmi_pll: regulator-vdd-hdmi-pll { compatible = "regulator-fixed"; regulator-name = "+1.05V_RUN_AVDD_HDMI_PLL_AP_GATE"; regulator-min-microvolt = <1050000>; @@ -1159,7 +1155,7 @@ vdd_hdmi_pll: regulator@9 { vin-supply = <&vdd_1v05_run>; }; - vdd_5v0_hdmi: regulator@10 { + vdd_5v0_hdmi: regulator-vdd-5v0-hdmi { compatible = "regulator-fixed"; regulator-name = "+5V_HDMI_CON"; regulator-min-microvolt = <5000000>; @@ -1169,7 +1165,7 @@ vdd_5v0_hdmi: regulator@10 { vin-supply = <&vdd_5v0_sys>; }; - vdd_5v0_ts: regulator@11 { + vdd_5v0_ts: regulator-vdd-5v0-ts { compatible = "regulator-fixed"; regulator-name = "+5V_VDD_TS"; regulator-min-microvolt = <5000000>; @@ -1180,7 +1176,7 @@ vdd_5v0_ts: regulator@11 { enable-active-high; }; - vdd_3v3_lp0: regulator@12 { + vdd_3v3_lp0: regulator-vdd-3v3-lp0 { compatible = "regulator-fixed"; regulator-name = "+3.3V_LP0"; regulator-min-microvolt = <3300000>; diff --git a/arch/arm64/boot/dts/nvidia/tegra132.dtsi b/arch/arm64/boot/dts/nvidia/tegra132.dtsi index b0bcda8cc5..3673f79adf 100644 --- a/arch/arm64/boot/dts/nvidia/tegra132.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra132.dtsi @@ -8,6 +8,8 @@ #include #include +#include "tegra132-peripherals-opp.dtsi" + / { compatible = "nvidia,tegra132", "nvidia,tegra124"; interrupt-parent = <&lic>; @@ -210,7 +212,7 @@ lic: interrupt-controller@60004000 { }; timer@60005000 { - compatible = "nvidia,tegra124-timer", "nvidia,tegra20-timer"; + compatible = "nvidia,tegra124-timer", "nvidia,tegra30-timer"; reg = <0x0 0x60005000 0x0 0x400>; interrupts = , , @@ -244,6 +246,10 @@ actmon@6000c800 { clock-names = "actmon", "emc"; resets = <&tegra_car 119>; reset-names = "actmon"; + operating-points-v2 = <&emc_bw_dfs_opp_table>; + interconnects = <&mc TEGRA124_MC_MPCORER &emc>; + interconnect-names = "cpu-read"; + #cooling-cells = <2>; }; gpio: gpio@6000d000 { @@ -394,7 +400,7 @@ pwm: pwm@7000a000 { }; i2c@7000c000 { - compatible = "nvidia,tegra124-i2c", "nvidia,tegra114-i2c"; + compatible = "nvidia,tegra124-i2c"; reg = <0x0 0x7000c000 0x0 0x100>; interrupts = ; #address-cells = <1>; @@ -409,7 +415,7 @@ i2c@7000c000 { }; i2c@7000c400 { - compatible = "nvidia,tegra124-i2c", "nvidia,tegra114-i2c"; + compatible = "nvidia,tegra124-i2c"; reg = <0x0 0x7000c400 0x0 0x100>; interrupts = ; #address-cells = <1>; @@ -424,7 +430,7 @@ i2c@7000c400 { }; i2c@7000c500 { - compatible = "nvidia,tegra124-i2c", "nvidia,tegra114-i2c"; + compatible = "nvidia,tegra124-i2c"; reg = <0x0 0x7000c500 0x0 0x100>; interrupts = ; #address-cells = <1>; @@ -439,7 +445,7 @@ i2c@7000c500 { }; i2c@7000c700 { - compatible = "nvidia,tegra124-i2c", "nvidia,tegra114-i2c"; + compatible = "nvidia,tegra124-i2c"; reg = <0x0 0x7000c700 0x0 0x100>; interrupts = ; #address-cells = <1>; @@ -454,7 +460,7 @@ i2c@7000c700 { }; i2c@7000d000 { - compatible = "nvidia,tegra124-i2c", "nvidia,tegra114-i2c"; + compatible = "nvidia,tegra124-i2c"; reg = <0x0 0x7000d000 0x0 0x100>; interrupts = ; #address-cells = <1>; @@ -469,7 +475,7 @@ i2c@7000d000 { }; i2c@7000d100 { - compatible = "nvidia,tegra124-i2c", "nvidia,tegra114-i2c"; + compatible = "nvidia,tegra124-i2c"; reg = <0x0 0x7000d100 0x0 0x100>; interrupts = ; #address-cells = <1>; @@ -607,15 +613,20 @@ mc: memory-controller@70019000 { interrupts = ; #iommu-cells = <1>; + #reset-cells = <1>; + #interconnect-cells = <1>; }; emc: external-memory-controller@7001b000 { - compatible = "nvidia,tegra132-emc"; + compatible = "nvidia,tegra132-emc", "nvidia,tegra124-emc"; reg = <0x0 0x7001b000 0x0 0x1000>; clocks = <&tegra_car TEGRA124_CLK_EMC>; clock-names = "emc"; nvidia,memory-controller = <&mc>; + operating-points-v2 = <&emc_icc_dvfs_opp_table>; + + #interconnect-cells = <0>; }; sata@70020000 { @@ -624,10 +635,8 @@ sata@70020000 { <0x0 0x70020000 0x0 0x7000>; /* SATA */ interrupts = ; clocks = <&tegra_car TEGRA124_CLK_SATA>, - <&tegra_car TEGRA124_CLK_SATA_OOB>, - <&tegra_car TEGRA124_CLK_CML1>, - <&tegra_car TEGRA124_CLK_PLL_E>; - clock-names = "sata", "sata-oob", "cml1", "pll_e"; + <&tegra_car TEGRA124_CLK_SATA_OOB>; + clock-names = "sata", "sata-oob"; resets = <&tegra_car 124>, <&tegra_car 129>, <&tegra_car 123>; @@ -665,8 +674,8 @@ usb@70090000 { <&tegra_car TEGRA124_CLK_XUSB_HOST_SRC>, <&tegra_car TEGRA124_CLK_XUSB_FALCON_SRC>, <&tegra_car TEGRA124_CLK_XUSB_SS>, - <&tegra_car TEGRA124_CLK_XUSB_SS_SRC>, <&tegra_car TEGRA124_CLK_XUSB_SS_DIV2>, + <&tegra_car TEGRA124_CLK_XUSB_SS_SRC>, <&tegra_car TEGRA124_CLK_XUSB_HS_SRC>, <&tegra_car TEGRA124_CLK_XUSB_FS_SRC>, <&tegra_car TEGRA124_CLK_PLL_U_480M>, @@ -674,7 +683,7 @@ usb@70090000 { <&tegra_car TEGRA124_CLK_PLL_E>; clock-names = "xusb_host", "xusb_host_src", "xusb_falcon_src", "xusb_ss", - "xusb_ss_src", "xusb_ss_div2", + "xusb_ss_div2", "xusb_ss_src", "xusb_hs_src", "xusb_fs_src", "pll_u_480m", "clk_m", "pll_e"; resets = <&tegra_car 89>, <&tegra_car 156>, @@ -886,7 +895,7 @@ throttle_heavy: heavy { }; thermal-zones { - cpu { + cpu-thermal { polling-delay-passive = <1000>; polling-delay = <0>; @@ -914,7 +923,8 @@ map0 { }; }; }; - mem { + + mem-thermal { polling-delay-passive = <0>; polling-delay = <0>; @@ -941,7 +951,8 @@ cooling-maps { */ }; }; - gpu { + + gpu-thermal { polling-delay-passive = <1000>; polling-delay = <0>; @@ -969,7 +980,8 @@ map0 { }; }; }; - pllx { + + pllx-thermal { polling-delay-passive = <0>; polling-delay = <0>; @@ -1107,7 +1119,7 @@ tegra_i2s4: i2s@70301400 { }; usb@7d000000 { - compatible = "nvidia,tegra124-ehci", "nvidia,tegra30-ehci", "usb-ehci"; + compatible = "nvidia,tegra124-ehci", "nvidia,tegra30-ehci"; reg = <0x0 0x7d000000 0x0 0x4000>; interrupts = ; phy_type = "utmi"; @@ -1123,6 +1135,7 @@ phy1: usb-phy@7d000000 { compatible = "nvidia,tegra124-usb-phy", "nvidia,tegra30-usb-phy"; reg = <0x0 0x7d000000 0x0 0x4000>, <0x0 0x7d000000 0x0 0x4000>; + interrupts = ; phy_type = "utmi"; clocks = <&tegra_car TEGRA124_CLK_USBD>, <&tegra_car TEGRA124_CLK_PLL_U>, @@ -1142,11 +1155,12 @@ phy1: usb-phy@7d000000 { nvidia,hsdiscon-level = <5>; nvidia,xcvr-hsslew = <12>; nvidia,has-utmi-pad-registers; + nvidia,pmc = <&tegra_pmc 0>; status = "disabled"; }; usb@7d004000 { - compatible = "nvidia,tegra124-ehci", "nvidia,tegra30-ehci", "usb-ehci"; + compatible = "nvidia,tegra124-ehci", "nvidia,tegra30-ehci"; reg = <0x0 0x7d004000 0x0 0x4000>; interrupts = ; phy_type = "utmi"; @@ -1162,6 +1176,7 @@ phy2: usb-phy@7d004000 { compatible = "nvidia,tegra124-usb-phy", "nvidia,tegra30-usb-phy"; reg = <0x0 0x7d004000 0x0 0x4000>, <0x0 0x7d000000 0x0 0x4000>; + interrupts = ; phy_type = "utmi"; clocks = <&tegra_car TEGRA124_CLK_USB2>, <&tegra_car TEGRA124_CLK_PLL_U>, @@ -1180,11 +1195,12 @@ phy2: usb-phy@7d004000 { nvidia,hssquelch-level = <2>; nvidia,hsdiscon-level = <5>; nvidia,xcvr-hsslew = <12>; + nvidia,pmc = <&tegra_pmc 1>; status = "disabled"; }; usb@7d008000 { - compatible = "nvidia,tegra124-ehci", "nvidia,tegra30-ehci", "usb-ehci"; + compatible = "nvidia,tegra124-ehci", "nvidia,tegra30-ehci"; reg = <0x0 0x7d008000 0x0 0x4000>; interrupts = ; phy_type = "utmi"; @@ -1200,6 +1216,7 @@ phy3: usb-phy@7d008000 { compatible = "nvidia,tegra124-usb-phy", "nvidia,tegra30-usb-phy"; reg = <0x0 0x7d008000 0x0 0x4000>, <0x0 0x7d000000 0x0 0x4000>; + interrupts = ; phy_type = "utmi"; clocks = <&tegra_car TEGRA124_CLK_USB3>, <&tegra_car TEGRA124_CLK_PLL_U>, @@ -1218,6 +1235,7 @@ phy3: usb-phy@7d008000 { nvidia,hssquelch-level = <2>; nvidia,hsdiscon-level = <5>; nvidia,xcvr-hsslew = <12>; + nvidia,pmc = <&tegra_pmc 2>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts b/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts index 74c1a5df3f..c4dee05f33 100644 --- a/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts +++ b/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts @@ -275,6 +275,542 @@ xbar_dspk2_ep: endpoint { remote-endpoint = <&dspk2_cif_ep>; }; }; + + xbar_sfc1_in_port: port@20 { + reg = <0x20>; + + xbar_sfc1_in_ep: endpoint { + remote-endpoint = <&sfc1_cif_in_ep>; + }; + }; + + port@21 { + reg = <0x21>; + + xbar_sfc1_out_ep: endpoint { + remote-endpoint = <&sfc1_cif_out_ep>; + }; + }; + + xbar_sfc2_in_port: port@22 { + reg = <0x22>; + + xbar_sfc2_in_ep: endpoint { + remote-endpoint = <&sfc2_cif_in_ep>; + }; + }; + + port@23 { + reg = <0x23>; + + xbar_sfc2_out_ep: endpoint { + remote-endpoint = <&sfc2_cif_out_ep>; + }; + }; + + xbar_sfc3_in_port: port@24 { + reg = <0x24>; + + xbar_sfc3_in_ep: endpoint { + remote-endpoint = <&sfc3_cif_in_ep>; + }; + }; + + port@25 { + reg = <0x25>; + + xbar_sfc3_out_ep: endpoint { + remote-endpoint = <&sfc3_cif_out_ep>; + }; + }; + + xbar_sfc4_in_port: port@26 { + reg = <0x26>; + + xbar_sfc4_in_ep: endpoint { + remote-endpoint = <&sfc4_cif_in_ep>; + }; + }; + + port@27 { + reg = <0x27>; + + xbar_sfc4_out_ep: endpoint { + remote-endpoint = <&sfc4_cif_out_ep>; + }; + }; + + xbar_mvc1_in_port: port@28 { + reg = <0x28>; + + xbar_mvc1_in_ep: endpoint { + remote-endpoint = <&mvc1_cif_in_ep>; + }; + }; + + port@29 { + reg = <0x29>; + + xbar_mvc1_out_ep: endpoint { + remote-endpoint = <&mvc1_cif_out_ep>; + }; + }; + + xbar_mvc2_in_port: port@2a { + reg = <0x2a>; + + xbar_mvc2_in_ep: endpoint { + remote-endpoint = <&mvc2_cif_in_ep>; + }; + }; + + port@2b { + reg = <0x2b>; + + xbar_mvc2_out_ep: endpoint { + remote-endpoint = <&mvc2_cif_out_ep>; + }; + }; + + xbar_amx1_in1_port: port@2c { + reg = <0x2c>; + + xbar_amx1_in1_ep: endpoint { + remote-endpoint = <&amx1_in1_ep>; + }; + }; + + xbar_amx1_in2_port: port@2d { + reg = <0x2d>; + + xbar_amx1_in2_ep: endpoint { + remote-endpoint = <&amx1_in2_ep>; + }; + }; + + xbar_amx1_in3_port: port@2e { + reg = <0x2e>; + + xbar_amx1_in3_ep: endpoint { + remote-endpoint = <&amx1_in3_ep>; + }; + }; + + xbar_amx1_in4_port: port@2f { + reg = <0x2f>; + + xbar_amx1_in4_ep: endpoint { + remote-endpoint = <&amx1_in4_ep>; + }; + }; + + port@30 { + reg = <0x30>; + + xbar_amx1_out_ep: endpoint { + remote-endpoint = <&amx1_out_ep>; + }; + }; + + xbar_amx2_in1_port: port@31 { + reg = <0x31>; + + xbar_amx2_in1_ep: endpoint { + remote-endpoint = <&amx2_in1_ep>; + }; + }; + + xbar_amx2_in2_port: port@32 { + reg = <0x32>; + + xbar_amx2_in2_ep: endpoint { + remote-endpoint = <&amx2_in2_ep>; + }; + }; + + xbar_amx2_in3_port: port@33 { + reg = <0x33>; + + xbar_amx2_in3_ep: endpoint { + remote-endpoint = <&amx2_in3_ep>; + }; + }; + + xbar_amx2_in4_port: port@34 { + reg = <0x34>; + + xbar_amx2_in4_ep: endpoint { + remote-endpoint = <&amx2_in4_ep>; + }; + }; + + port@35 { + reg = <0x35>; + + xbar_amx2_out_ep: endpoint { + remote-endpoint = <&amx2_out_ep>; + }; + }; + + xbar_amx3_in1_port: port@36 { + reg = <0x36>; + + xbar_amx3_in1_ep: endpoint { + remote-endpoint = <&amx3_in1_ep>; + }; + }; + + xbar_amx3_in2_port: port@37 { + reg = <0x37>; + + xbar_amx3_in2_ep: endpoint { + remote-endpoint = <&amx3_in2_ep>; + }; + }; + + xbar_amx3_in3_port: port@38 { + reg = <0x38>; + + xbar_amx3_in3_ep: endpoint { + remote-endpoint = <&amx3_in3_ep>; + }; + }; + + xbar_amx3_in4_port: port@39 { + reg = <0x39>; + + xbar_amx3_in4_ep: endpoint { + remote-endpoint = <&amx3_in4_ep>; + }; + }; + + port@3a { + reg = <0x3a>; + + xbar_amx3_out_ep: endpoint { + remote-endpoint = <&amx3_out_ep>; + }; + }; + + xbar_amx4_in1_port: port@3b { + reg = <0x3b>; + + xbar_amx4_in1_ep: endpoint { + remote-endpoint = <&amx4_in1_ep>; + }; + }; + + xbar_amx4_in2_port: port@3c { + reg = <0x3c>; + + xbar_amx4_in2_ep: endpoint { + remote-endpoint = <&amx4_in2_ep>; + }; + }; + + xbar_amx4_in3_port: port@3d { + reg = <0x3d>; + + xbar_amx4_in3_ep: endpoint { + remote-endpoint = <&amx4_in3_ep>; + }; + }; + + xbar_amx4_in4_port: port@3e { + reg = <0x3e>; + + xbar_amx4_in4_ep: endpoint { + remote-endpoint = <&amx4_in4_ep>; + }; + }; + + port@3f { + reg = <0x3f>; + + xbar_amx4_out_ep: endpoint { + remote-endpoint = <&amx4_out_ep>; + }; + }; + + xbar_adx1_in_port: port@40 { + reg = <0x40>; + + xbar_adx1_in_ep: endpoint { + remote-endpoint = <&adx1_in_ep>; + }; + }; + + port@41 { + reg = <0x41>; + + xbar_adx1_out1_ep: endpoint { + remote-endpoint = <&adx1_out1_ep>; + }; + }; + + port@42 { + reg = <0x42>; + + xbar_adx1_out2_ep: endpoint { + remote-endpoint = <&adx1_out2_ep>; + }; + }; + + port@43 { + reg = <0x43>; + + xbar_adx1_out3_ep: endpoint { + remote-endpoint = <&adx1_out3_ep>; + }; + }; + + port@44 { + reg = <0x44>; + + xbar_adx1_out4_ep: endpoint { + remote-endpoint = <&adx1_out4_ep>; + }; + }; + + xbar_adx2_in_port: port@45 { + reg = <0x45>; + + xbar_adx2_in_ep: endpoint { + remote-endpoint = <&adx2_in_ep>; + }; + }; + + port@46 { + reg = <0x46>; + + xbar_adx2_out1_ep: endpoint { + remote-endpoint = <&adx2_out1_ep>; + }; + }; + + port@47 { + reg = <0x47>; + + xbar_adx2_out2_ep: endpoint { + remote-endpoint = <&adx2_out2_ep>; + }; + }; + + port@48 { + reg = <0x48>; + + xbar_adx2_out3_ep: endpoint { + remote-endpoint = <&adx2_out3_ep>; + }; + }; + + port@49 { + reg = <0x49>; + + xbar_adx2_out4_ep: endpoint { + remote-endpoint = <&adx2_out4_ep>; + }; + }; + + xbar_adx3_in_port: port@4a { + reg = <0x4a>; + + xbar_adx3_in_ep: endpoint { + remote-endpoint = <&adx3_in_ep>; + }; + }; + + port@4b { + reg = <0x4b>; + + xbar_adx3_out1_ep: endpoint { + remote-endpoint = <&adx3_out1_ep>; + }; + }; + + port@4c { + reg = <0x4c>; + + xbar_adx3_out2_ep: endpoint { + remote-endpoint = <&adx3_out2_ep>; + }; + }; + + port@4d { + reg = <0x4d>; + + xbar_adx3_out3_ep: endpoint { + remote-endpoint = <&adx3_out3_ep>; + }; + }; + + port@4e { + reg = <0x4e>; + + xbar_adx3_out4_ep: endpoint { + remote-endpoint = <&adx3_out4_ep>; + }; + }; + + xbar_adx4_in_port: port@4f { + reg = <0x4f>; + + xbar_adx4_in_ep: endpoint { + remote-endpoint = <&adx4_in_ep>; + }; + }; + + port@50 { + reg = <0x50>; + + xbar_adx4_out1_ep: endpoint { + remote-endpoint = <&adx4_out1_ep>; + }; + }; + + port@51 { + reg = <0x51>; + + xbar_adx4_out2_ep: endpoint { + remote-endpoint = <&adx4_out2_ep>; + }; + }; + + port@52 { + reg = <0x52>; + + xbar_adx4_out3_ep: endpoint { + remote-endpoint = <&adx4_out3_ep>; + }; + }; + + port@53 { + reg = <0x53>; + + xbar_adx4_out4_ep: endpoint { + remote-endpoint = <&adx4_out4_ep>; + }; + }; + + xbar_mixer_in1_port: port@54 { + reg = <0x54>; + + xbar_mixer_in1_ep: endpoint { + remote-endpoint = <&mixer_in1_ep>; + }; + }; + + xbar_mixer_in2_port: port@55 { + reg = <0x55>; + + xbar_mixer_in2_ep: endpoint { + remote-endpoint = <&mixer_in2_ep>; + }; + }; + + xbar_mixer_in3_port: port@56 { + reg = <0x56>; + + xbar_mixer_in3_ep: endpoint { + remote-endpoint = <&mixer_in3_ep>; + }; + }; + + xbar_mixer_in4_port: port@57 { + reg = <0x57>; + + xbar_mixer_in4_ep: endpoint { + remote-endpoint = <&mixer_in4_ep>; + }; + }; + + xbar_mixer_in5_port: port@58 { + reg = <0x58>; + + xbar_mixer_in5_ep: endpoint { + remote-endpoint = <&mixer_in5_ep>; + }; + }; + + xbar_mixer_in6_port: port@59 { + reg = <0x59>; + + xbar_mixer_in6_ep: endpoint { + remote-endpoint = <&mixer_in6_ep>; + }; + }; + + xbar_mixer_in7_port: port@5a { + reg = <0x5a>; + + xbar_mixer_in7_ep: endpoint { + remote-endpoint = <&mixer_in7_ep>; + }; + }; + + xbar_mixer_in8_port: port@5b { + reg = <0x5b>; + + xbar_mixer_in8_ep: endpoint { + remote-endpoint = <&mixer_in8_ep>; + }; + }; + + xbar_mixer_in9_port: port@5c { + reg = <0x5c>; + + xbar_mixer_in9_ep: endpoint { + remote-endpoint = <&mixer_in9_ep>; + }; + }; + + xbar_mixer_in10_port: port@5d { + reg = <0x5d>; + + xbar_mixer_in10_ep: endpoint { + remote-endpoint = <&mixer_in10_ep>; + }; + }; + + port@5e { + reg = <0x5e>; + + xbar_mixer_out1_ep: endpoint { + remote-endpoint = <&mixer_out1_ep>; + }; + }; + + port@5f { + reg = <0x5f>; + + xbar_mixer_out2_ep: endpoint { + remote-endpoint = <&mixer_out2_ep>; + }; + }; + + port@60 { + reg = <0x60>; + + xbar_mixer_out3_ep: endpoint { + remote-endpoint = <&mixer_out3_ep>; + }; + }; + + port@61 { + reg = <0x61>; + + xbar_mixer_out4_ep: endpoint { + remote-endpoint = <&mixer_out4_ep>; + }; + }; + + port@62 { + reg = <0x62>; + + xbar_mixer_out5_ep: endpoint { + remote-endpoint = <&mixer_out5_ep>; + }; + }; }; admaif@290f000 { @@ -726,6 +1262,679 @@ dspk2_dap_ep: endpoint { }; }; }; + + sfc@2902000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc1_in_ep>; + convert-rate = <44100>; + }; + }; + + sfc1_out_port: port@1 { + reg = <1>; + + sfc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc1_out_ep>; + convert-rate = <48000>; + }; + }; + }; + }; + + sfc@2902200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc2_in_ep>; + }; + }; + + sfc2_out_port: port@1 { + reg = <1>; + + sfc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc2_out_ep>; + }; + }; + }; + }; + + sfc@2902400 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc3_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc3_in_ep>; + }; + }; + + sfc3_out_port: port@1 { + reg = <1>; + + sfc3_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc3_out_ep>; + }; + }; + }; + }; + + sfc@2902600 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc4_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc4_in_ep>; + }; + }; + + sfc4_out_port: port@1 { + reg = <1>; + + sfc4_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc4_out_ep>; + }; + }; + }; + }; + + mvc@290a000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc1_in_ep>; + }; + }; + + mvc1_out_port: port@1 { + reg = <1>; + + mvc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc1_out_ep>; + }; + }; + }; + }; + + mvc@290a200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc2_in_ep>; + }; + }; + + mvc2_out_port: port@1 { + reg = <1>; + + mvc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc2_out_ep>; + }; + }; + }; + }; + + amx@2903000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx1_in1_ep: endpoint { + remote-endpoint = <&xbar_amx1_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx1_in2_ep: endpoint { + remote-endpoint = <&xbar_amx1_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx1_in3_ep: endpoint { + remote-endpoint = <&xbar_amx1_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx1_in4_ep: endpoint { + remote-endpoint = <&xbar_amx1_in4_ep>; + }; + }; + + amx1_out_port: port@4 { + reg = <4>; + + amx1_out_ep: endpoint { + remote-endpoint = <&xbar_amx1_out_ep>; + }; + }; + }; + }; + + amx@2903100 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx2_in1_ep: endpoint { + remote-endpoint = <&xbar_amx2_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx2_in2_ep: endpoint { + remote-endpoint = <&xbar_amx2_in2_ep>; + }; + }; + + amx2_in3_port: port@2 { + reg = <2>; + + amx2_in3_ep: endpoint { + remote-endpoint = <&xbar_amx2_in3_ep>; + }; + }; + + amx2_in4_port: port@3 { + reg = <3>; + + amx2_in4_ep: endpoint { + remote-endpoint = <&xbar_amx2_in4_ep>; + }; + }; + + amx2_out_port: port@4 { + reg = <4>; + + amx2_out_ep: endpoint { + remote-endpoint = <&xbar_amx2_out_ep>; + }; + }; + }; + }; + + amx@2903200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx3_in1_ep: endpoint { + remote-endpoint = <&xbar_amx3_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx3_in2_ep: endpoint { + remote-endpoint = <&xbar_amx3_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx3_in3_ep: endpoint { + remote-endpoint = <&xbar_amx3_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx3_in4_ep: endpoint { + remote-endpoint = <&xbar_amx3_in4_ep>; + }; + }; + + amx3_out_port: port@4 { + reg = <4>; + + amx3_out_ep: endpoint { + remote-endpoint = <&xbar_amx3_out_ep>; + }; + }; + }; + }; + + amx@2903300 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx4_in1_ep: endpoint { + remote-endpoint = <&xbar_amx4_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx4_in2_ep: endpoint { + remote-endpoint = <&xbar_amx4_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx4_in3_ep: endpoint { + remote-endpoint = <&xbar_amx4_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx4_in4_ep: endpoint { + remote-endpoint = <&xbar_amx4_in4_ep>; + }; + }; + + amx4_out_port: port@4 { + reg = <4>; + + amx4_out_ep: endpoint { + remote-endpoint = <&xbar_amx4_out_ep>; + }; + }; + }; + }; + + adx@2903800 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx1_in_ep: endpoint { + remote-endpoint = <&xbar_adx1_in_ep>; + }; + }; + + adx1_out1_port: port@1 { + reg = <1>; + + adx1_out1_ep: endpoint { + remote-endpoint = <&xbar_adx1_out1_ep>; + }; + }; + + adx1_out2_port: port@2 { + reg = <2>; + + adx1_out2_ep: endpoint { + remote-endpoint = <&xbar_adx1_out2_ep>; + }; + }; + + adx1_out3_port: port@3 { + reg = <3>; + + adx1_out3_ep: endpoint { + remote-endpoint = <&xbar_adx1_out3_ep>; + }; + }; + + adx1_out4_port: port@4 { + reg = <4>; + + adx1_out4_ep: endpoint { + remote-endpoint = <&xbar_adx1_out4_ep>; + }; + }; + }; + }; + + adx@2903900 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx2_in_ep: endpoint { + remote-endpoint = <&xbar_adx2_in_ep>; + }; + }; + + adx2_out1_port: port@1 { + reg = <1>; + + adx2_out1_ep: endpoint { + remote-endpoint = <&xbar_adx2_out1_ep>; + }; + }; + + adx2_out2_port: port@2 { + reg = <2>; + + adx2_out2_ep: endpoint { + remote-endpoint = <&xbar_adx2_out2_ep>; + }; + }; + + adx2_out3_port: port@3 { + reg = <3>; + + adx2_out3_ep: endpoint { + remote-endpoint = <&xbar_adx2_out3_ep>; + }; + }; + + adx2_out4_port: port@4 { + reg = <4>; + + adx2_out4_ep: endpoint { + remote-endpoint = <&xbar_adx2_out4_ep>; + }; + }; + }; + }; + + adx@2903a00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx3_in_ep: endpoint { + remote-endpoint = <&xbar_adx3_in_ep>; + }; + }; + + adx3_out1_port: port@1 { + reg = <1>; + + adx3_out1_ep: endpoint { + remote-endpoint = <&xbar_adx3_out1_ep>; + }; + }; + + adx3_out2_port: port@2 { + reg = <2>; + + adx3_out2_ep: endpoint { + remote-endpoint = <&xbar_adx3_out2_ep>; + }; + }; + + adx3_out3_port: port@3 { + reg = <3>; + + adx3_out3_ep: endpoint { + remote-endpoint = <&xbar_adx3_out3_ep>; + }; + }; + + adx3_out4_port: port@4 { + reg = <4>; + + adx3_out4_ep: endpoint { + remote-endpoint = <&xbar_adx3_out4_ep>; + }; + }; + }; + }; + + adx@2903b00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx4_in_ep: endpoint { + remote-endpoint = <&xbar_adx4_in_ep>; + }; + }; + + adx4_out1_port: port@1 { + reg = <1>; + + adx4_out1_ep: endpoint { + remote-endpoint = <&xbar_adx4_out1_ep>; + }; + }; + + adx4_out2_port: port@2 { + reg = <2>; + + adx4_out2_ep: endpoint { + remote-endpoint = <&xbar_adx4_out2_ep>; + }; + }; + + adx4_out3_port: port@3 { + reg = <3>; + + adx4_out3_ep: endpoint { + remote-endpoint = <&xbar_adx4_out3_ep>; + }; + }; + + adx4_out4_port: port@4 { + reg = <4>; + + adx4_out4_ep: endpoint { + remote-endpoint = <&xbar_adx4_out4_ep>; + }; + }; + }; + }; + + amixer@290bb00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0x0>; + + mixer_in1_ep: endpoint { + remote-endpoint = <&xbar_mixer_in1_ep>; + }; + }; + + port@1 { + reg = <0x1>; + + mixer_in2_ep: endpoint { + remote-endpoint = <&xbar_mixer_in2_ep>; + }; + }; + + port@2 { + reg = <0x2>; + + mixer_in3_ep: endpoint { + remote-endpoint = <&xbar_mixer_in3_ep>; + }; + }; + + port@3 { + reg = <0x3>; + + mixer_in4_ep: endpoint { + remote-endpoint = <&xbar_mixer_in4_ep>; + }; + }; + + port@4 { + reg = <0x4>; + + mixer_in5_ep: endpoint { + remote-endpoint = <&xbar_mixer_in5_ep>; + }; + }; + + port@5 { + reg = <0x5>; + + mixer_in6_ep: endpoint { + remote-endpoint = <&xbar_mixer_in6_ep>; + }; + }; + + port@6 { + reg = <0x6>; + + mixer_in7_ep: endpoint { + remote-endpoint = <&xbar_mixer_in7_ep>; + }; + }; + + port@7 { + reg = <0x7>; + + mixer_in8_ep: endpoint { + remote-endpoint = <&xbar_mixer_in8_ep>; + }; + }; + + port@8 { + reg = <0x8>; + + mixer_in9_ep: endpoint { + remote-endpoint = <&xbar_mixer_in9_ep>; + }; + }; + + port@9 { + reg = <0x9>; + + mixer_in10_ep: endpoint { + remote-endpoint = <&xbar_mixer_in10_ep>; + }; + }; + + mixer_out1_port: port@a { + reg = <0xa>; + + mixer_out1_ep: endpoint { + remote-endpoint = <&xbar_mixer_out1_ep>; + }; + }; + + mixer_out2_port: port@b { + reg = <0xb>; + + mixer_out2_ep: endpoint { + remote-endpoint = <&xbar_mixer_out2_ep>; + }; + }; + + mixer_out3_port: port@c { + reg = <0xc>; + + mixer_out3_ep: endpoint { + remote-endpoint = <&xbar_mixer_out3_ep>; + }; + }; + + mixer_out4_port: port@d { + reg = <0xd>; + + mixer_out4_ep: endpoint { + remote-endpoint = <&xbar_mixer_out4_ep>; + }; + }; + + mixer_out5_port: port@e { + reg = <0xe>; + + mixer_out5_ep: endpoint { + remote-endpoint = <&xbar_mixer_out5_ep>; + }; + }; + }; + }; }; }; @@ -736,19 +1945,19 @@ power-monitor@42 { #address-cells = <1>; #size-cells = <0>; - channel@0 { + input@0 { reg = <0x0>; label = "VDD_MUX"; shunt-resistor-micro-ohms = <20000>; }; - channel@1 { + input@1 { reg = <0x1>; label = "VDD_5V0_IO_SYS"; shunt-resistor-micro-ohms = <5000>; }; - channel@2 { + input@2 { reg = <0x2>; label = "VDD_3V3_SYS"; shunt-resistor-micro-ohms = <10000>; @@ -761,19 +1970,19 @@ power-monitor@43 { #address-cells = <1>; #size-cells = <0>; - channel@0 { + input@0 { reg = <0x0>; label = "VDD_3V3_IO_SLP"; shunt-resistor-micro-ohms = <10000>; }; - channel@1 { + input@1 { reg = <0x1>; label = "VDD_1V8_IO"; shunt-resistor-micro-ohms = <10000>; }; - channel@2 { + input@2 { reg = <0x2>; label = "VDD_M2_IN"; shunt-resistor-micro-ohms = <10000>; @@ -1041,7 +2250,7 @@ volume-down { }; }; - vdd_sd: regulator@100 { + vdd_sd: regulator-vdd-sd { compatible = "regulator-fixed"; regulator-name = "SD_CARD_SW_PWR"; regulator-min-microvolt = <3300000>; @@ -1053,7 +2262,7 @@ vdd_sd: regulator@100 { vin-supply = <&vdd_3v3_sys>; }; - vdd_hdmi: regulator@101 { + vdd_hdmi: regulator-vdd-hdmi { compatible = "regulator-fixed"; regulator-name = "VDD_HDMI_5V0"; regulator-min-microvolt = <5000000>; @@ -1065,7 +2274,7 @@ vdd_hdmi: regulator@101 { vin-supply = <&vdd_5v0_sys>; }; - vdd_usb0: regulator@102 { + vdd_usb0: regulator-vdd-usb0 { compatible = "regulator-fixed"; regulator-name = "VDD_USB0"; regulator-min-microvolt = <5000000>; @@ -1077,7 +2286,7 @@ vdd_usb0: regulator@102 { vin-supply = <&vdd_5v0_sys>; }; - vdd_usb1: regulator@103 { + vdd_usb1: regulator-vdd-usb1 { compatible = "regulator-fixed"; regulator-name = "VDD_USB1"; regulator-min-microvolt = <5000000>; @@ -1104,6 +2313,41 @@ sound { <&xbar_i2s4_port>, <&xbar_i2s5_port>, <&xbar_i2s6_port>, <&xbar_dmic1_port>, <&xbar_dmic2_port>, <&xbar_dmic3_port>, <&xbar_dspk1_port>, <&xbar_dspk2_port>, + <&xbar_sfc1_in_port>, <&xbar_sfc2_in_port>, + <&xbar_sfc3_in_port>, <&xbar_sfc4_in_port>, + <&xbar_mvc1_in_port>, <&xbar_mvc2_in_port>, + <&xbar_amx1_in1_port>, <&xbar_amx1_in2_port>, + <&xbar_amx1_in3_port>, <&xbar_amx1_in4_port>, + <&xbar_amx2_in1_port>, <&xbar_amx2_in2_port>, + <&xbar_amx2_in3_port>, <&xbar_amx2_in4_port>, + <&xbar_amx3_in1_port>, <&xbar_amx3_in2_port>, + <&xbar_amx3_in3_port>, <&xbar_amx3_in4_port>, + <&xbar_amx4_in1_port>, <&xbar_amx4_in2_port>, + <&xbar_amx4_in3_port>, <&xbar_amx4_in4_port>, + <&xbar_adx1_in_port>, <&xbar_adx2_in_port>, + <&xbar_adx3_in_port>, <&xbar_adx4_in_port>, + <&xbar_mixer_in1_port>, <&xbar_mixer_in2_port>, + <&xbar_mixer_in3_port>, <&xbar_mixer_in4_port>, + <&xbar_mixer_in5_port>, <&xbar_mixer_in6_port>, + <&xbar_mixer_in7_port>, <&xbar_mixer_in8_port>, + <&xbar_mixer_in9_port>, <&xbar_mixer_in10_port>, + /* HW accelerators */ + <&sfc1_out_port>, <&sfc2_out_port>, + <&sfc3_out_port>, <&sfc4_out_port>, + <&mvc1_out_port>, <&mvc2_out_port>, + <&amx1_out_port>, <&amx2_out_port>, + <&amx3_out_port>, <&amx4_out_port>, + <&adx1_out1_port>, <&adx1_out2_port>, + <&adx1_out3_port>, <&adx1_out4_port>, + <&adx2_out1_port>, <&adx2_out2_port>, + <&adx2_out3_port>, <&adx2_out4_port>, + <&adx3_out1_port>, <&adx3_out2_port>, + <&adx3_out3_port>, <&adx3_out4_port>, + <&adx4_out1_port>, <&adx4_out2_port>, + <&adx4_out3_port>, <&adx4_out4_port>, + <&mixer_out1_port>, <&mixer_out2_port>, + <&mixer_out3_port>, <&mixer_out4_port>, + <&mixer_out5_port>, /* I/O */ <&i2s1_port>, <&i2s2_port>, <&i2s3_port>, <&i2s4_port>, <&i2s5_port>, <&i2s6_port>, <&dmic1_port>, <&dmic2_port>, diff --git a/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi b/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi index fcd71bfc67..aff857df25 100644 --- a/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi @@ -44,7 +44,7 @@ mdio { #address-cells = <1>; #size-cells = <0>; - phy: phy@0 { + phy: ethernet-phy@0 { compatible = "ethernet-phy-ieee802.3-c22"; reg = <0x0>; interrupt-parent = <&gpio>; @@ -73,19 +73,19 @@ power-monitor@40 { #address-cells = <1>; #size-cells = <0>; - channel@0 { + input@0 { reg = <0x0>; label = "VDD_SYS_GPU"; shunt-resistor-micro-ohms = <10000>; }; - channel@1 { + input@1 { reg = <0x1>; label = "VDD_SYS_SOC"; shunt-resistor-micro-ohms = <10000>; }; - channel@2 { + input@2 { reg = <0x2>; label = "VDD_3V8_WIFI"; shunt-resistor-micro-ohms = <10000>; @@ -98,19 +98,19 @@ power-monitor@41 { #address-cells = <1>; #size-cells = <0>; - channel@0 { + input@0 { reg = <0x0>; label = "VDD_IN"; shunt-resistor-micro-ohms = <5000>; }; - channel@1 { + input@1 { reg = <0x1>; label = "VDD_SYS_CPU"; shunt-resistor-micro-ohms = <10000>; }; - channel@2 { + input@2 { reg = <0x2>; label = "VDD_5V0_DDR"; shunt-resistor-micro-ohms = <10000>; @@ -393,7 +393,7 @@ psci { method = "smc"; }; - gnd: regulator@0 { + gnd: regulator-gnd { compatible = "regulator-fixed"; regulator-name = "GND"; regulator-min-microvolt = <0>; @@ -402,7 +402,7 @@ gnd: regulator@0 { regulator-boot-on; }; - vdd_5v0_sys: regulator@1 { + vdd_5v0_sys: regulator-vdd-5v0-sys { compatible = "regulator-fixed"; regulator-name = "VDD_5V0_SYS"; regulator-min-microvolt = <5000000>; @@ -411,7 +411,7 @@ vdd_5v0_sys: regulator@1 { regulator-boot-on; }; - vdd_1v8_ap: regulator@2 { + vdd_1v8_ap: regulator-vdd-1v8-ap { compatible = "regulator-fixed"; regulator-name = "VDD_1V8_AP"; regulator-min-microvolt = <1800000>; diff --git a/arch/arm64/boot/dts/nvidia/tegra186-p3509-0000+p3636-0001.dts b/arch/arm64/boot/dts/nvidia/tegra186-p3509-0000+p3636-0001.dts index 936b106e73..4631504c3c 100644 --- a/arch/arm64/boot/dts/nvidia/tegra186-p3509-0000+p3636-0001.dts +++ b/arch/arm64/boot/dts/nvidia/tegra186-p3509-0000+p3636-0001.dts @@ -46,7 +46,7 @@ mdio { #address-cells = <1>; #size-cells = <0>; - phy: phy@0 { + phy: ethernet-phy@0 { compatible = "ethernet-phy-ieee802.3-c22"; reg = <0x0>; interrupt-parent = <&gpio_aon>; @@ -81,22 +81,22 @@ power-monitor@40 { #address-cells = <1>; #size-cells = <0>; - channel@0 { + input@0 { reg = <0>; label = "VDD_IN"; shunt-resistor-micro-ohms = <5>; }; - channel@1 { + input@1 { reg = <1>; label = "VDD_CPU_GPU"; shunt-resistor-micro-ohms = <5>; }; - channel@2 { + input@2 { reg = <2>; label = "VDD_SOC"; - shunt-resistor-micro-ohms = <>; + shunt-resistor-micro-ohms = <5>; }; }; }; @@ -124,7 +124,7 @@ mmc@3460000 { }; hda@3510000 { - nvidia,model = "jetson-tx2-hda"; + nvidia,model = "NVIDIA Jetson TX2 NX HDA"; status = "okay"; }; @@ -593,7 +593,7 @@ psci { method = "smc"; }; - gnd: regulator@0 { + gnd: regulator-gnd { compatible = "regulator-fixed"; regulator-name = "GND"; regulator-min-microvolt = <0>; @@ -602,7 +602,7 @@ gnd: regulator@0 { regulator-boot-on; }; - vdd_5v0_sys: regulator@1 { + vdd_5v0_sys: regulator-vdd-5v0-sys { compatible = "regulator-fixed"; regulator-name = "VDD_5V0_SYS"; regulator-min-microvolt = <5000000>; @@ -611,7 +611,7 @@ vdd_5v0_sys: regulator@1 { regulator-boot-on; }; - vdd_1v8_ap: regulator@2 { + vdd_1v8_ap: regulator-vdd-1v8-ap { compatible = "regulator-fixed"; regulator-name = "VDD_1V8_AP"; regulator-min-microvolt = <1800000>; @@ -623,7 +623,7 @@ vdd_1v8_ap: regulator@2 { vin-supply = <&vdd_1v8>; }; - vdd_hdmi: regulator@3 { + vdd_hdmi: regulator-vdd-hdmi { compatible = "regulator-fixed"; regulator-name = "VDD_5V0_HDMI_CON"; regulator-min-microvolt = <5000000>; @@ -633,7 +633,7 @@ vdd_hdmi: regulator@3 { }; thermal-zones { - cpu { + cpu-thermal { polling-delay = <0>; polling-delay-passive = <500>; status = "okay"; @@ -687,21 +687,7 @@ cpu-passive { }; }; - gpu { - polling-delay = <0>; - polling-delay-passive = <500>; - status = "okay"; - - trips { - gpu_alert0: critical { - temperature = <99000>; - hysteresis = <0>; - type = "critical"; - }; - }; - }; - - aux { + aux-thermal { polling-delay = <0>; polling-delay-passive = <500>; status = "okay"; @@ -714,5 +700,523 @@ aux_alert0: critical { }; }; }; + + gpu-thermal { + polling-delay = <0>; + polling-delay-passive = <500>; + status = "okay"; + + trips { + gpu_alert0: critical { + temperature = <99000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + }; + + aconnect@2900000 { + status = "okay"; + + dma-controller@2930000 { + status = "okay"; + }; + + interrupt-controller@2a40000 { + status = "okay"; + }; + + ahub@2900800 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0x0>; + + xbar_admaif0_ep: endpoint { + remote-endpoint = <&admaif0_ep>; + }; + }; + + port@1 { + reg = <0x1>; + + xbar_admaif1_ep: endpoint { + remote-endpoint = <&admaif1_ep>; + }; + }; + + port@2 { + reg = <0x2>; + + xbar_admaif2_ep: endpoint { + remote-endpoint = <&admaif2_ep>; + }; + }; + + port@3 { + reg = <0x3>; + + xbar_admaif3_ep: endpoint { + remote-endpoint = <&admaif3_ep>; + }; + }; + + port@4 { + reg = <0x4>; + + xbar_admaif4_ep: endpoint { + remote-endpoint = <&admaif4_ep>; + }; + }; + + port@5 { + reg = <0x5>; + + xbar_admaif5_ep: endpoint { + remote-endpoint = <&admaif5_ep>; + }; + }; + + port@6 { + reg = <0x6>; + + xbar_admaif6_ep: endpoint { + remote-endpoint = <&admaif6_ep>; + }; + }; + + port@7 { + reg = <0x7>; + + xbar_admaif7_ep: endpoint { + remote-endpoint = <&admaif7_ep>; + }; + }; + + port@8 { + reg = <0x8>; + + xbar_admaif8_ep: endpoint { + remote-endpoint = <&admaif8_ep>; + }; + }; + + port@9 { + reg = <0x9>; + + xbar_admaif9_ep: endpoint { + remote-endpoint = <&admaif9_ep>; + }; + }; + + port@a { + reg = <0xa>; + + xbar_admaif10_ep: endpoint { + remote-endpoint = <&admaif10_ep>; + }; + }; + + port@b { + reg = <0xb>; + + xbar_admaif11_ep: endpoint { + remote-endpoint = <&admaif11_ep>; + }; + }; + + port@c { + reg = <0xc>; + + xbar_admaif12_ep: endpoint { + remote-endpoint = <&admaif12_ep>; + }; + }; + + port@d { + reg = <0xd>; + + xbar_admaif13_ep: endpoint { + remote-endpoint = <&admaif13_ep>; + }; + }; + + port@e { + reg = <0xe>; + + xbar_admaif14_ep: endpoint { + remote-endpoint = <&admaif14_ep>; + }; + }; + + port@f { + reg = <0xf>; + + xbar_admaif15_ep: endpoint { + remote-endpoint = <&admaif15_ep>; + }; + }; + + port@10 { + reg = <0x10>; + + xbar_admaif16_ep: endpoint { + remote-endpoint = <&admaif16_ep>; + }; + }; + + port@11 { + reg = <0x11>; + + xbar_admaif17_ep: endpoint { + remote-endpoint = <&admaif17_ep>; + }; + }; + + port@12 { + reg = <0x12>; + + xbar_admaif18_ep: endpoint { + remote-endpoint = <&admaif18_ep>; + }; + }; + + port@13 { + reg = <0x13>; + + xbar_admaif19_ep: endpoint { + remote-endpoint = <&admaif19_ep>; + }; + }; + + xbar_i2s1_port: port@14 { + reg = <0x14>; + + xbar_i2s1_ep: endpoint { + remote-endpoint = <&i2s1_cif_ep>; + }; + }; + + xbar_i2s3_port: port@16 { + reg = <0x16>; + + xbar_i2s3_ep: endpoint { + remote-endpoint = <&i2s3_cif_ep>; + }; + }; + + xbar_dmic1_port: port@1a { + reg = <0x1a>; + + xbar_dmic1_ep: endpoint { + remote-endpoint = <&dmic1_cif_ep>; + }; + }; + + xbar_dmic2_port: port@1b { + reg = <0x1b>; + + xbar_dmic2_ep: endpoint { + remote-endpoint = <&dmic2_cif_ep>; + }; + }; + }; + + admaif@290f000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + admaif0_port: port@0 { + reg = <0x0>; + + admaif0_ep: endpoint { + remote-endpoint = <&xbar_admaif0_ep>; + }; + }; + + admaif1_port: port@1 { + reg = <0x1>; + + admaif1_ep: endpoint { + remote-endpoint = <&xbar_admaif1_ep>; + }; + }; + + admaif2_port: port@2 { + reg = <0x2>; + + admaif2_ep: endpoint { + remote-endpoint = <&xbar_admaif2_ep>; + }; + }; + + admaif3_port: port@3 { + reg = <0x3>; + + admaif3_ep: endpoint { + remote-endpoint = <&xbar_admaif3_ep>; + }; + }; + + admaif4_port: port@4 { + reg = <0x4>; + + admaif4_ep: endpoint { + remote-endpoint = <&xbar_admaif4_ep>; + }; + }; + + admaif5_port: port@5 { + reg = <0x5>; + + admaif5_ep: endpoint { + remote-endpoint = <&xbar_admaif5_ep>; + }; + }; + + admaif6_port: port@6 { + reg = <0x6>; + + admaif6_ep: endpoint { + remote-endpoint = <&xbar_admaif6_ep>; + }; + }; + + admaif7_port: port@7 { + reg = <0x7>; + + admaif7_ep: endpoint { + remote-endpoint = <&xbar_admaif7_ep>; + }; + }; + + admaif8_port: port@8 { + reg = <0x8>; + + admaif8_ep: endpoint { + remote-endpoint = <&xbar_admaif8_ep>; + }; + }; + + admaif9_port: port@9 { + reg = <0x9>; + + admaif9_ep: endpoint { + remote-endpoint = <&xbar_admaif9_ep>; + }; + }; + + admaif10_port: port@a { + reg = <0xa>; + + admaif10_ep: endpoint { + remote-endpoint = <&xbar_admaif10_ep>; + }; + }; + + admaif11_port: port@b { + reg = <0xb>; + + admaif11_ep: endpoint { + remote-endpoint = <&xbar_admaif11_ep>; + }; + }; + + admaif12_port: port@c { + reg = <0xc>; + + admaif12_ep: endpoint { + remote-endpoint = <&xbar_admaif12_ep>; + }; + }; + + admaif13_port: port@d { + reg = <0xd>; + + admaif13_ep: endpoint { + remote-endpoint = <&xbar_admaif13_ep>; + }; + }; + + admaif14_port: port@e { + reg = <0xe>; + + admaif14_ep: endpoint { + remote-endpoint = <&xbar_admaif14_ep>; + }; + }; + + admaif15_port: port@f { + reg = <0xf>; + + admaif15_ep: endpoint { + remote-endpoint = <&xbar_admaif15_ep>; + }; + }; + + admaif16_port: port@10 { + reg = <0x10>; + + admaif16_ep: endpoint { + remote-endpoint = <&xbar_admaif16_ep>; + }; + }; + + admaif17_port: port@11 { + reg = <0x11>; + + admaif17_ep: endpoint { + remote-endpoint = <&xbar_admaif17_ep>; + }; + }; + + admaif18_port: port@12 { + reg = <0x12>; + + admaif18_ep: endpoint { + remote-endpoint = <&xbar_admaif18_ep>; + }; + }; + + admaif19_port: port@13 { + reg = <0x13>; + + admaif19_ep: endpoint { + remote-endpoint = <&xbar_admaif19_ep>; + }; + }; + }; + }; + + i2s@2901000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + i2s1_cif_ep: endpoint { + remote-endpoint = <&xbar_i2s1_ep>; + }; + }; + + i2s1_port: port@1 { + reg = <1>; + + i2s1_dap_ep: endpoint { + dai-format = "i2s"; + /* Placeholder for external Codec */ + }; + }; + }; + }; + + i2s@2901200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + i2s3_cif_ep: endpoint { + remote-endpoint = <&xbar_i2s3_ep>; + }; + }; + + i2s3_port: port@1 { + reg = <1>; + + i2s3_dap_ep: endpoint { + dai-format = "i2s"; + /* Placeholder for external Codec */ + }; + }; + }; + }; + + dmic@2904000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dmic1_cif_ep: endpoint { + remote-endpoint = <&xbar_dmic1_ep>; + }; + }; + + dmic1_port: port@1 { + reg = <1>; + + dmic1_dap_ep: endpoint { + /* Place holder for external Codec */ + }; + }; + }; + }; + + dmic@2904100 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dmic2_cif_ep: endpoint { + remote-endpoint = <&xbar_dmic2_ep>; + }; + }; + + dmic2_port: port@1 { + reg = <1>; + + dmic2_dap_ep: endpoint { + /* Place holder for external Codec */ + }; + }; + }; + }; + }; + }; + + sound { + compatible = "nvidia,tegra186-audio-graph-card"; + status = "okay"; + + dais = /* FE */ + <&admaif0_port>, <&admaif1_port>, <&admaif2_port>, <&admaif3_port>, + <&admaif4_port>, <&admaif5_port>, <&admaif6_port>, <&admaif7_port>, + <&admaif8_port>, <&admaif9_port>, <&admaif10_port>, <&admaif11_port>, + <&admaif12_port>, <&admaif13_port>, <&admaif14_port>, <&admaif15_port>, + <&admaif16_port>, <&admaif17_port>, <&admaif18_port>, <&admaif19_port>, + /* Router */ + <&xbar_i2s1_port>, <&xbar_i2s3_port>, + <&xbar_dmic1_port>, <&xbar_dmic2_port>, + /* I/O */ + <&i2s1_port>, <&i2s3_port>, + <&dmic1_port>, <&dmic2_port>; + + label = "NVIDIA Jetson TX2 NX APE"; }; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra186.dtsi b/arch/arm64/boot/dts/nvidia/tegra186.dtsi index 062e87e893..c91afff1b7 100644 --- a/arch/arm64/boot/dts/nvidia/tegra186.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra186.dtsi @@ -354,6 +354,126 @@ tegra_dspk2: dspk@2905100 { sound-name-prefix = "DSPK2"; status = "disabled"; }; + + tegra_sfc1: sfc@2902000 { + compatible = "nvidia,tegra186-sfc", + "nvidia,tegra210-sfc"; + reg = <0x2902000 0x200>; + sound-name-prefix = "SFC1"; + status = "disabled"; + }; + + tegra_sfc2: sfc@2902200 { + compatible = "nvidia,tegra186-sfc", + "nvidia,tegra210-sfc"; + reg = <0x2902200 0x200>; + sound-name-prefix = "SFC2"; + status = "disabled"; + }; + + tegra_sfc3: sfc@2902400 { + compatible = "nvidia,tegra186-sfc", + "nvidia,tegra210-sfc"; + reg = <0x2902400 0x200>; + sound-name-prefix = "SFC3"; + status = "disabled"; + }; + + tegra_sfc4: sfc@2902600 { + compatible = "nvidia,tegra186-sfc", + "nvidia,tegra210-sfc"; + reg = <0x2902600 0x200>; + sound-name-prefix = "SFC4"; + status = "disabled"; + }; + + tegra_mvc1: mvc@290a000 { + compatible = "nvidia,tegra186-mvc", + "nvidia,tegra210-mvc"; + reg = <0x290a000 0x200>; + sound-name-prefix = "MVC1"; + status = "disabled"; + }; + + tegra_mvc2: mvc@290a200 { + compatible = "nvidia,tegra186-mvc", + "nvidia,tegra210-mvc"; + reg = <0x290a200 0x200>; + sound-name-prefix = "MVC2"; + status = "disabled"; + }; + + tegra_amx1: amx@2903000 { + compatible = "nvidia,tegra186-amx", + "nvidia,tegra210-amx"; + reg = <0x2903000 0x100>; + sound-name-prefix = "AMX1"; + status = "disabled"; + }; + + tegra_amx2: amx@2903100 { + compatible = "nvidia,tegra186-amx", + "nvidia,tegra210-amx"; + reg = <0x2903100 0x100>; + sound-name-prefix = "AMX2"; + status = "disabled"; + }; + + tegra_amx3: amx@2903200 { + compatible = "nvidia,tegra186-amx", + "nvidia,tegra210-amx"; + reg = <0x2903200 0x100>; + sound-name-prefix = "AMX3"; + status = "disabled"; + }; + + tegra_amx4: amx@2903300 { + compatible = "nvidia,tegra186-amx", + "nvidia,tegra210-amx"; + reg = <0x2903300 0x100>; + sound-name-prefix = "AMX4"; + status = "disabled"; + }; + + tegra_adx1: adx@2903800 { + compatible = "nvidia,tegra186-adx", + "nvidia,tegra210-adx"; + reg = <0x2903800 0x100>; + sound-name-prefix = "ADX1"; + status = "disabled"; + }; + + tegra_adx2: adx@2903900 { + compatible = "nvidia,tegra186-adx", + "nvidia,tegra210-adx"; + reg = <0x2903900 0x100>; + sound-name-prefix = "ADX2"; + status = "disabled"; + }; + + tegra_adx3: adx@2903a00 { + compatible = "nvidia,tegra186-adx", + "nvidia,tegra210-adx"; + reg = <0x2903a00 0x100>; + sound-name-prefix = "ADX3"; + status = "disabled"; + }; + + tegra_adx4: adx@2903b00 { + compatible = "nvidia,tegra186-adx", + "nvidia,tegra210-adx"; + reg = <0x2903b00 0x100>; + sound-name-prefix = "ADX4"; + status = "disabled"; + }; + + tegra_amixer: amixer@290bb00 { + compatible = "nvidia,tegra186-amixer", + "nvidia,tegra210-amixer"; + reg = <0x290bb00 0x800>; + sound-name-prefix = "MIXER1"; + status = "disabled"; + }; }; }; @@ -388,6 +508,22 @@ emc: external-memory-controller@2c60000 { }; }; + timer@3010000 { + compatible = "nvidia,tegra186-timer"; + reg = <0x0 0x03010000 0x0 0x000e0000>; + interrupts = , + , + , + , + , + , + , + , + , + ; + status = "disabled"; + }; + uarta: serial@3100000 { compatible = "nvidia,tegra186-uart", "nvidia,tegra20-uart"; reg = <0x0 0x03100000 0x0 0x40>; @@ -449,7 +585,7 @@ uartf: serial@3150000 { }; gen1_i2c: i2c@3160000 { - compatible = "nvidia,tegra186-i2c", "nvidia,tegra210-i2c"; + compatible = "nvidia,tegra186-i2c"; reg = <0x0 0x03160000 0x0 0x10000>; interrupts = ; #address-cells = <1>; @@ -462,7 +598,7 @@ gen1_i2c: i2c@3160000 { }; cam_i2c: i2c@3180000 { - compatible = "nvidia,tegra186-i2c", "nvidia,tegra210-i2c"; + compatible = "nvidia,tegra186-i2c"; reg = <0x0 0x03180000 0x0 0x10000>; interrupts = ; #address-cells = <1>; @@ -476,7 +612,7 @@ cam_i2c: i2c@3180000 { /* shares pads with dpaux1 */ dp_aux_ch1_i2c: i2c@3190000 { - compatible = "nvidia,tegra186-i2c", "nvidia,tegra210-i2c"; + compatible = "nvidia,tegra186-i2c"; reg = <0x0 0x03190000 0x0 0x10000>; interrupts = ; #address-cells = <1>; @@ -493,7 +629,7 @@ dp_aux_ch1_i2c: i2c@3190000 { /* controlled by BPMP, should not be enabled */ pwr_i2c: i2c@31a0000 { - compatible = "nvidia,tegra186-i2c", "nvidia,tegra210-i2c"; + compatible = "nvidia,tegra186-i2c"; reg = <0x0 0x031a0000 0x0 0x10000>; interrupts = ; #address-cells = <1>; @@ -507,7 +643,7 @@ pwr_i2c: i2c@31a0000 { /* shares pads with dpaux0 */ dp_aux_ch0_i2c: i2c@31b0000 { - compatible = "nvidia,tegra186-i2c", "nvidia,tegra210-i2c"; + compatible = "nvidia,tegra186-i2c"; reg = <0x0 0x031b0000 0x0 0x10000>; interrupts = ; #address-cells = <1>; @@ -523,7 +659,7 @@ dp_aux_ch0_i2c: i2c@31b0000 { }; gen7_i2c: i2c@31c0000 { - compatible = "nvidia,tegra186-i2c", "nvidia,tegra210-i2c"; + compatible = "nvidia,tegra186-i2c"; reg = <0x0 0x031c0000 0x0 0x10000>; interrupts = ; #address-cells = <1>; @@ -536,7 +672,7 @@ gen7_i2c: i2c@31c0000 { }; gen9_i2c: i2c@31e0000 { - compatible = "nvidia,tegra186-i2c", "nvidia,tegra210-i2c"; + compatible = "nvidia,tegra186-i2c"; reg = <0x0 0x031e0000 0x0 0x10000>; interrupts = ; #address-cells = <1>; @@ -953,7 +1089,7 @@ hsp_top0: hsp@3c00000 { }; gen2_i2c: i2c@c240000 { - compatible = "nvidia,tegra186-i2c", "nvidia,tegra210-i2c"; + compatible = "nvidia,tegra186-i2c"; reg = <0x0 0x0c240000 0x0 0x10000>; interrupts = ; #address-cells = <1>; @@ -966,7 +1102,7 @@ gen2_i2c: i2c@c240000 { }; gen8_i2c: i2c@c250000 { - compatible = "nvidia,tegra186-i2c", "nvidia,tegra210-i2c"; + compatible = "nvidia,tegra186-i2c"; reg = <0x0 0x0c250000 0x0 0x10000>; interrupts = ; #address-cells = <1>; @@ -1418,6 +1554,21 @@ vic@15340000 { iommus = <&smmu TEGRA186_SID_VIC>; }; + nvjpg@15380000 { + compatible = "nvidia,tegra186-nvjpg"; + reg = <0x15380000 0x40000>; + clocks = <&bpmp TEGRA186_CLK_NVJPG>; + clock-names = "nvjpg"; + resets = <&bpmp TEGRA186_RESET_NVJPG>; + reset-names = "nvjpg"; + + power-domains = <&bpmp TEGRA186_POWER_DOMAIN_NVJPG>; + interconnects = <&mc TEGRA186_MEMORY_CLIENT_NVJPGSRD &emc>, + <&mc TEGRA186_MEMORY_CLIENT_NVJPGSWR &emc>; + interconnect-names = "dma-mem", "write"; + iommus = <&smmu TEGRA186_SID_NVJPG>; + }; + dsib: dsi@15400000 { compatible = "nvidia,tegra186-dsi"; reg = <0x15400000 0x10000>; @@ -1433,6 +1584,37 @@ dsib: dsi@15400000 { power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>; }; + nvdec@15480000 { + compatible = "nvidia,tegra186-nvdec"; + reg = <0x15480000 0x40000>; + clocks = <&bpmp TEGRA186_CLK_NVDEC>; + clock-names = "nvdec"; + resets = <&bpmp TEGRA186_RESET_NVDEC>; + reset-names = "nvdec"; + + power-domains = <&bpmp TEGRA186_POWER_DOMAIN_NVDEC>; + interconnects = <&mc TEGRA186_MEMORY_CLIENT_NVDECSRD &emc>, + <&mc TEGRA186_MEMORY_CLIENT_NVDECSRD1 &emc>, + <&mc TEGRA186_MEMORY_CLIENT_NVDECSWR &emc>; + interconnect-names = "dma-mem", "read-1", "write"; + iommus = <&smmu TEGRA186_SID_NVDEC>; + }; + + nvenc@154c0000 { + compatible = "nvidia,tegra186-nvenc"; + reg = <0x154c0000 0x40000>; + clocks = <&bpmp TEGRA186_CLK_NVENC>; + clock-names = "nvenc"; + resets = <&bpmp TEGRA186_RESET_NVENC>; + reset-names = "nvenc"; + + power-domains = <&bpmp TEGRA186_POWER_DOMAIN_MPE>; + interconnects = <&mc TEGRA186_MEMORY_CLIENT_NVENCSRD &emc>, + <&mc TEGRA186_MEMORY_CLIENT_NVENCSWR &emc>; + interconnect-names = "dma-mem", "write"; + iommus = <&smmu TEGRA186_SID_NVENC>; + }; + sor0: sor@15540000 { compatible = "nvidia,tegra186-sor"; reg = <0x15540000 0x10000>; @@ -1635,7 +1817,7 @@ bpmp: bpmp { iommus = <&smmu TEGRA186_SID_BPMP>; mboxes = <&hsp_top0 TEGRA_HSP_MBOX_TYPE_DB TEGRA_HSP_DB_MASTER_BPMP>; - shmem = <&cpu_bpmp_tx &cpu_bpmp_rx>; + shmem = <&cpu_bpmp_tx>, <&cpu_bpmp_rx>; #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; @@ -1794,12 +1976,12 @@ sound { }; thermal-zones { - a57 { + /* Cortex-A57 cluster */ + cpu-thermal { polling-delay = <0>; polling-delay-passive = <1000>; - thermal-sensors = - <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_CPU>; + thermal-sensors = <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_CPU>; trips { critical { @@ -1813,12 +1995,12 @@ cooling-maps { }; }; - denver { + /* Denver cluster */ + aux-thermal { polling-delay = <0>; polling-delay-passive = <1000>; - thermal-sensors = - <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_AUX>; + thermal-sensors = <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_AUX>; trips { critical { @@ -1832,12 +2014,11 @@ cooling-maps { }; }; - gpu { + gpu-thermal { polling-delay = <0>; polling-delay-passive = <1000>; - thermal-sensors = - <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_GPU>; + thermal-sensors = <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_GPU>; trips { critical { @@ -1851,12 +2032,11 @@ cooling-maps { }; }; - pll { + pll-thermal { polling-delay = <0>; polling-delay-passive = <1000>; - thermal-sensors = - <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_PLLX>; + thermal-sensors = <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_PLLX>; trips { critical { @@ -1870,12 +2050,11 @@ cooling-maps { }; }; - always_on { + ao-thermal { polling-delay = <0>; polling-delay-passive = <1000>; - thermal-sensors = - <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_AO>; + thermal-sensors = <&bpmp_thermal TEGRA186_BPMP_THERMAL_ZONE_AO>; trips { critical { diff --git a/arch/arm64/boot/dts/nvidia/tegra194-p2888.dtsi b/arch/arm64/boot/dts/nvidia/tegra194-p2888.dtsi index c4058ee36f..a7d7cfd663 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194-p2888.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194-p2888.dtsi @@ -39,7 +39,7 @@ mdio { #address-cells = <1>; #size-cells = <0>; - phy: phy@0 { + phy: ethernet-phy@0 { compatible = "ethernet-phy-ieee802.3-c22"; reg = <0x0>; interrupt-parent = <&gpio>; @@ -317,7 +317,7 @@ temperature-sensor@4c { }; }; - vdd_5v0_sys: regulator@0 { + vdd_5v0_sys: regulator-vdd-5v0-sys { compatible = "regulator-fixed"; regulator-name = "VIN_SYS_5V0"; regulator-min-microvolt = <5000000>; @@ -326,7 +326,7 @@ vdd_5v0_sys: regulator@0 { regulator-boot-on; }; - vdd_hdmi: regulator@1 { + vdd_hdmi: regulator-vdd-hdmi { compatible = "regulator-fixed"; regulator-name = "VDD_5V0_HDMI_CON"; regulator-min-microvolt = <5000000>; @@ -335,7 +335,7 @@ vdd_hdmi: regulator@1 { enable-active-high; }; - vdd_3v3_pcie: regulator@2 { + vdd_3v3_pcie: regulator-vdd-3v3-pcie { compatible = "regulator-fixed"; regulator-name = "PEX_3V3"; regulator-min-microvolt = <3300000>; @@ -345,7 +345,7 @@ vdd_3v3_pcie: regulator@2 { enable-active-high; }; - vdd_12v_pcie: regulator@3 { + vdd_12v_pcie: regulator-vdd-12v-pcie { compatible = "regulator-fixed"; regulator-name = "VDD_12V"; regulator-min-microvolt = <1200000>; @@ -354,7 +354,7 @@ vdd_12v_pcie: regulator@3 { regulator-boot-on; }; - vdd_5v_sata: regulator@4 { + vdd_5v_sata: regulator-vdd-5v0-sata { compatible = "regulator-fixed"; regulator-name = "VDD_5V_SATA"; regulator-min-microvolt = <5000000>; diff --git a/arch/arm64/boot/dts/nvidia/tegra194-p2972-0000.dts b/arch/arm64/boot/dts/nvidia/tegra194-p2972-0000.dts index 96bd01cadb..2478ece9e6 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194-p2972-0000.dts +++ b/arch/arm64/boot/dts/nvidia/tegra194-p2972-0000.dts @@ -228,6 +228,542 @@ xbar_dmic3_ep: endpoint { remote-endpoint = <&dmic3_cif_ep>; }; }; + + xbar_sfc1_in_port: port@20 { + reg = <0x20>; + + xbar_sfc1_in_ep: endpoint { + remote-endpoint = <&sfc1_cif_in_ep>; + }; + }; + + port@21 { + reg = <0x21>; + + xbar_sfc1_out_ep: endpoint { + remote-endpoint = <&sfc1_cif_out_ep>; + }; + }; + + xbar_sfc2_in_port: port@22 { + reg = <0x22>; + + xbar_sfc2_in_ep: endpoint { + remote-endpoint = <&sfc2_cif_in_ep>; + }; + }; + + port@23 { + reg = <0x23>; + + xbar_sfc2_out_ep: endpoint { + remote-endpoint = <&sfc2_cif_out_ep>; + }; + }; + + xbar_sfc3_in_port: port@24 { + reg = <0x24>; + + xbar_sfc3_in_ep: endpoint { + remote-endpoint = <&sfc3_cif_in_ep>; + }; + }; + + port@25 { + reg = <0x25>; + + xbar_sfc3_out_ep: endpoint { + remote-endpoint = <&sfc3_cif_out_ep>; + }; + }; + + xbar_sfc4_in_port: port@26 { + reg = <0x26>; + + xbar_sfc4_in_ep: endpoint { + remote-endpoint = <&sfc4_cif_in_ep>; + }; + }; + + port@27 { + reg = <0x27>; + + xbar_sfc4_out_ep: endpoint { + remote-endpoint = <&sfc4_cif_out_ep>; + }; + }; + + xbar_mvc1_in_port: port@28 { + reg = <0x28>; + + xbar_mvc1_in_ep: endpoint { + remote-endpoint = <&mvc1_cif_in_ep>; + }; + }; + + port@29 { + reg = <0x29>; + + xbar_mvc1_out_ep: endpoint { + remote-endpoint = <&mvc1_cif_out_ep>; + }; + }; + + xbar_mvc2_in_port: port@2a { + reg = <0x2a>; + + xbar_mvc2_in_ep: endpoint { + remote-endpoint = <&mvc2_cif_in_ep>; + }; + }; + + port@2b { + reg = <0x2b>; + + xbar_mvc2_out_ep: endpoint { + remote-endpoint = <&mvc2_cif_out_ep>; + }; + }; + + xbar_amx1_in1_port: port@2c { + reg = <0x2c>; + + xbar_amx1_in1_ep: endpoint { + remote-endpoint = <&amx1_in1_ep>; + }; + }; + + xbar_amx1_in2_port: port@2d { + reg = <0x2d>; + + xbar_amx1_in2_ep: endpoint { + remote-endpoint = <&amx1_in2_ep>; + }; + }; + + xbar_amx1_in3_port: port@2e { + reg = <0x2e>; + + xbar_amx1_in3_ep: endpoint { + remote-endpoint = <&amx1_in3_ep>; + }; + }; + + xbar_amx1_in4_port: port@2f { + reg = <0x2f>; + + xbar_amx1_in4_ep: endpoint { + remote-endpoint = <&amx1_in4_ep>; + }; + }; + + port@30 { + reg = <0x30>; + + xbar_amx1_out_ep: endpoint { + remote-endpoint = <&amx1_out_ep>; + }; + }; + + xbar_amx2_in1_port: port@31 { + reg = <0x31>; + + xbar_amx2_in1_ep: endpoint { + remote-endpoint = <&amx2_in1_ep>; + }; + }; + + xbar_amx2_in2_port: port@32 { + reg = <0x32>; + + xbar_amx2_in2_ep: endpoint { + remote-endpoint = <&amx2_in2_ep>; + }; + }; + + xbar_amx2_in3_port: port@33 { + reg = <0x33>; + + xbar_amx2_in3_ep: endpoint { + remote-endpoint = <&amx2_in3_ep>; + }; + }; + + xbar_amx2_in4_port: port@34 { + reg = <0x34>; + + xbar_amx2_in4_ep: endpoint { + remote-endpoint = <&amx2_in4_ep>; + }; + }; + + port@35 { + reg = <0x35>; + + xbar_amx2_out_ep: endpoint { + remote-endpoint = <&amx2_out_ep>; + }; + }; + + xbar_amx3_in1_port: port@36 { + reg = <0x36>; + + xbar_amx3_in1_ep: endpoint { + remote-endpoint = <&amx3_in1_ep>; + }; + }; + + xbar_amx3_in2_port: port@37 { + reg = <0x37>; + + xbar_amx3_in2_ep: endpoint { + remote-endpoint = <&amx3_in2_ep>; + }; + }; + + xbar_amx3_in3_port: port@38 { + reg = <0x38>; + + xbar_amx3_in3_ep: endpoint { + remote-endpoint = <&amx3_in3_ep>; + }; + }; + + xbar_amx3_in4_port: port@39 { + reg = <0x39>; + + xbar_amx3_in4_ep: endpoint { + remote-endpoint = <&amx3_in4_ep>; + }; + }; + + port@3a { + reg = <0x3a>; + + xbar_amx3_out_ep: endpoint { + remote-endpoint = <&amx3_out_ep>; + }; + }; + + xbar_amx4_in1_port: port@3b { + reg = <0x3b>; + + xbar_amx4_in1_ep: endpoint { + remote-endpoint = <&amx4_in1_ep>; + }; + }; + + xbar_amx4_in2_port: port@3c { + reg = <0x3c>; + + xbar_amx4_in2_ep: endpoint { + remote-endpoint = <&amx4_in2_ep>; + }; + }; + + xbar_amx4_in3_port: port@3d { + reg = <0x3d>; + + xbar_amx4_in3_ep: endpoint { + remote-endpoint = <&amx4_in3_ep>; + }; + }; + + xbar_amx4_in4_port: port@3e { + reg = <0x3e>; + + xbar_amx4_in4_ep: endpoint { + remote-endpoint = <&amx4_in4_ep>; + }; + }; + + port@3f { + reg = <0x3f>; + + xbar_amx4_out_ep: endpoint { + remote-endpoint = <&amx4_out_ep>; + }; + }; + + xbar_adx1_in_port: port@40 { + reg = <0x40>; + + xbar_adx1_in_ep: endpoint { + remote-endpoint = <&adx1_in_ep>; + }; + }; + + port@41 { + reg = <0x41>; + + xbar_adx1_out1_ep: endpoint { + remote-endpoint = <&adx1_out1_ep>; + }; + }; + + port@42 { + reg = <0x42>; + + xbar_adx1_out2_ep: endpoint { + remote-endpoint = <&adx1_out2_ep>; + }; + }; + + port@43 { + reg = <0x43>; + + xbar_adx1_out3_ep: endpoint { + remote-endpoint = <&adx1_out3_ep>; + }; + }; + + port@44 { + reg = <0x44>; + + xbar_adx1_out4_ep: endpoint { + remote-endpoint = <&adx1_out4_ep>; + }; + }; + + xbar_adx2_in_port: port@45 { + reg = <0x45>; + + xbar_adx2_in_ep: endpoint { + remote-endpoint = <&adx2_in_ep>; + }; + }; + + port@46 { + reg = <0x46>; + + xbar_adx2_out1_ep: endpoint { + remote-endpoint = <&adx2_out1_ep>; + }; + }; + + port@47 { + reg = <0x47>; + + xbar_adx2_out2_ep: endpoint { + remote-endpoint = <&adx2_out2_ep>; + }; + }; + + port@48 { + reg = <0x48>; + + xbar_adx2_out3_ep: endpoint { + remote-endpoint = <&adx2_out3_ep>; + }; + }; + + port@49 { + reg = <0x49>; + + xbar_adx2_out4_ep: endpoint { + remote-endpoint = <&adx2_out4_ep>; + }; + }; + + xbar_adx3_in_port: port@4a { + reg = <0x4a>; + + xbar_adx3_in_ep: endpoint { + remote-endpoint = <&adx3_in_ep>; + }; + }; + + port@4b { + reg = <0x4b>; + + xbar_adx3_out1_ep: endpoint { + remote-endpoint = <&adx3_out1_ep>; + }; + }; + + port@4c { + reg = <0x4c>; + + xbar_adx3_out2_ep: endpoint { + remote-endpoint = <&adx3_out2_ep>; + }; + }; + + port@4d { + reg = <0x4d>; + + xbar_adx3_out3_ep: endpoint { + remote-endpoint = <&adx3_out3_ep>; + }; + }; + + port@4e { + reg = <0x4e>; + + xbar_adx3_out4_ep: endpoint { + remote-endpoint = <&adx3_out4_ep>; + }; + }; + + xbar_adx4_in_port: port@4f { + reg = <0x4f>; + + xbar_adx4_in_ep: endpoint { + remote-endpoint = <&adx4_in_ep>; + }; + }; + + port@50 { + reg = <0x50>; + + xbar_adx4_out1_ep: endpoint { + remote-endpoint = <&adx4_out1_ep>; + }; + }; + + port@51 { + reg = <0x51>; + + xbar_adx4_out2_ep: endpoint { + remote-endpoint = <&adx4_out2_ep>; + }; + }; + + port@52 { + reg = <0x52>; + + xbar_adx4_out3_ep: endpoint { + remote-endpoint = <&adx4_out3_ep>; + }; + }; + + port@53 { + reg = <0x53>; + + xbar_adx4_out4_ep: endpoint { + remote-endpoint = <&adx4_out4_ep>; + }; + }; + + xbar_mixer_in1_port: port@54 { + reg = <0x54>; + + xbar_mixer_in1_ep: endpoint { + remote-endpoint = <&mixer_in1_ep>; + }; + }; + + xbar_mixer_in2_port: port@55 { + reg = <0x55>; + + xbar_mixer_in2_ep: endpoint { + remote-endpoint = <&mixer_in2_ep>; + }; + }; + + xbar_mixer_in3_port: port@56 { + reg = <0x56>; + + xbar_mixer_in3_ep: endpoint { + remote-endpoint = <&mixer_in3_ep>; + }; + }; + + xbar_mixer_in4_port: port@57 { + reg = <0x57>; + + xbar_mixer_in4_ep: endpoint { + remote-endpoint = <&mixer_in4_ep>; + }; + }; + + xbar_mixer_in5_port: port@58 { + reg = <0x58>; + + xbar_mixer_in5_ep: endpoint { + remote-endpoint = <&mixer_in5_ep>; + }; + }; + + xbar_mixer_in6_port: port@59 { + reg = <0x59>; + + xbar_mixer_in6_ep: endpoint { + remote-endpoint = <&mixer_in6_ep>; + }; + }; + + xbar_mixer_in7_port: port@5a { + reg = <0x5a>; + + xbar_mixer_in7_ep: endpoint { + remote-endpoint = <&mixer_in7_ep>; + }; + }; + + xbar_mixer_in8_port: port@5b { + reg = <0x5b>; + + xbar_mixer_in8_ep: endpoint { + remote-endpoint = <&mixer_in8_ep>; + }; + }; + + xbar_mixer_in9_port: port@5c { + reg = <0x5c>; + + xbar_mixer_in9_ep: endpoint { + remote-endpoint = <&mixer_in9_ep>; + }; + }; + + xbar_mixer_in10_port: port@5d { + reg = <0x5d>; + + xbar_mixer_in10_ep: endpoint { + remote-endpoint = <&mixer_in10_ep>; + }; + }; + + port@5e { + reg = <0x5e>; + + xbar_mixer_out1_ep: endpoint { + remote-endpoint = <&mixer_out1_ep>; + }; + }; + + port@5f { + reg = <0x5f>; + + xbar_mixer_out2_ep: endpoint { + remote-endpoint = <&mixer_out2_ep>; + }; + }; + + port@60 { + reg = <0x60>; + + xbar_mixer_out3_ep: endpoint { + remote-endpoint = <&mixer_out3_ep>; + }; + }; + + port@61 { + reg = <0x61>; + + xbar_mixer_out4_ep: endpoint { + remote-endpoint = <&mixer_out4_ep>; + }; + }; + + port@62 { + reg = <0x62>; + + xbar_mixer_out5_ep: endpoint { + remote-endpoint = <&mixer_out5_ep>; + }; + }; }; admaif@290f000 { @@ -495,7 +1031,7 @@ i2s6_cif_ep: endpoint { i2s6_port: port@1 { reg = <1>; - i2s6_dap_ep: endpoint@0 { + i2s6_dap_ep: endpoint { dai-format = "i2s"; /* Place holder for external Codec */ }; @@ -527,6 +1063,677 @@ dmic3_dap_ep: endpoint { }; }; }; + + sfc@2902000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc1_in_ep>; + }; + }; + + sfc1_out_port: port@1 { + reg = <1>; + + sfc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc1_out_ep>; + }; + }; + }; + }; + + sfc@2902200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc2_in_ep>; + }; + }; + + sfc2_out_port: port@1 { + reg = <1>; + + sfc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc2_out_ep>; + }; + }; + }; + }; + + sfc@2902400 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc3_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc3_in_ep>; + }; + }; + + sfc3_out_port: port@1 { + reg = <1>; + + sfc3_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc3_out_ep>; + }; + }; + }; + }; + + sfc@2902600 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc4_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc4_in_ep>; + }; + }; + + sfc4_out_port: port@1 { + reg = <1>; + + sfc4_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc4_out_ep>; + }; + }; + }; + }; + + mvc@290a000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc1_in_ep>; + }; + }; + + mvc1_out_port: port@1 { + reg = <1>; + + mvc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc1_out_ep>; + }; + }; + }; + }; + + mvc@290a200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc2_in_ep>; + }; + }; + + mvc2_out_port: port@1 { + reg = <1>; + + mvc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc2_out_ep>; + }; + }; + }; + }; + + amx@2903000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx1_in1_ep: endpoint { + remote-endpoint = <&xbar_amx1_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx1_in2_ep: endpoint { + remote-endpoint = <&xbar_amx1_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx1_in3_ep: endpoint { + remote-endpoint = <&xbar_amx1_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx1_in4_ep: endpoint { + remote-endpoint = <&xbar_amx1_in4_ep>; + }; + }; + + amx1_out_port: port@4 { + reg = <4>; + + amx1_out_ep: endpoint { + remote-endpoint = <&xbar_amx1_out_ep>; + }; + }; + }; + }; + + amx@2903100 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx2_in1_ep: endpoint { + remote-endpoint = <&xbar_amx2_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx2_in2_ep: endpoint { + remote-endpoint = <&xbar_amx2_in2_ep>; + }; + }; + + amx2_in3_port: port@2 { + reg = <2>; + + amx2_in3_ep: endpoint { + remote-endpoint = <&xbar_amx2_in3_ep>; + }; + }; + + amx2_in4_port: port@3 { + reg = <3>; + + amx2_in4_ep: endpoint { + remote-endpoint = <&xbar_amx2_in4_ep>; + }; + }; + + amx2_out_port: port@4 { + reg = <4>; + + amx2_out_ep: endpoint { + remote-endpoint = <&xbar_amx2_out_ep>; + }; + }; + }; + }; + + amx@2903200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx3_in1_ep: endpoint { + remote-endpoint = <&xbar_amx3_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx3_in2_ep: endpoint { + remote-endpoint = <&xbar_amx3_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx3_in3_ep: endpoint { + remote-endpoint = <&xbar_amx3_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx3_in4_ep: endpoint { + remote-endpoint = <&xbar_amx3_in4_ep>; + }; + }; + + amx3_out_port: port@4 { + reg = <4>; + + amx3_out_ep: endpoint { + remote-endpoint = <&xbar_amx3_out_ep>; + }; + }; + }; + }; + + amx@2903300 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx4_in1_ep: endpoint { + remote-endpoint = <&xbar_amx4_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx4_in2_ep: endpoint { + remote-endpoint = <&xbar_amx4_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx4_in3_ep: endpoint { + remote-endpoint = <&xbar_amx4_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx4_in4_ep: endpoint { + remote-endpoint = <&xbar_amx4_in4_ep>; + }; + }; + + amx4_out_port: port@4 { + reg = <4>; + + amx4_out_ep: endpoint { + remote-endpoint = <&xbar_amx4_out_ep>; + }; + }; + }; + }; + + adx@2903800 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx1_in_ep: endpoint { + remote-endpoint = <&xbar_adx1_in_ep>; + }; + }; + + adx1_out1_port: port@1 { + reg = <1>; + + adx1_out1_ep: endpoint { + remote-endpoint = <&xbar_adx1_out1_ep>; + }; + }; + + adx1_out2_port: port@2 { + reg = <2>; + + adx1_out2_ep: endpoint { + remote-endpoint = <&xbar_adx1_out2_ep>; + }; + }; + + adx1_out3_port: port@3 { + reg = <3>; + + adx1_out3_ep: endpoint { + remote-endpoint = <&xbar_adx1_out3_ep>; + }; + }; + + adx1_out4_port: port@4 { + reg = <4>; + + adx1_out4_ep: endpoint { + remote-endpoint = <&xbar_adx1_out4_ep>; + }; + }; + }; + }; + + adx@2903900 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx2_in_ep: endpoint { + remote-endpoint = <&xbar_adx2_in_ep>; + }; + }; + + adx2_out1_port: port@1 { + reg = <1>; + + adx2_out1_ep: endpoint { + remote-endpoint = <&xbar_adx2_out1_ep>; + }; + }; + + adx2_out2_port: port@2 { + reg = <2>; + + adx2_out2_ep: endpoint { + remote-endpoint = <&xbar_adx2_out2_ep>; + }; + }; + + adx2_out3_port: port@3 { + reg = <3>; + + adx2_out3_ep: endpoint { + remote-endpoint = <&xbar_adx2_out3_ep>; + }; + }; + + adx2_out4_port: port@4 { + reg = <4>; + + adx2_out4_ep: endpoint { + remote-endpoint = <&xbar_adx2_out4_ep>; + }; + }; + }; + }; + + adx@2903a00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx3_in_ep: endpoint { + remote-endpoint = <&xbar_adx3_in_ep>; + }; + }; + + adx3_out1_port: port@1 { + reg = <1>; + + adx3_out1_ep: endpoint { + remote-endpoint = <&xbar_adx3_out1_ep>; + }; + }; + + adx3_out2_port: port@2 { + reg = <2>; + + adx3_out2_ep: endpoint { + remote-endpoint = <&xbar_adx3_out2_ep>; + }; + }; + + adx3_out3_port: port@3 { + reg = <3>; + + adx3_out3_ep: endpoint { + remote-endpoint = <&xbar_adx3_out3_ep>; + }; + }; + + adx3_out4_port: port@4 { + reg = <4>; + + adx3_out4_ep: endpoint { + remote-endpoint = <&xbar_adx3_out4_ep>; + }; + }; + }; + }; + + adx@2903b00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx4_in_ep: endpoint { + remote-endpoint = <&xbar_adx4_in_ep>; + }; + }; + + adx4_out1_port: port@1 { + reg = <1>; + + adx4_out1_ep: endpoint { + remote-endpoint = <&xbar_adx4_out1_ep>; + }; + }; + + adx4_out2_port: port@2 { + reg = <2>; + + adx4_out2_ep: endpoint { + remote-endpoint = <&xbar_adx4_out2_ep>; + }; + }; + + adx4_out3_port: port@3 { + reg = <3>; + + adx4_out3_ep: endpoint { + remote-endpoint = <&xbar_adx4_out3_ep>; + }; + }; + + adx4_out4_port: port@4 { + reg = <4>; + + adx4_out4_ep: endpoint { + remote-endpoint = <&xbar_adx4_out4_ep>; + }; + }; + }; + }; + + amixer@290bb00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0x0>; + + mixer_in1_ep: endpoint { + remote-endpoint = <&xbar_mixer_in1_ep>; + }; + }; + + port@1 { + reg = <0x1>; + + mixer_in2_ep: endpoint { + remote-endpoint = <&xbar_mixer_in2_ep>; + }; + }; + + port@2 { + reg = <0x2>; + + mixer_in3_ep: endpoint { + remote-endpoint = <&xbar_mixer_in3_ep>; + }; + }; + + port@3 { + reg = <0x3>; + + mixer_in4_ep: endpoint { + remote-endpoint = <&xbar_mixer_in4_ep>; + }; + }; + + port@4 { + reg = <0x4>; + + mixer_in5_ep: endpoint { + remote-endpoint = <&xbar_mixer_in5_ep>; + }; + }; + + port@5 { + reg = <0x5>; + + mixer_in6_ep: endpoint { + remote-endpoint = <&xbar_mixer_in6_ep>; + }; + }; + + port@6 { + reg = <0x6>; + + mixer_in7_ep: endpoint { + remote-endpoint = <&xbar_mixer_in7_ep>; + }; + }; + + port@7 { + reg = <0x7>; + + mixer_in8_ep: endpoint { + remote-endpoint = <&xbar_mixer_in8_ep>; + }; + }; + + port@8 { + reg = <0x8>; + + mixer_in9_ep: endpoint { + remote-endpoint = <&xbar_mixer_in9_ep>; + }; + }; + + port@9 { + reg = <0x9>; + + mixer_in10_ep: endpoint { + remote-endpoint = <&xbar_mixer_in10_ep>; + }; + }; + + mixer_out1_port: port@a { + reg = <0xa>; + + mixer_out1_ep: endpoint { + remote-endpoint = <&xbar_mixer_out1_ep>; + }; + }; + + mixer_out2_port: port@b { + reg = <0xb>; + + mixer_out2_ep: endpoint { + remote-endpoint = <&xbar_mixer_out2_ep>; + }; + }; + + mixer_out3_port: port@c { + reg = <0xc>; + + mixer_out3_ep: endpoint { + remote-endpoint = <&xbar_mixer_out3_ep>; + }; + }; + + mixer_out4_port: port@d { + reg = <0xd>; + + mixer_out4_ep: endpoint { + remote-endpoint = <&xbar_mixer_out4_ep>; + }; + }; + + mixer_out5_port: port@e { + reg = <0xe>; + + mixer_out5_ep: endpoint { + remote-endpoint = <&xbar_mixer_out5_ep>; + }; + }; + }; + }; }; }; @@ -764,7 +1971,7 @@ pcie@141a0000 { "p2u-5", "p2u-6", "p2u-7"; }; - pcie_ep@141a0000 { + pcie-ep@141a0000 { status = "disabled"; vddio-pex-ctl-supply = <&vdd_1v8ao>; @@ -827,6 +2034,40 @@ sound { /* XBAR Ports */ <&xbar_i2s1_port>, <&xbar_i2s2_port>, <&xbar_i2s4_port>, <&xbar_i2s6_port>, <&xbar_dmic3_port>, + <&xbar_sfc1_in_port>, <&xbar_sfc2_in_port>, + <&xbar_sfc3_in_port>, <&xbar_sfc4_in_port>, + <&xbar_mvc1_in_port>, <&xbar_mvc2_in_port>, + <&xbar_amx1_in1_port>, <&xbar_amx1_in2_port>, + <&xbar_amx1_in3_port>, <&xbar_amx1_in4_port>, + <&xbar_amx2_in1_port>, <&xbar_amx2_in2_port>, + <&xbar_amx2_in3_port>, <&xbar_amx2_in4_port>, + <&xbar_amx3_in1_port>, <&xbar_amx3_in2_port>, + <&xbar_amx3_in3_port>, <&xbar_amx3_in4_port>, + <&xbar_amx4_in1_port>, <&xbar_amx4_in2_port>, + <&xbar_amx4_in3_port>, <&xbar_amx4_in4_port>, + <&xbar_adx1_in_port>, <&xbar_adx2_in_port>, + <&xbar_adx3_in_port>, <&xbar_adx4_in_port>, + <&xbar_mixer_in1_port>, <&xbar_mixer_in2_port>, + <&xbar_mixer_in3_port>, <&xbar_mixer_in4_port>, + <&xbar_mixer_in5_port>, <&xbar_mixer_in6_port>, + <&xbar_mixer_in7_port>, <&xbar_mixer_in8_port>, + <&xbar_mixer_in9_port>, <&xbar_mixer_in10_port>, + /* HW accelerators */ + <&sfc1_out_port>, <&sfc2_out_port>, + <&sfc3_out_port>, <&sfc4_out_port>, + <&mvc1_out_port>, <&mvc2_out_port>, + <&amx1_out_port>, <&amx2_out_port>, + <&amx3_out_port>, <&amx4_out_port>, + <&adx1_out1_port>, <&adx1_out2_port>, + <&adx1_out3_port>, <&adx1_out4_port>, + <&adx2_out1_port>, <&adx2_out2_port>, + <&adx2_out3_port>, <&adx2_out4_port>, + <&adx3_out1_port>, <&adx3_out2_port>, + <&adx3_out3_port>, <&adx3_out4_port>, + <&adx4_out1_port>, <&adx4_out2_port>, + <&adx4_out3_port>, <&adx4_out4_port>, + <&mixer_out1_port>, <&mixer_out2_port>, <&mixer_out3_port>, + <&mixer_out4_port>, <&mixer_out5_port>, /* BE I/O Ports */ <&i2s1_port>, <&i2s2_port>, <&i2s4_port>, <&i2s6_port>, <&dmic3_port>; @@ -856,7 +2097,7 @@ sound { }; thermal-zones { - cpu { + cpu-thermal { polling-delay = <0>; polling-delay-passive = <500>; status = "okay"; @@ -910,7 +2151,7 @@ cpu-passive { }; }; - gpu { + gpu-thermal { polling-delay = <0>; polling-delay-passive = <500>; status = "okay"; @@ -924,7 +2165,7 @@ gpu_alert0: critical { }; }; - aux { + aux-thermal { polling-delay = <0>; polling-delay-passive = <500>; status = "okay"; diff --git a/arch/arm64/boot/dts/nvidia/tegra194-p3509-0000.dtsi b/arch/arm64/boot/dts/nvidia/tegra194-p3509-0000.dtsi index 836a7e0a42..1323fa9b83 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194-p3509-0000.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194-p3509-0000.dtsi @@ -238,6 +238,542 @@ xbar_dspk2_ep: endpoint { remote-endpoint = <&dspk2_cif_ep>; }; }; + + xbar_sfc1_in_port: port@20 { + reg = <0x20>; + + xbar_sfc1_in_ep: endpoint { + remote-endpoint = <&sfc1_cif_in_ep>; + }; + }; + + port@21 { + reg = <0x21>; + + xbar_sfc1_out_ep: endpoint { + remote-endpoint = <&sfc1_cif_out_ep>; + }; + }; + + xbar_sfc2_in_port: port@22 { + reg = <0x22>; + + xbar_sfc2_in_ep: endpoint { + remote-endpoint = <&sfc2_cif_in_ep>; + }; + }; + + port@23 { + reg = <0x23>; + + xbar_sfc2_out_ep: endpoint { + remote-endpoint = <&sfc2_cif_out_ep>; + }; + }; + + xbar_sfc3_in_port: port@24 { + reg = <0x24>; + + xbar_sfc3_in_ep: endpoint { + remote-endpoint = <&sfc3_cif_in_ep>; + }; + }; + + port@25 { + reg = <0x25>; + + xbar_sfc3_out_ep: endpoint { + remote-endpoint = <&sfc3_cif_out_ep>; + }; + }; + + xbar_sfc4_in_port: port@26 { + reg = <0x26>; + + xbar_sfc4_in_ep: endpoint { + remote-endpoint = <&sfc4_cif_in_ep>; + }; + }; + + port@27 { + reg = <0x27>; + + xbar_sfc4_out_ep: endpoint { + remote-endpoint = <&sfc4_cif_out_ep>; + }; + }; + + xbar_mvc1_in_port: port@28 { + reg = <0x28>; + + xbar_mvc1_in_ep: endpoint { + remote-endpoint = <&mvc1_cif_in_ep>; + }; + }; + + port@29 { + reg = <0x29>; + + xbar_mvc1_out_ep: endpoint { + remote-endpoint = <&mvc1_cif_out_ep>; + }; + }; + + xbar_mvc2_in_port: port@2a { + reg = <0x2a>; + + xbar_mvc2_in_ep: endpoint { + remote-endpoint = <&mvc2_cif_in_ep>; + }; + }; + + port@2b { + reg = <0x2b>; + + xbar_mvc2_out_ep: endpoint { + remote-endpoint = <&mvc2_cif_out_ep>; + }; + }; + + xbar_amx1_in1_port: port@2c { + reg = <0x2c>; + + xbar_amx1_in1_ep: endpoint { + remote-endpoint = <&amx1_in1_ep>; + }; + }; + + xbar_amx1_in2_port: port@2d { + reg = <0x2d>; + + xbar_amx1_in2_ep: endpoint { + remote-endpoint = <&amx1_in2_ep>; + }; + }; + + xbar_amx1_in3_port: port@2e { + reg = <0x2e>; + + xbar_amx1_in3_ep: endpoint { + remote-endpoint = <&amx1_in3_ep>; + }; + }; + + xbar_amx1_in4_port: port@2f { + reg = <0x2f>; + + xbar_amx1_in4_ep: endpoint { + remote-endpoint = <&amx1_in4_ep>; + }; + }; + + port@30 { + reg = <0x30>; + + xbar_amx1_out_ep: endpoint { + remote-endpoint = <&amx1_out_ep>; + }; + }; + + xbar_amx2_in1_port: port@31 { + reg = <0x31>; + + xbar_amx2_in1_ep: endpoint { + remote-endpoint = <&amx2_in1_ep>; + }; + }; + + xbar_amx2_in2_port: port@32 { + reg = <0x32>; + + xbar_amx2_in2_ep: endpoint { + remote-endpoint = <&amx2_in2_ep>; + }; + }; + + xbar_amx2_in3_port: port@33 { + reg = <0x33>; + + xbar_amx2_in3_ep: endpoint { + remote-endpoint = <&amx2_in3_ep>; + }; + }; + + xbar_amx2_in4_port: port@34 { + reg = <0x34>; + + xbar_amx2_in4_ep: endpoint { + remote-endpoint = <&amx2_in4_ep>; + }; + }; + + port@35 { + reg = <0x35>; + + xbar_amx2_out_ep: endpoint { + remote-endpoint = <&amx2_out_ep>; + }; + }; + + xbar_amx3_in1_port: port@36 { + reg = <0x36>; + + xbar_amx3_in1_ep: endpoint { + remote-endpoint = <&amx3_in1_ep>; + }; + }; + + xbar_amx3_in2_port: port@37 { + reg = <0x37>; + + xbar_amx3_in2_ep: endpoint { + remote-endpoint = <&amx3_in2_ep>; + }; + }; + + xbar_amx3_in3_port: port@38 { + reg = <0x38>; + + xbar_amx3_in3_ep: endpoint { + remote-endpoint = <&amx3_in3_ep>; + }; + }; + + xbar_amx3_in4_port: port@39 { + reg = <0x39>; + + xbar_amx3_in4_ep: endpoint { + remote-endpoint = <&amx3_in4_ep>; + }; + }; + + port@3a { + reg = <0x3a>; + + xbar_amx3_out_ep: endpoint { + remote-endpoint = <&amx3_out_ep>; + }; + }; + + xbar_amx4_in1_port: port@3b { + reg = <0x3b>; + + xbar_amx4_in1_ep: endpoint { + remote-endpoint = <&amx4_in1_ep>; + }; + }; + + xbar_amx4_in2_port: port@3c { + reg = <0x3c>; + + xbar_amx4_in2_ep: endpoint { + remote-endpoint = <&amx4_in2_ep>; + }; + }; + + xbar_amx4_in3_port: port@3d { + reg = <0x3d>; + + xbar_amx4_in3_ep: endpoint { + remote-endpoint = <&amx4_in3_ep>; + }; + }; + + xbar_amx4_in4_port: port@3e { + reg = <0x3e>; + + xbar_amx4_in4_ep: endpoint { + remote-endpoint = <&amx4_in4_ep>; + }; + }; + + port@3f { + reg = <0x3f>; + + xbar_amx4_out_ep: endpoint { + remote-endpoint = <&amx4_out_ep>; + }; + }; + + xbar_adx1_in_port: port@40 { + reg = <0x40>; + + xbar_adx1_in_ep: endpoint { + remote-endpoint = <&adx1_in_ep>; + }; + }; + + port@41 { + reg = <0x41>; + + xbar_adx1_out1_ep: endpoint { + remote-endpoint = <&adx1_out1_ep>; + }; + }; + + port@42 { + reg = <0x42>; + + xbar_adx1_out2_ep: endpoint { + remote-endpoint = <&adx1_out2_ep>; + }; + }; + + port@43 { + reg = <0x43>; + + xbar_adx1_out3_ep: endpoint { + remote-endpoint = <&adx1_out3_ep>; + }; + }; + + port@44 { + reg = <0x44>; + + xbar_adx1_out4_ep: endpoint { + remote-endpoint = <&adx1_out4_ep>; + }; + }; + + xbar_adx2_in_port: port@45 { + reg = <0x45>; + + xbar_adx2_in_ep: endpoint { + remote-endpoint = <&adx2_in_ep>; + }; + }; + + port@46 { + reg = <0x46>; + + xbar_adx2_out1_ep: endpoint { + remote-endpoint = <&adx2_out1_ep>; + }; + }; + + port@47 { + reg = <0x47>; + + xbar_adx2_out2_ep: endpoint { + remote-endpoint = <&adx2_out2_ep>; + }; + }; + + port@48 { + reg = <0x48>; + + xbar_adx2_out3_ep: endpoint { + remote-endpoint = <&adx2_out3_ep>; + }; + }; + + port@49 { + reg = <0x49>; + + xbar_adx2_out4_ep: endpoint { + remote-endpoint = <&adx2_out4_ep>; + }; + }; + + xbar_adx3_in_port: port@4a { + reg = <0x4a>; + + xbar_adx3_in_ep: endpoint { + remote-endpoint = <&adx3_in_ep>; + }; + }; + + port@4b { + reg = <0x4b>; + + xbar_adx3_out1_ep: endpoint { + remote-endpoint = <&adx3_out1_ep>; + }; + }; + + port@4c { + reg = <0x4c>; + + xbar_adx3_out2_ep: endpoint { + remote-endpoint = <&adx3_out2_ep>; + }; + }; + + port@4d { + reg = <0x4d>; + + xbar_adx3_out3_ep: endpoint { + remote-endpoint = <&adx3_out3_ep>; + }; + }; + + port@4e { + reg = <0x4e>; + + xbar_adx3_out4_ep: endpoint { + remote-endpoint = <&adx3_out4_ep>; + }; + }; + + xbar_adx4_in_port: port@4f { + reg = <0x4f>; + + xbar_adx4_in_ep: endpoint { + remote-endpoint = <&adx4_in_ep>; + }; + }; + + port@50 { + reg = <0x50>; + + xbar_adx4_out1_ep: endpoint { + remote-endpoint = <&adx4_out1_ep>; + }; + }; + + port@51 { + reg = <0x51>; + + xbar_adx4_out2_ep: endpoint { + remote-endpoint = <&adx4_out2_ep>; + }; + }; + + port@52 { + reg = <0x52>; + + xbar_adx4_out3_ep: endpoint { + remote-endpoint = <&adx4_out3_ep>; + }; + }; + + port@53 { + reg = <0x53>; + + xbar_adx4_out4_ep: endpoint { + remote-endpoint = <&adx4_out4_ep>; + }; + }; + + xbar_mixer_in1_port: port@54 { + reg = <0x54>; + + xbar_mixer_in1_ep: endpoint { + remote-endpoint = <&mixer_in1_ep>; + }; + }; + + xbar_mixer_in2_port: port@55 { + reg = <0x55>; + + xbar_mixer_in2_ep: endpoint { + remote-endpoint = <&mixer_in2_ep>; + }; + }; + + xbar_mixer_in3_port: port@56 { + reg = <0x56>; + + xbar_mixer_in3_ep: endpoint { + remote-endpoint = <&mixer_in3_ep>; + }; + }; + + xbar_mixer_in4_port: port@57 { + reg = <0x57>; + + xbar_mixer_in4_ep: endpoint { + remote-endpoint = <&mixer_in4_ep>; + }; + }; + + xbar_mixer_in5_port: port@58 { + reg = <0x58>; + + xbar_mixer_in5_ep: endpoint { + remote-endpoint = <&mixer_in5_ep>; + }; + }; + + xbar_mixer_in6_port: port@59 { + reg = <0x59>; + + xbar_mixer_in6_ep: endpoint { + remote-endpoint = <&mixer_in6_ep>; + }; + }; + + xbar_mixer_in7_port: port@5a { + reg = <0x5a>; + + xbar_mixer_in7_ep: endpoint { + remote-endpoint = <&mixer_in7_ep>; + }; + }; + + xbar_mixer_in8_port: port@5b { + reg = <0x5b>; + + xbar_mixer_in8_ep: endpoint { + remote-endpoint = <&mixer_in8_ep>; + }; + }; + + xbar_mixer_in9_port: port@5c { + reg = <0x5c>; + + xbar_mixer_in9_ep: endpoint { + remote-endpoint = <&mixer_in9_ep>; + }; + }; + + xbar_mixer_in10_port: port@5d { + reg = <0x5d>; + + xbar_mixer_in10_ep: endpoint { + remote-endpoint = <&mixer_in10_ep>; + }; + }; + + port@5e { + reg = <0x5e>; + + xbar_mixer_out1_ep: endpoint { + remote-endpoint = <&mixer_out1_ep>; + }; + }; + + port@5f { + reg = <0x5f>; + + xbar_mixer_out2_ep: endpoint { + remote-endpoint = <&mixer_out2_ep>; + }; + }; + + port@60 { + reg = <0x60>; + + xbar_mixer_out3_ep: endpoint { + remote-endpoint = <&mixer_out3_ep>; + }; + }; + + port@61 { + reg = <0x61>; + + xbar_mixer_out4_ep: endpoint { + remote-endpoint = <&mixer_out4_ep>; + }; + }; + + port@62 { + reg = <0x62>; + + xbar_mixer_out5_ep: endpoint { + remote-endpoint = <&mixer_out5_ep>; + }; + }; }; admaif@290f000 { @@ -453,7 +989,7 @@ i2s5_cif_ep: endpoint { i2s5_port: port@1 { reg = <1>; - i2s5_dap_ep: endpoint@0 { + i2s5_dap_ep: endpoint { dai-format = "i2s"; /* Place holder for external Codec */ }; @@ -585,6 +1121,679 @@ dspk2_dap_ep: endpoint { }; }; }; + + sfc@2902000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc1_in_ep>; + convert-rate = <44100>; + }; + }; + + sfc1_out_port: port@1 { + reg = <1>; + + sfc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc1_out_ep>; + convert-rate = <48000>; + }; + }; + }; + }; + + sfc@2902200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc2_in_ep>; + }; + }; + + sfc2_out_port: port@1 { + reg = <1>; + + sfc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc2_out_ep>; + }; + }; + }; + }; + + sfc@2902400 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc3_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc3_in_ep>; + }; + }; + + sfc3_out_port: port@1 { + reg = <1>; + + sfc3_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc3_out_ep>; + }; + }; + }; + }; + + sfc@2902600 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc4_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc4_in_ep>; + }; + }; + + sfc4_out_port: port@1 { + reg = <1>; + + sfc4_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc4_out_ep>; + }; + }; + }; + }; + + mvc@290a000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc1_in_ep>; + }; + }; + + mvc1_out_port: port@1 { + reg = <1>; + + mvc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc1_out_ep>; + }; + }; + }; + }; + + mvc@290a200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc2_in_ep>; + }; + }; + + mvc2_out_port: port@1 { + reg = <1>; + + mvc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc2_out_ep>; + }; + }; + }; + }; + + amx@2903000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx1_in1_ep: endpoint { + remote-endpoint = <&xbar_amx1_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx1_in2_ep: endpoint { + remote-endpoint = <&xbar_amx1_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx1_in3_ep: endpoint { + remote-endpoint = <&xbar_amx1_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx1_in4_ep: endpoint { + remote-endpoint = <&xbar_amx1_in4_ep>; + }; + }; + + amx1_out_port: port@4 { + reg = <4>; + + amx1_out_ep: endpoint { + remote-endpoint = <&xbar_amx1_out_ep>; + }; + }; + }; + }; + + amx@2903100 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx2_in1_ep: endpoint { + remote-endpoint = <&xbar_amx2_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx2_in2_ep: endpoint { + remote-endpoint = <&xbar_amx2_in2_ep>; + }; + }; + + amx2_in3_port: port@2 { + reg = <2>; + + amx2_in3_ep: endpoint { + remote-endpoint = <&xbar_amx2_in3_ep>; + }; + }; + + amx2_in4_port: port@3 { + reg = <3>; + + amx2_in4_ep: endpoint { + remote-endpoint = <&xbar_amx2_in4_ep>; + }; + }; + + amx2_out_port: port@4 { + reg = <4>; + + amx2_out_ep: endpoint { + remote-endpoint = <&xbar_amx2_out_ep>; + }; + }; + }; + }; + + amx@2903200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx3_in1_ep: endpoint { + remote-endpoint = <&xbar_amx3_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx3_in2_ep: endpoint { + remote-endpoint = <&xbar_amx3_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx3_in3_ep: endpoint { + remote-endpoint = <&xbar_amx3_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx3_in4_ep: endpoint { + remote-endpoint = <&xbar_amx3_in4_ep>; + }; + }; + + amx3_out_port: port@4 { + reg = <4>; + + amx3_out_ep: endpoint { + remote-endpoint = <&xbar_amx3_out_ep>; + }; + }; + }; + }; + + amx@2903300 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx4_in1_ep: endpoint { + remote-endpoint = <&xbar_amx4_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx4_in2_ep: endpoint { + remote-endpoint = <&xbar_amx4_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx4_in3_ep: endpoint { + remote-endpoint = <&xbar_amx4_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx4_in4_ep: endpoint { + remote-endpoint = <&xbar_amx4_in4_ep>; + }; + }; + + amx4_out_port: port@4 { + reg = <4>; + + amx4_out_ep: endpoint { + remote-endpoint = <&xbar_amx4_out_ep>; + }; + }; + }; + }; + + adx@2903800 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx1_in_ep: endpoint { + remote-endpoint = <&xbar_adx1_in_ep>; + }; + }; + + adx1_out1_port: port@1 { + reg = <1>; + + adx1_out1_ep: endpoint { + remote-endpoint = <&xbar_adx1_out1_ep>; + }; + }; + + adx1_out2_port: port@2 { + reg = <2>; + + adx1_out2_ep: endpoint { + remote-endpoint = <&xbar_adx1_out2_ep>; + }; + }; + + adx1_out3_port: port@3 { + reg = <3>; + + adx1_out3_ep: endpoint { + remote-endpoint = <&xbar_adx1_out3_ep>; + }; + }; + + adx1_out4_port: port@4 { + reg = <4>; + + adx1_out4_ep: endpoint { + remote-endpoint = <&xbar_adx1_out4_ep>; + }; + }; + }; + }; + + adx@2903900 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx2_in_ep: endpoint { + remote-endpoint = <&xbar_adx2_in_ep>; + }; + }; + + adx2_out1_port: port@1 { + reg = <1>; + + adx2_out1_ep: endpoint { + remote-endpoint = <&xbar_adx2_out1_ep>; + }; + }; + + adx2_out2_port: port@2 { + reg = <2>; + + adx2_out2_ep: endpoint { + remote-endpoint = <&xbar_adx2_out2_ep>; + }; + }; + + adx2_out3_port: port@3 { + reg = <3>; + + adx2_out3_ep: endpoint { + remote-endpoint = <&xbar_adx2_out3_ep>; + }; + }; + + adx2_out4_port: port@4 { + reg = <4>; + + adx2_out4_ep: endpoint { + remote-endpoint = <&xbar_adx2_out4_ep>; + }; + }; + }; + }; + + adx@2903a00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx3_in_ep: endpoint { + remote-endpoint = <&xbar_adx3_in_ep>; + }; + }; + + adx3_out1_port: port@1 { + reg = <1>; + + adx3_out1_ep: endpoint { + remote-endpoint = <&xbar_adx3_out1_ep>; + }; + }; + + adx3_out2_port: port@2 { + reg = <2>; + + adx3_out2_ep: endpoint { + remote-endpoint = <&xbar_adx3_out2_ep>; + }; + }; + + adx3_out3_port: port@3 { + reg = <3>; + + adx3_out3_ep: endpoint { + remote-endpoint = <&xbar_adx3_out3_ep>; + }; + }; + + adx3_out4_port: port@4 { + reg = <4>; + + adx3_out4_ep: endpoint { + remote-endpoint = <&xbar_adx3_out4_ep>; + }; + }; + }; + }; + + adx@2903b00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx4_in_ep: endpoint { + remote-endpoint = <&xbar_adx4_in_ep>; + }; + }; + + adx4_out1_port: port@1 { + reg = <1>; + + adx4_out1_ep: endpoint { + remote-endpoint = <&xbar_adx4_out1_ep>; + }; + }; + + adx4_out2_port: port@2 { + reg = <2>; + + adx4_out2_ep: endpoint { + remote-endpoint = <&xbar_adx4_out2_ep>; + }; + }; + + adx4_out3_port: port@3 { + reg = <3>; + + adx4_out3_ep: endpoint { + remote-endpoint = <&xbar_adx4_out3_ep>; + }; + }; + + adx4_out4_port: port@4 { + reg = <4>; + + adx4_out4_ep: endpoint { + remote-endpoint = <&xbar_adx4_out4_ep>; + }; + }; + }; + }; + + amixer@290bb00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0x0>; + + mixer_in1_ep: endpoint { + remote-endpoint = <&xbar_mixer_in1_ep>; + }; + }; + + port@1 { + reg = <0x1>; + + mixer_in2_ep: endpoint { + remote-endpoint = <&xbar_mixer_in2_ep>; + }; + }; + + port@2 { + reg = <0x2>; + + mixer_in3_ep: endpoint { + remote-endpoint = <&xbar_mixer_in3_ep>; + }; + }; + + port@3 { + reg = <0x3>; + + mixer_in4_ep: endpoint { + remote-endpoint = <&xbar_mixer_in4_ep>; + }; + }; + + port@4 { + reg = <0x4>; + + mixer_in5_ep: endpoint { + remote-endpoint = <&xbar_mixer_in5_ep>; + }; + }; + + port@5 { + reg = <0x5>; + + mixer_in6_ep: endpoint { + remote-endpoint = <&xbar_mixer_in6_ep>; + }; + }; + + port@6 { + reg = <0x6>; + + mixer_in7_ep: endpoint { + remote-endpoint = <&xbar_mixer_in7_ep>; + }; + }; + + port@7 { + reg = <0x7>; + + mixer_in8_ep: endpoint { + remote-endpoint = <&xbar_mixer_in8_ep>; + }; + }; + + port@8 { + reg = <0x8>; + + mixer_in9_ep: endpoint { + remote-endpoint = <&xbar_mixer_in9_ep>; + }; + }; + + port@9 { + reg = <0x9>; + + mixer_in10_ep: endpoint { + remote-endpoint = <&xbar_mixer_in10_ep>; + }; + }; + + mixer_out1_port: port@a { + reg = <0xa>; + + mixer_out1_ep: endpoint { + remote-endpoint = <&xbar_mixer_out1_ep>; + }; + }; + + mixer_out2_port: port@b { + reg = <0xb>; + + mixer_out2_ep: endpoint { + remote-endpoint = <&xbar_mixer_out2_ep>; + }; + }; + + mixer_out3_port: port@c { + reg = <0xc>; + + mixer_out3_ep: endpoint { + remote-endpoint = <&xbar_mixer_out3_ep>; + }; + }; + + mixer_out4_port: port@d { + reg = <0xd>; + + mixer_out4_ep: endpoint { + remote-endpoint = <&xbar_mixer_out4_ep>; + }; + }; + + mixer_out5_port: port@e { + reg = <0xe>; + + mixer_out5_ep: endpoint { + remote-endpoint = <&xbar_mixer_out5_ep>; + }; + }; + }; + }; }; }; @@ -669,7 +1878,7 @@ spi@3270000 { status = "okay"; flash@0 { - compatible = "spi-nor"; + compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <102000000>; spi-tx-bus-width = <4>; @@ -741,7 +1950,7 @@ pcie@141a0000 { "p2u-5", "p2u-6", "p2u-7"; }; - pcie_ep@141a0000 { + pcie-ep@141a0000 { status = "disabled"; vddio-pex-ctl-supply = <&vdd_1v8ao>; @@ -791,7 +2000,7 @@ power { }; }; - vdd_5v0_sys: regulator@100 { + vdd_5v0_sys: regulator-vdd-5v0-sys { compatible = "regulator-fixed"; regulator-name = "VDD_5V_SYS"; regulator-min-microvolt = <5000000>; @@ -800,7 +2009,7 @@ vdd_5v0_sys: regulator@100 { regulator-boot-on; }; - vdd_3v3_sys: regulator@101 { + vdd_3v3_sys: regulator-vdd-3v3-sys { compatible = "regulator-fixed"; regulator-name = "VDD_3V3_SYS"; regulator-min-microvolt = <3300000>; @@ -809,7 +2018,7 @@ vdd_3v3_sys: regulator@101 { regulator-boot-on; }; - vdd_3v3_ao: regulator@102 { + vdd_3v3_ao: regulator-vdd-3v3-ao { compatible = "regulator-fixed"; regulator-name = "VDD_3V3_AO"; regulator-min-microvolt = <3300000>; @@ -818,7 +2027,7 @@ vdd_3v3_ao: regulator@102 { regulator-boot-on; }; - vdd_1v8: regulator@103 { + vdd_1v8: regulator-vdd-1v8 { compatible = "regulator-fixed"; regulator-name = "VDD_1V8"; regulator-min-microvolt = <1800000>; @@ -827,7 +2036,7 @@ vdd_1v8: regulator@103 { regulator-boot-on; }; - vdd_hdmi: regulator@104 { + vdd_hdmi: regulator-vdd-hdmi { compatible = "regulator-fixed"; regulator-name = "VDD_5V0_HDMI_CON"; regulator-min-microvolt = <5000000>; @@ -850,6 +2059,41 @@ sound { <&xbar_i2s3_port>, <&xbar_i2s5_port>, <&xbar_dmic1_port>, <&xbar_dmic2_port>, <&xbar_dmic4_port>, <&xbar_dspk1_port>, <&xbar_dspk2_port>, + <&xbar_sfc1_in_port>, <&xbar_sfc2_in_port>, + <&xbar_sfc3_in_port>, <&xbar_sfc4_in_port>, + <&xbar_mvc1_in_port>, <&xbar_mvc2_in_port>, + <&xbar_amx1_in1_port>, <&xbar_amx1_in2_port>, + <&xbar_amx1_in3_port>, <&xbar_amx1_in4_port>, + <&xbar_amx2_in1_port>, <&xbar_amx2_in2_port>, + <&xbar_amx2_in3_port>, <&xbar_amx2_in4_port>, + <&xbar_amx3_in1_port>, <&xbar_amx3_in2_port>, + <&xbar_amx3_in3_port>, <&xbar_amx3_in4_port>, + <&xbar_amx4_in1_port>, <&xbar_amx4_in2_port>, + <&xbar_amx4_in3_port>, <&xbar_amx4_in4_port>, + <&xbar_adx1_in_port>, <&xbar_adx2_in_port>, + <&xbar_adx3_in_port>, <&xbar_adx4_in_port>, + <&xbar_mixer_in1_port>, <&xbar_mixer_in2_port>, + <&xbar_mixer_in3_port>, <&xbar_mixer_in4_port>, + <&xbar_mixer_in5_port>, <&xbar_mixer_in6_port>, + <&xbar_mixer_in7_port>, <&xbar_mixer_in8_port>, + <&xbar_mixer_in9_port>, <&xbar_mixer_in10_port>, + /* HW accelerators */ + <&sfc1_out_port>, <&sfc2_out_port>, + <&sfc3_out_port>, <&sfc4_out_port>, + <&mvc1_out_port>, <&mvc2_out_port>, + <&amx1_out_port>, <&amx2_out_port>, + <&amx3_out_port>, <&amx4_out_port>, + <&adx1_out1_port>, <&adx1_out2_port>, + <&adx1_out3_port>, <&adx1_out4_port>, + <&adx2_out1_port>, <&adx2_out2_port>, + <&adx2_out3_port>, <&adx2_out4_port>, + <&adx3_out1_port>, <&adx3_out2_port>, + <&adx3_out3_port>, <&adx3_out4_port>, + <&adx4_out1_port>, <&adx4_out2_port>, + <&adx4_out3_port>, <&adx4_out4_port>, + <&mixer_out1_port>, <&mixer_out2_port>, + <&mixer_out3_port>, <&mixer_out4_port>, + <&mixer_out5_port>, /* BE I/O Ports */ <&i2s3_port>, <&i2s5_port>, <&dmic1_port>, <&dmic2_port>, <&dmic4_port>, @@ -859,7 +2103,7 @@ sound { }; thermal-zones { - cpu { + cpu-thermal { polling-delay = <0>; polling-delay-passive = <500>; status = "okay"; @@ -913,7 +2157,7 @@ cpu-passive { }; }; - gpu { + gpu-thermal { polling-delay = <0>; polling-delay-passive = <500>; status = "okay"; @@ -927,7 +2171,7 @@ gpu_alert0: critical { }; }; - aux { + aux-thermal { polling-delay = <0>; polling-delay-passive = <500>; status = "okay"; diff --git a/arch/arm64/boot/dts/nvidia/tegra194-p3668-0000.dtsi b/arch/arm64/boot/dts/nvidia/tegra194-p3668-0000.dtsi index 14da4206ea..8c2c709dd5 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194-p3668-0000.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194-p3668-0000.dtsi @@ -20,7 +20,7 @@ mmc@3400000 { }; }; - vdd_3v3_sd: regulator@0 { + vdd_3v3_sd: regulator-vdd-3v3-sd { compatible = "regulator-fixed"; regulator-name = "VDD_3V3_SD"; regulator-min-microvolt = <3300000>; diff --git a/arch/arm64/boot/dts/nvidia/tegra194-p3668.dtsi b/arch/arm64/boot/dts/nvidia/tegra194-p3668.dtsi index f16b0aa8a3..0bd66f9c62 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194-p3668.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194-p3668.dtsi @@ -36,7 +36,7 @@ mdio { #address-cells = <1>; #size-cells = <0>; - phy: phy@0 { + phy: ethernet-phy@0 { compatible = "ethernet-phy-ieee802.3-c22"; reg = <0x0>; interrupt-parent = <&gpio>; diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi index 510d297447..aaa00da535 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -34,11 +35,53 @@ gpio: gpio@2200000 { reg = <0x2200000 0x10000>, <0x2210000 0x10000>; interrupts = , + , + , + , + , + , + , + , , + , + , + , + , + , + , + , , + , + , + , + , + , + , + , , + , + , + , + , + , + , + , , - ; + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; #interrupt-cells = <2>; interrupt-controller; #gpio-cells = <2>; @@ -363,6 +406,122 @@ tegra_dspk2: dspk@2905100 { sound-name-prefix = "DSPK2"; status = "disabled"; }; + + tegra_sfc1: sfc@2902000 { + compatible = "nvidia,tegra194-sfc", + "nvidia,tegra210-sfc"; + reg = <0x2902000 0x200>; + sound-name-prefix = "SFC1"; + status = "disabled"; + }; + + tegra_sfc2: sfc@2902200 { + compatible = "nvidia,tegra194-sfc", + "nvidia,tegra210-sfc"; + reg = <0x2902200 0x200>; + sound-name-prefix = "SFC2"; + status = "disabled"; + }; + + tegra_sfc3: sfc@2902400 { + compatible = "nvidia,tegra194-sfc", + "nvidia,tegra210-sfc"; + reg = <0x2902400 0x200>; + sound-name-prefix = "SFC3"; + status = "disabled"; + }; + + tegra_sfc4: sfc@2902600 { + compatible = "nvidia,tegra194-sfc", + "nvidia,tegra210-sfc"; + reg = <0x2902600 0x200>; + sound-name-prefix = "SFC4"; + status = "disabled"; + }; + + tegra_mvc1: mvc@290a000 { + compatible = "nvidia,tegra194-mvc", + "nvidia,tegra210-mvc"; + reg = <0x290a000 0x200>; + sound-name-prefix = "MVC1"; + status = "disabled"; + }; + + tegra_mvc2: mvc@290a200 { + compatible = "nvidia,tegra194-mvc", + "nvidia,tegra210-mvc"; + reg = <0x290a200 0x200>; + sound-name-prefix = "MVC2"; + status = "disabled"; + }; + + tegra_amx1: amx@2903000 { + compatible = "nvidia,tegra194-amx"; + reg = <0x2903000 0x100>; + sound-name-prefix = "AMX1"; + status = "disabled"; + }; + + tegra_amx2: amx@2903100 { + compatible = "nvidia,tegra194-amx"; + reg = <0x2903100 0x100>; + sound-name-prefix = "AMX2"; + status = "disabled"; + }; + + tegra_amx3: amx@2903200 { + compatible = "nvidia,tegra194-amx"; + reg = <0x2903200 0x100>; + sound-name-prefix = "AMX3"; + status = "disabled"; + }; + + tegra_amx4: amx@2903300 { + compatible = "nvidia,tegra194-amx"; + reg = <0x2903300 0x100>; + sound-name-prefix = "AMX4"; + status = "disabled"; + }; + + tegra_adx1: adx@2903800 { + compatible = "nvidia,tegra194-adx", + "nvidia,tegra210-adx"; + reg = <0x2903800 0x100>; + sound-name-prefix = "ADX1"; + status = "disabled"; + }; + + tegra_adx2: adx@2903900 { + compatible = "nvidia,tegra194-adx", + "nvidia,tegra210-adx"; + reg = <0x2903900 0x100>; + sound-name-prefix = "ADX2"; + status = "disabled"; + }; + + tegra_adx3: adx@2903a00 { + compatible = "nvidia,tegra194-adx", + "nvidia,tegra210-adx"; + reg = <0x2903a00 0x100>; + sound-name-prefix = "ADX3"; + status = "disabled"; + }; + + tegra_adx4: adx@2903b00 { + compatible = "nvidia,tegra194-adx", + "nvidia,tegra210-adx"; + reg = <0x2903b00 0x100>; + sound-name-prefix = "ADX4"; + status = "disabled"; + }; + + tegra_amixer: amixer@290bb00 { + compatible = "nvidia,tegra194-amixer", + "nvidia,tegra210-amixer"; + reg = <0x290bb00 0x800>; + sound-name-prefix = "MIXER1"; + status = "disabled"; + }; }; }; @@ -377,7 +536,6 @@ pex_rst_c5_out_state: pex_rst_c5_out { pex_rst { nvidia,pins = "pex_l5_rst_n_pgg1"; nvidia,schmitt = ; - nvidia,lpdr = ; nvidia,enable-input = ; nvidia,io-hv = ; nvidia,tristate = ; @@ -389,7 +547,6 @@ clkreq_c5_bi_dir_state: clkreq_c5_bi_dir { clkreq { nvidia,pins = "pex_l5_clkreq_n_pgg0"; nvidia,schmitt = ; - nvidia,lpdr = ; nvidia,enable-input = ; nvidia,io-hv = ; nvidia,tristate = ; @@ -435,6 +592,7 @@ emc: external-memory-controller@2c60000 { compatible = "nvidia,tegra194-emc"; reg = <0x0 0x02c60000 0x0 0x90000>, <0x0 0x01780000 0x0 0x80000>; + interrupts = ; clocks = <&bpmp TEGRA194_CLK_EMC>; clock-names = "emc"; @@ -735,6 +893,9 @@ sdmmc1: mmc@3400000 { <&mc TEGRA194_MEMORY_CLIENT_SDMMCWA &emc>; interconnect-names = "dma-mem", "write"; iommus = <&smmu TEGRA194_SID_SDMMC1>; + pinctrl-names = "sdmmc-3v3", "sdmmc-1v8"; + pinctrl-0 = <&sdmmc1_3v3>; + pinctrl-1 = <&sdmmc1_1v8>; nvidia,pad-autocal-pull-up-offset-3v3-timeout = <0x07>; nvidia,pad-autocal-pull-down-offset-3v3-timeout = @@ -746,6 +907,10 @@ sdmmc1: mmc@3400000 { nvidia,pad-autocal-pull-down-offset-sdr104 = <0x00>; nvidia,default-tap = <0x9>; nvidia,default-trim = <0x5>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-ddr50; + sd-uhs-sdr104; status = "disabled"; }; @@ -762,6 +927,9 @@ sdmmc3: mmc@3440000 { <&mc TEGRA194_MEMORY_CLIENT_SDMMCW &emc>; interconnect-names = "dma-mem", "write"; iommus = <&smmu TEGRA194_SID_SDMMC3>; + pinctrl-names = "sdmmc-3v3", "sdmmc-1v8"; + pinctrl-0 = <&sdmmc3_3v3>; + pinctrl-1 = <&sdmmc3_1v8>; nvidia,pad-autocal-pull-up-offset-1v8 = <0x00>; nvidia,pad-autocal-pull-down-offset-1v8 = <0x7a>; nvidia,pad-autocal-pull-up-offset-3v3-timeout = <0x07>; @@ -774,6 +942,10 @@ sdmmc3: mmc@3440000 { nvidia,pad-autocal-pull-down-offset-sdr104 = <0x00>; nvidia,default-tap = <0x9>; nvidia,default-trim = <0x5>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-ddr50; + sd-uhs-sdr104; status = "disabled"; }; @@ -805,6 +977,11 @@ sdmmc4: mmc@3460000 { nvidia,default-tap = <0x8>; nvidia,default-trim = <0x14>; nvidia,dqs-trim = <40>; + cap-mmc-highspeed; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; supports-cqe; status = "disabled"; }; @@ -1023,7 +1200,7 @@ cec@3960000 { }; hsp_top0: hsp@3c00000 { - compatible = "nvidia,tegra194-hsp", "nvidia,tegra186-hsp"; + compatible = "nvidia,tegra194-hsp"; reg = <0x03c00000 0xa0000>; interrupts = , , @@ -1201,7 +1378,7 @@ p2u_hsio_11: phy@3f40000 { }; hsp_aon: hsp@c150000 { - compatible = "nvidia,tegra194-hsp", "nvidia,tegra186-hsp"; + compatible = "nvidia,tegra194-hsp"; reg = <0x0c150000 0x90000>; interrupts = , , @@ -1280,7 +1457,10 @@ gpio_aon: gpio@c2f0000 { reg-names = "security", "gpio"; reg = <0xc2f0000 0x1000>, <0xc2f1000 0x1000>; - interrupts = ; + interrupts = , + , + , + ; gpio-controller; #gpio-cells = <2>; interrupt-controller; @@ -1310,6 +1490,101 @@ pmc: pmc@c360000 { #interrupt-cells = <2>; interrupt-controller; + sdmmc1_3v3: sdmmc1-3v3 { + pins = "sdmmc1-hv"; + power-source = ; + }; + + sdmmc1_1v8: sdmmc1-1v8 { + pins = "sdmmc1-hv"; + power-source = ; + }; + sdmmc3_3v3: sdmmc3-3v3 { + pins = "sdmmc3-hv"; + power-source = ; + }; + + sdmmc3_1v8: sdmmc3-1v8 { + pins = "sdmmc3-hv"; + power-source = ; + }; + + }; + + iommu@10000000 { + compatible = "nvidia,tegra194-smmu", "nvidia,smmu-500"; + reg = <0x10000000 0x800000>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + stream-match-mask = <0x7f80>; + #global-interrupts = <1>; + #iommu-cells = <1>; + + nvidia,memory-controller = <&mc>; + status = "disabled"; }; smmu: iommu@12000000 { @@ -1411,6 +1686,25 @@ host1x@13e00000 { interconnect-names = "dma-mem"; iommus = <&smmu TEGRA194_SID_HOST1X>; + nvdec@15140000 { + compatible = "nvidia,tegra194-nvdec"; + reg = <0x15140000 0x00040000>; + clocks = <&bpmp TEGRA194_CLK_NVDEC1>; + clock-names = "nvdec"; + resets = <&bpmp TEGRA194_RESET_NVDEC1>; + reset-names = "nvdec"; + + power-domains = <&bpmp TEGRA194_POWER_DOMAIN_NVDECB>; + interconnects = <&mc TEGRA194_MEMORY_CLIENT_NVDEC1SRD &emc>, + <&mc TEGRA194_MEMORY_CLIENT_NVDEC1SRD1 &emc>, + <&mc TEGRA194_MEMORY_CLIENT_NVDEC1SWR &emc>; + interconnect-names = "dma-mem", "read-1", "write"; + iommus = <&smmu TEGRA194_SID_NVDEC1>; + dma-coherent; + + nvidia,host1x-class = <0xf5>; + }; + display-hub@15200000 { compatible = "nvidia,tegra194-display"; reg = <0x15200000 0x00040000>; @@ -1522,6 +1816,61 @@ vic@15340000 { <&mc TEGRA194_MEMORY_CLIENT_VICSWR &emc>; interconnect-names = "dma-mem", "write"; iommus = <&smmu TEGRA194_SID_VIC>; + dma-coherent; + }; + + nvjpg@15380000 { + compatible = "nvidia,tegra194-nvjpg"; + reg = <0x15380000 0x40000>; + clocks = <&bpmp TEGRA194_CLK_NVJPG>; + clock-names = "nvjpg"; + resets = <&bpmp TEGRA194_RESET_NVJPG>; + reset-names = "nvjpg"; + + power-domains = <&bpmp TEGRA194_POWER_DOMAIN_NVJPG>; + interconnects = <&mc TEGRA194_MEMORY_CLIENT_NVJPGSRD &emc>, + <&mc TEGRA194_MEMORY_CLIENT_NVJPGSWR &emc>; + interconnect-names = "dma-mem", "write"; + iommus = <&smmu TEGRA194_SID_NVJPG>; + dma-coherent; + }; + + nvdec@15480000 { + compatible = "nvidia,tegra194-nvdec"; + reg = <0x15480000 0x00040000>; + clocks = <&bpmp TEGRA194_CLK_NVDEC>; + clock-names = "nvdec"; + resets = <&bpmp TEGRA194_RESET_NVDEC>; + reset-names = "nvdec"; + + power-domains = <&bpmp TEGRA194_POWER_DOMAIN_NVDECA>; + interconnects = <&mc TEGRA194_MEMORY_CLIENT_NVDECSRD &emc>, + <&mc TEGRA194_MEMORY_CLIENT_NVDECSRD1 &emc>, + <&mc TEGRA194_MEMORY_CLIENT_NVDECSWR &emc>; + interconnect-names = "dma-mem", "read-1", "write"; + iommus = <&smmu TEGRA194_SID_NVDEC>; + dma-coherent; + + nvidia,host1x-class = <0xf0>; + }; + + nvenc@154c0000 { + compatible = "nvidia,tegra194-nvenc"; + reg = <0x154c0000 0x40000>; + clocks = <&bpmp TEGRA194_CLK_NVENC>; + clock-names = "nvenc"; + resets = <&bpmp TEGRA194_RESET_NVENC>; + reset-names = "nvenc"; + + power-domains = <&bpmp TEGRA194_POWER_DOMAIN_NVENCA>; + interconnects = <&mc TEGRA194_MEMORY_CLIENT_NVENCSRD &emc>, + <&mc TEGRA194_MEMORY_CLIENT_NVENCSRD1 &emc>, + <&mc TEGRA194_MEMORY_CLIENT_NVENCSWR &emc>; + interconnect-names = "dma-mem", "read-1", "write"; + iommus = <&smmu TEGRA194_SID_NVENC>; + dma-coherent; + + nvidia,host1x-class = <0x21>; }; dpaux0: dpaux@155c0000 { @@ -1660,6 +2009,25 @@ i2c-bus { }; }; + nvenc@15a80000 { + compatible = "nvidia,tegra194-nvenc"; + reg = <0x15a80000 0x00040000>; + clocks = <&bpmp TEGRA194_CLK_NVENC1>; + clock-names = "nvenc"; + resets = <&bpmp TEGRA194_RESET_NVENC1>; + reset-names = "nvenc"; + + power-domains = <&bpmp TEGRA194_POWER_DOMAIN_NVENCB>; + interconnects = <&mc TEGRA194_MEMORY_CLIENT_NVENC1SRD &emc>, + <&mc TEGRA194_MEMORY_CLIENT_NVENC1SRD1 &emc>, + <&mc TEGRA194_MEMORY_CLIENT_NVENC1SWR &emc>; + interconnect-names = "dma-mem", "read-1", "write"; + iommus = <&smmu TEGRA194_SID_NVENC1>; + dma-coherent; + + nvidia,host1x-class = <0x22>; + }; + sor0: sor@15b00000 { compatible = "nvidia,tegra194-sor"; reg = <0x15b00000 0x40000>; @@ -1807,7 +2175,6 @@ pcie@14100000 { #size-cells = <2>; device_type = "pci"; num-lanes = <1>; - num-viewport = <8>; linux,pci-domain = <1>; clocks = <&bpmp TEGRA194_CLK_PEX0_CORE_1>; @@ -1840,7 +2207,6 @@ pcie@14100000 { interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE1R &emc>, <&mc TEGRA194_MEMORY_CLIENT_PCIE1W &emc>; interconnect-names = "dma-mem", "write"; - iommus = <&smmu TEGRA194_SID_PCIE1>; iommu-map = <0x0 &smmu TEGRA194_SID_PCIE1 0x1000>; iommu-map-mask = <0x0>; dma-coherent; @@ -1861,7 +2227,6 @@ pcie@14120000 { #size-cells = <2>; device_type = "pci"; num-lanes = <1>; - num-viewport = <8>; linux,pci-domain = <2>; clocks = <&bpmp TEGRA194_CLK_PEX0_CORE_2>; @@ -1894,7 +2259,6 @@ pcie@14120000 { interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE2AR &emc>, <&mc TEGRA194_MEMORY_CLIENT_PCIE2AW &emc>; interconnect-names = "dma-mem", "write"; - iommus = <&smmu TEGRA194_SID_PCIE2>; iommu-map = <0x0 &smmu TEGRA194_SID_PCIE2 0x1000>; iommu-map-mask = <0x0>; dma-coherent; @@ -1915,7 +2279,6 @@ pcie@14140000 { #size-cells = <2>; device_type = "pci"; num-lanes = <1>; - num-viewport = <8>; linux,pci-domain = <3>; clocks = <&bpmp TEGRA194_CLK_PEX0_CORE_3>; @@ -1948,7 +2311,6 @@ pcie@14140000 { interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE3R &emc>, <&mc TEGRA194_MEMORY_CLIENT_PCIE3W &emc>; interconnect-names = "dma-mem", "write"; - iommus = <&smmu TEGRA194_SID_PCIE3>; iommu-map = <0x0 &smmu TEGRA194_SID_PCIE3 0x1000>; iommu-map-mask = <0x0>; dma-coherent; @@ -1969,7 +2331,6 @@ pcie@14160000 { #size-cells = <2>; device_type = "pci"; num-lanes = <4>; - num-viewport = <8>; linux,pci-domain = <4>; clocks = <&bpmp TEGRA194_CLK_PEX0_CORE_4>; @@ -2002,7 +2363,6 @@ pcie@14160000 { interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE4R &emc>, <&mc TEGRA194_MEMORY_CLIENT_PCIE4W &emc>; interconnect-names = "dma-mem", "write"; - iommus = <&smmu TEGRA194_SID_PCIE4>; iommu-map = <0x0 &smmu TEGRA194_SID_PCIE4 0x1000>; iommu-map-mask = <0x0>; dma-coherent; @@ -2023,7 +2383,6 @@ pcie@14180000 { #size-cells = <2>; device_type = "pci"; num-lanes = <8>; - num-viewport = <8>; linux,pci-domain = <0>; clocks = <&bpmp TEGRA194_CLK_PEX0_CORE_0>; @@ -2056,7 +2415,6 @@ pcie@14180000 { interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE0R &emc>, <&mc TEGRA194_MEMORY_CLIENT_PCIE0W &emc>; interconnect-names = "dma-mem", "write"; - iommus = <&smmu TEGRA194_SID_PCIE0>; iommu-map = <0x0 &smmu TEGRA194_SID_PCIE0 0x1000>; iommu-map-mask = <0x0>; dma-coherent; @@ -2077,15 +2435,13 @@ pcie@141a0000 { #size-cells = <2>; device_type = "pci"; num-lanes = <8>; - num-viewport = <8>; linux,pci-domain = <5>; pinctrl-names = "default"; pinctrl-0 = <&pex_rst_c5_out_state>, <&clkreq_c5_bi_dir_state>; - clocks = <&bpmp TEGRA194_CLK_PEX1_CORE_5>, - <&bpmp TEGRA194_CLK_PEX1_CORE_5M>; - clock-names = "core", "core_m"; + clocks = <&bpmp TEGRA194_CLK_PEX1_CORE_5>; + clock-names = "core"; resets = <&bpmp TEGRA194_RESET_PEX1_CORE_5_APB>, <&bpmp TEGRA194_RESET_PEX1_CORE_5>; @@ -2114,13 +2470,12 @@ pcie@141a0000 { interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE5R &emc>, <&mc TEGRA194_MEMORY_CLIENT_PCIE5W &emc>; interconnect-names = "dma-mem", "write"; - iommus = <&smmu TEGRA194_SID_PCIE5>; iommu-map = <0x0 &smmu TEGRA194_SID_PCIE5 0x1000>; iommu-map-mask = <0x0>; dma-coherent; }; - pcie_ep@14160000 { + pcie-ep@14160000 { compatible = "nvidia,tegra194-pcie-ep"; power-domains = <&bpmp TEGRA194_POWER_DOMAIN_PCIEX4A>; reg = <0x00 0x14160000 0x0 0x00020000>, /* appl registers (128K) */ @@ -2154,13 +2509,12 @@ pcie_ep@14160000 { interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE4R &emc>, <&mc TEGRA194_MEMORY_CLIENT_PCIE4W &emc>; interconnect-names = "dma-mem", "write"; - iommus = <&smmu TEGRA194_SID_PCIE4>; iommu-map = <0x0 &smmu TEGRA194_SID_PCIE4 0x1000>; iommu-map-mask = <0x0>; dma-coherent; }; - pcie_ep@14180000 { + pcie-ep@14180000 { compatible = "nvidia,tegra194-pcie-ep"; power-domains = <&bpmp TEGRA194_POWER_DOMAIN_PCIEX8B>; reg = <0x00 0x14180000 0x0 0x00020000>, /* appl registers (128K) */ @@ -2194,13 +2548,12 @@ pcie_ep@14180000 { interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE0R &emc>, <&mc TEGRA194_MEMORY_CLIENT_PCIE0W &emc>; interconnect-names = "dma-mem", "write"; - iommus = <&smmu TEGRA194_SID_PCIE0>; iommu-map = <0x0 &smmu TEGRA194_SID_PCIE0 0x1000>; iommu-map-mask = <0x0>; dma-coherent; }; - pcie_ep@141a0000 { + pcie-ep@141a0000 { compatible = "nvidia,tegra194-pcie-ep"; power-domains = <&bpmp TEGRA194_POWER_DOMAIN_PCIEX8A>; reg = <0x00 0x141a0000 0x0 0x00020000>, /* appl registers (128K) */ @@ -2237,7 +2590,6 @@ pcie_ep@141a0000 { interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE5R &emc>, <&mc TEGRA194_MEMORY_CLIENT_PCIE5W &emc>; interconnect-names = "dma-mem", "write"; - iommus = <&smmu TEGRA194_SID_PCIE5>; iommu-map = <0x0 &smmu TEGRA194_SID_PCIE5 0x1000>; iommu-map-mask = <0x0>; dma-coherent; @@ -2267,7 +2619,7 @@ bpmp: bpmp { compatible = "nvidia,tegra186-bpmp"; mboxes = <&hsp_top0 TEGRA_HSP_MBOX_TYPE_DB TEGRA_HSP_DB_MASTER_BPMP>; - shmem = <&cpu_bpmp_tx &cpu_bpmp_rx>; + shmem = <&cpu_bpmp_tx>, <&cpu_bpmp_rx>; #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; @@ -2531,7 +2883,7 @@ sound { iommus = <&smmu TEGRA194_SID_APE>; }; - tcu: tcu { + tcu: serial { compatible = "nvidia,tegra194-tcu"; mboxes = <&hsp_top0 TEGRA_HSP_MBOX_TYPE_SM TEGRA_HSP_SM_RX(0)>, <&hsp_aon TEGRA_HSP_MBOX_TYPE_SM TEGRA_HSP_SM_TX(1)>; @@ -2539,39 +2891,33 @@ tcu: tcu { }; thermal-zones { - cpu { - thermal-sensors = <&{/bpmp/thermal} - TEGRA194_BPMP_THERMAL_ZONE_CPU>; + cpu-thermal { + thermal-sensors = <&{/bpmp/thermal} TEGRA194_BPMP_THERMAL_ZONE_CPU>; status = "disabled"; }; - gpu { - thermal-sensors = <&{/bpmp/thermal} - TEGRA194_BPMP_THERMAL_ZONE_GPU>; + gpu-thermal { + thermal-sensors = <&{/bpmp/thermal} TEGRA194_BPMP_THERMAL_ZONE_GPU>; status = "disabled"; }; - aux { - thermal-sensors = <&{/bpmp/thermal} - TEGRA194_BPMP_THERMAL_ZONE_AUX>; + aux-thermal { + thermal-sensors = <&{/bpmp/thermal} TEGRA194_BPMP_THERMAL_ZONE_AUX>; status = "disabled"; }; - pllx { - thermal-sensors = <&{/bpmp/thermal} - TEGRA194_BPMP_THERMAL_ZONE_PLLX>; + pllx-thermal { + thermal-sensors = <&{/bpmp/thermal} TEGRA194_BPMP_THERMAL_ZONE_PLLX>; status = "disabled"; }; - ao { - thermal-sensors = <&{/bpmp/thermal} - TEGRA194_BPMP_THERMAL_ZONE_AO>; + ao-thermal { + thermal-sensors = <&{/bpmp/thermal} TEGRA194_BPMP_THERMAL_ZONE_AO>; status = "disabled"; }; - tj { - thermal-sensors = <&{/bpmp/thermal} - TEGRA194_BPMP_THERMAL_ZONE_TJ_MAX>; + tj-thermal { + thermal-sensors = <&{/bpmp/thermal} TEGRA194_BPMP_THERMAL_ZONE_TJ_MAX>; status = "disabled"; }; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi b/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi index 6077d572d8..75eb743a72 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi @@ -301,7 +301,7 @@ mmc@700b0600 { vqmmc-supply = <&vdd_1v8>; }; - clk32k_in: clock@0 { + clk32k_in: clock-32k { compatible = "fixed-clock"; clock-frequency = <32768>; #clock-cells = <0>; @@ -336,7 +336,7 @@ psci { method = "smc"; }; - vdd_gpu: regulator@100 { + vdd_gpu: regulator-vdd-gpu { compatible = "pwm-regulator"; pwms = <&pwm 1 8000>; regulator-name = "VDD_GPU"; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts index 7d3e363474..328fbfec4e 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts @@ -11,11 +11,8 @@ / { pcie@1003000 { status = "okay"; - avdd-pll-uerefe-supply = <&avdd_1v05_pll>; hvddio-pex-supply = <&vdd_1v8>; dvddio-pex-supply = <&vdd_pex_1v05>; - dvdd-pex-pll-supply = <&vdd_pex_1v05>; - hvdd-pex-pll-e-supply = <&vdd_1v8>; vddio-pex-ctl-supply = <&vdd_1v8>; pci@1,0 { @@ -339,6 +336,481 @@ dmic3_dap_ep: endpoint { }; }; + sfc@702d2000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc1_in_ep>; + }; + }; + + sfc1_out_port: port@1 { + reg = <1>; + + sfc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc1_out_ep>; + }; + }; + }; + }; + + sfc@702d2200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc2_in_ep>; + }; + }; + + sfc2_out_port: port@1 { + reg = <1>; + + sfc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc2_out_ep>; + }; + }; + }; + }; + + sfc@702d2400 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc3_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc3_in_ep>; + }; + }; + + sfc3_out_port: port@1 { + reg = <1>; + + sfc3_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc3_out_ep>; + }; + }; + }; + }; + + sfc@702d2600 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc4_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc4_in_ep>; + }; + }; + + sfc4_out_port: port@1 { + reg = <1>; + + sfc4_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc4_out_ep>; + }; + }; + }; + }; + + mvc@702da000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc1_in_ep>; + }; + }; + + mvc1_out_port: port@1 { + reg = <1>; + + mvc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc1_out_ep>; + }; + }; + }; + }; + + mvc@702da200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc2_in_ep>; + }; + }; + + mvc2_out_port: port@1 { + reg = <1>; + + mvc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc2_out_ep>; + }; + }; + }; + }; + + amx@702d3000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx1_in1_ep: endpoint { + remote-endpoint = <&xbar_amx1_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx1_in2_ep: endpoint { + remote-endpoint = <&xbar_amx1_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx1_in3_ep: endpoint { + remote-endpoint = <&xbar_amx1_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx1_in4_ep: endpoint { + remote-endpoint = <&xbar_amx1_in4_ep>; + }; + }; + + amx1_out_port: port@4 { + reg = <4>; + + amx1_out_ep: endpoint { + remote-endpoint = <&xbar_amx1_out_ep>; + }; + }; + }; + }; + + amx@702d3100 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx2_in1_ep: endpoint { + remote-endpoint = <&xbar_amx2_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx2_in2_ep: endpoint { + remote-endpoint = <&xbar_amx2_in2_ep>; + }; + }; + + amx2_in3_port: port@2 { + reg = <2>; + + amx2_in3_ep: endpoint { + remote-endpoint = <&xbar_amx2_in3_ep>; + }; + }; + + amx2_in4_port: port@3 { + reg = <3>; + + amx2_in4_ep: endpoint { + remote-endpoint = <&xbar_amx2_in4_ep>; + }; + }; + + amx2_out_port: port@4 { + reg = <4>; + + amx2_out_ep: endpoint { + remote-endpoint = <&xbar_amx2_out_ep>; + }; + }; + }; + }; + + adx@702d3800 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx1_in_ep: endpoint { + remote-endpoint = <&xbar_adx1_in_ep>; + }; + }; + + adx1_out1_port: port@1 { + reg = <1>; + + adx1_out1_ep: endpoint { + remote-endpoint = <&xbar_adx1_out1_ep>; + }; + }; + + adx1_out2_port: port@2 { + reg = <2>; + + adx1_out2_ep: endpoint { + remote-endpoint = <&xbar_adx1_out2_ep>; + }; + }; + + adx1_out3_port: port@3 { + reg = <3>; + + adx1_out3_ep: endpoint { + remote-endpoint = <&xbar_adx1_out3_ep>; + }; + }; + + adx1_out4_port: port@4 { + reg = <4>; + + adx1_out4_ep: endpoint { + remote-endpoint = <&xbar_adx1_out4_ep>; + }; + }; + }; + }; + + adx@702d3900 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx2_in_ep: endpoint { + remote-endpoint = <&xbar_adx2_in_ep>; + }; + }; + + adx2_out1_port: port@1 { + reg = <1>; + + adx2_out1_ep: endpoint { + remote-endpoint = <&xbar_adx2_out1_ep>; + }; + }; + + adx2_out2_port: port@2 { + reg = <2>; + + adx2_out2_ep: endpoint { + remote-endpoint = <&xbar_adx2_out2_ep>; + }; + }; + + adx2_out3_port: port@3 { + reg = <3>; + + adx2_out3_ep: endpoint { + remote-endpoint = <&xbar_adx2_out3_ep>; + }; + }; + + adx2_out4_port: port@4 { + reg = <4>; + + adx2_out4_ep: endpoint { + remote-endpoint = <&xbar_adx2_out4_ep>; + }; + }; + }; + }; + + amixer@702dbb00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0x0>; + + mixer_in1_ep: endpoint { + remote-endpoint = <&xbar_mixer_in1_ep>; + }; + }; + + port@1 { + reg = <0x1>; + + mixer_in2_ep: endpoint { + remote-endpoint = <&xbar_mixer_in2_ep>; + }; + }; + + port@2 { + reg = <0x2>; + + mixer_in3_ep: endpoint { + remote-endpoint = <&xbar_mixer_in3_ep>; + }; + }; + + port@3 { + reg = <0x3>; + + mixer_in4_ep: endpoint { + remote-endpoint = <&xbar_mixer_in4_ep>; + }; + }; + + port@4 { + reg = <0x4>; + + mixer_in5_ep: endpoint { + remote-endpoint = <&xbar_mixer_in5_ep>; + }; + }; + + port@5 { + reg = <0x5>; + + mixer_in6_ep: endpoint { + remote-endpoint = <&xbar_mixer_in6_ep>; + }; + }; + + port@6 { + reg = <0x6>; + + mixer_in7_ep: endpoint { + remote-endpoint = <&xbar_mixer_in7_ep>; + }; + }; + + port@7 { + reg = <0x7>; + + mixer_in8_ep: endpoint { + remote-endpoint = <&xbar_mixer_in8_ep>; + }; + }; + + port@8 { + reg = <0x8>; + + mixer_in9_ep: endpoint { + remote-endpoint = <&xbar_mixer_in9_ep>; + }; + }; + + port@9 { + reg = <0x9>; + + mixer_in10_ep: endpoint { + remote-endpoint = <&xbar_mixer_in10_ep>; + }; + }; + + mixer_out1_port: port@a { + reg = <0xa>; + + mixer_out1_ep: endpoint { + remote-endpoint = <&xbar_mixer_out1_ep>; + }; + }; + + mixer_out2_port: port@b { + reg = <0xb>; + + mixer_out2_ep: endpoint { + remote-endpoint = <&xbar_mixer_out2_ep>; + }; + }; + + mixer_out3_port: port@c { + reg = <0xc>; + + mixer_out3_ep: endpoint { + remote-endpoint = <&xbar_mixer_out3_ep>; + }; + }; + + mixer_out4_port: port@d { + reg = <0xd>; + + mixer_out4_ep: endpoint { + remote-endpoint = <&xbar_mixer_out4_ep>; + }; + }; + + mixer_out5_port: port@e { + reg = <0xe>; + + mixer_out5_ep: endpoint { + remote-endpoint = <&xbar_mixer_out5_ep>; + }; + }; + }; + }; + ports { xbar_i2s1_port: port@a { reg = <0xa>; @@ -403,6 +875,382 @@ xbar_dmic3_ep: endpoint { remote-endpoint = <&dmic3_cif_ep>; }; }; + + xbar_sfc1_in_port: port@12 { + reg = <0x12>; + + xbar_sfc1_in_ep: endpoint { + remote-endpoint = <&sfc1_cif_in_ep>; + }; + }; + + port@13 { + reg = <0x13>; + + xbar_sfc1_out_ep: endpoint { + remote-endpoint = <&sfc1_cif_out_ep>; + }; + }; + + xbar_sfc2_in_port: port@14 { + reg = <0x14>; + + xbar_sfc2_in_ep: endpoint { + remote-endpoint = <&sfc2_cif_in_ep>; + }; + }; + + port@15 { + reg = <0x15>; + + xbar_sfc2_out_ep: endpoint { + remote-endpoint = <&sfc2_cif_out_ep>; + }; + }; + + xbar_sfc3_in_port: port@16 { + reg = <0x16>; + + xbar_sfc3_in_ep: endpoint { + remote-endpoint = <&sfc3_cif_in_ep>; + }; + }; + + port@17 { + reg = <0x17>; + + xbar_sfc3_out_ep: endpoint { + remote-endpoint = <&sfc3_cif_out_ep>; + }; + }; + + xbar_sfc4_in_port: port@18 { + reg = <0x18>; + + xbar_sfc4_in_ep: endpoint { + remote-endpoint = <&sfc4_cif_in_ep>; + }; + }; + + port@19 { + reg = <0x19>; + + xbar_sfc4_out_ep: endpoint { + remote-endpoint = <&sfc4_cif_out_ep>; + }; + }; + + xbar_mvc1_in_port: port@1a { + reg = <0x1a>; + + xbar_mvc1_in_ep: endpoint { + remote-endpoint = <&mvc1_cif_in_ep>; + }; + }; + + port@1b { + reg = <0x1b>; + + xbar_mvc1_out_ep: endpoint { + remote-endpoint = <&mvc1_cif_out_ep>; + }; + }; + + xbar_mvc2_in_port: port@1c { + reg = <0x1c>; + + xbar_mvc2_in_ep: endpoint { + remote-endpoint = <&mvc2_cif_in_ep>; + }; + }; + + port@1d { + reg = <0x1d>; + + xbar_mvc2_out_ep: endpoint { + remote-endpoint = <&mvc2_cif_out_ep>; + }; + }; + + xbar_amx1_in1_port: port@1e { + reg = <0x1e>; + + xbar_amx1_in1_ep: endpoint { + remote-endpoint = <&amx1_in1_ep>; + }; + }; + + xbar_amx1_in2_port: port@1f { + reg = <0x1f>; + + xbar_amx1_in2_ep: endpoint { + remote-endpoint = <&amx1_in2_ep>; + }; + }; + + xbar_amx1_in3_port: port@20 { + reg = <0x20>; + + xbar_amx1_in3_ep: endpoint { + remote-endpoint = <&amx1_in3_ep>; + }; + }; + + xbar_amx1_in4_port: port@21 { + reg = <0x21>; + + xbar_amx1_in4_ep: endpoint { + remote-endpoint = <&amx1_in4_ep>; + }; + }; + + port@22 { + reg = <0x22>; + + xbar_amx1_out_ep: endpoint { + remote-endpoint = <&amx1_out_ep>; + }; + }; + + xbar_amx2_in1_port: port@23 { + reg = <0x23>; + + xbar_amx2_in1_ep: endpoint { + remote-endpoint = <&amx2_in1_ep>; + }; + }; + + xbar_amx2_in2_port: port@24 { + reg = <0x24>; + + xbar_amx2_in2_ep: endpoint { + remote-endpoint = <&amx2_in2_ep>; + }; + }; + + xbar_amx2_in3_port: port@25 { + reg = <0x25>; + + xbar_amx2_in3_ep: endpoint { + remote-endpoint = <&amx2_in3_ep>; + }; + }; + + xbar_amx2_in4_port: port@26 { + reg = <0x26>; + + xbar_amx2_in4_ep: endpoint { + remote-endpoint = <&amx2_in4_ep>; + }; + }; + + port@27 { + reg = <0x27>; + + xbar_amx2_out_ep: endpoint { + remote-endpoint = <&amx2_out_ep>; + }; + }; + + xbar_adx1_in_port: port@28 { + reg = <0x28>; + + xbar_adx1_in_ep: endpoint { + remote-endpoint = <&adx1_in_ep>; + }; + }; + + port@29 { + reg = <0x29>; + + xbar_adx1_out1_ep: endpoint { + remote-endpoint = <&adx1_out1_ep>; + }; + }; + + port@2a { + reg = <0x2a>; + + xbar_adx1_out2_ep: endpoint { + remote-endpoint = <&adx1_out2_ep>; + }; + }; + + port@2b { + reg = <0x2b>; + + xbar_adx1_out3_ep: endpoint { + remote-endpoint = <&adx1_out3_ep>; + }; + }; + + port@2c { + reg = <0x2c>; + + xbar_adx1_out4_ep: endpoint { + remote-endpoint = <&adx1_out4_ep>; + }; + }; + + xbar_adx2_in_port: port@2d { + reg = <0x2d>; + + xbar_adx2_in_ep: endpoint { + remote-endpoint = <&adx2_in_ep>; + }; + }; + + port@2e { + reg = <0x2e>; + + xbar_adx2_out1_ep: endpoint { + remote-endpoint = <&adx2_out1_ep>; + }; + }; + + port@2f { + reg = <0x2f>; + + xbar_adx2_out2_ep: endpoint { + remote-endpoint = <&adx2_out2_ep>; + }; + }; + + port@30 { + reg = <0x30>; + + xbar_adx2_out3_ep: endpoint { + remote-endpoint = <&adx2_out3_ep>; + }; + }; + + port@31 { + reg = <0x31>; + + xbar_adx2_out4_ep: endpoint { + remote-endpoint = <&adx2_out4_ep>; + }; + }; + + xbar_mixer_in1_port: port@32 { + reg = <0x32>; + + xbar_mixer_in1_ep: endpoint { + remote-endpoint = <&mixer_in1_ep>; + }; + }; + + xbar_mixer_in2_port: port@33 { + reg = <0x33>; + + xbar_mixer_in2_ep: endpoint { + remote-endpoint = <&mixer_in2_ep>; + }; + }; + + xbar_mixer_in3_port: port@34 { + reg = <0x34>; + + xbar_mixer_in3_ep: endpoint { + remote-endpoint = <&mixer_in3_ep>; + }; + }; + + xbar_mixer_in4_port: port@35 { + reg = <0x35>; + + xbar_mixer_in4_ep: endpoint { + remote-endpoint = <&mixer_in4_ep>; + }; + }; + + xbar_mixer_in5_port: port@36 { + reg = <0x36>; + + xbar_mixer_in5_ep: endpoint { + remote-endpoint = <&mixer_in5_ep>; + }; + }; + + xbar_mixer_in6_port: port@37 { + reg = <0x37>; + + xbar_mixer_in6_ep: endpoint { + remote-endpoint = <&mixer_in6_ep>; + }; + }; + + xbar_mixer_in7_port: port@38 { + reg = <0x38>; + + xbar_mixer_in7_ep: endpoint { + remote-endpoint = <&mixer_in7_ep>; + }; + }; + + xbar_mixer_in8_port: port@39 { + reg = <0x39>; + + xbar_mixer_in8_ep: endpoint { + remote-endpoint = <&mixer_in8_ep>; + }; + }; + + xbar_mixer_in9_port: port@3a { + reg = <0x3a>; + + xbar_mixer_in9_ep: endpoint { + remote-endpoint = <&mixer_in9_ep>; + }; + }; + + xbar_mixer_in10_port: port@3b { + reg = <0x3b>; + + xbar_mixer_in10_ep: endpoint { + remote-endpoint = <&mixer_in10_ep>; + }; + }; + + port@3c { + reg = <0x3c>; + + xbar_mixer_out1_ep: endpoint { + remote-endpoint = <&mixer_out1_ep>; + }; + }; + + port@3d { + reg = <0x3d>; + + xbar_mixer_out2_ep: endpoint { + remote-endpoint = <&mixer_out2_ep>; + }; + }; + + port@3e { + reg = <0x3e>; + + xbar_mixer_out3_ep: endpoint { + remote-endpoint = <&mixer_out3_ep>; + }; + }; + + port@3f { + reg = <0x3f>; + + xbar_mixer_out4_ep: endpoint { + remote-endpoint = <&mixer_out4_ep>; + }; + }; + + port@40 { + reg = <0x40>; + + xbar_mixer_out5_ep: endpoint { + remote-endpoint = <&mixer_out5_ep>; + }; + }; }; }; }; @@ -420,6 +1268,31 @@ sound { <&xbar_i2s1_port>, <&xbar_i2s2_port>, <&xbar_i2s3_port>, <&xbar_i2s4_port>, <&xbar_i2s5_port>, <&xbar_dmic1_port>, <&xbar_dmic2_port>, <&xbar_dmic3_port>, + <&xbar_sfc1_in_port>, <&xbar_sfc2_in_port>, + <&xbar_sfc3_in_port>, <&xbar_sfc4_in_port>, + <&xbar_mvc1_in_port>, <&xbar_mvc2_in_port>, + <&xbar_amx1_in1_port>, <&xbar_amx1_in2_port>, + <&xbar_amx1_in3_port>, <&xbar_amx1_in4_port>, + <&xbar_amx2_in1_port>, <&xbar_amx2_in2_port>, + <&xbar_amx2_in3_port>, <&xbar_amx2_in4_port>, + <&xbar_adx1_in_port>, <&xbar_adx2_in_port>, + <&xbar_mixer_in1_port>, <&xbar_mixer_in2_port>, + <&xbar_mixer_in3_port>, <&xbar_mixer_in4_port>, + <&xbar_mixer_in5_port>, <&xbar_mixer_in6_port>, + <&xbar_mixer_in7_port>, <&xbar_mixer_in8_port>, + <&xbar_mixer_in9_port>, <&xbar_mixer_in10_port>, + /* HW accelerators */ + <&sfc1_out_port>, <&sfc2_out_port>, + <&sfc3_out_port>, <&sfc4_out_port>, + <&mvc1_out_port>, <&mvc2_out_port>, + <&amx1_out_port>, <&amx2_out_port>, + <&adx1_out1_port>, <&adx1_out2_port>, + <&adx1_out3_port>, <&adx1_out4_port>, + <&adx2_out1_port>, <&adx2_out2_port>, + <&adx2_out3_port>, <&adx2_out4_port>, + <&mixer_out1_port>, <&mixer_out2_port>, + <&mixer_out3_port>, <&mixer_out4_port>, + <&mixer_out5_port>, /* I/O DAP Ports */ <&i2s1_port>, <&i2s2_port>, <&i2s3_port>, <&i2s4_port>, <&i2s5_port>, <&dmic1_port>, <&dmic2_port>, <&dmic3_port>; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2530.dtsi b/arch/arm64/boot/dts/nvidia/tegra210-p2530.dtsi index 58aa051896..0a70daeffd 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-p2530.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2530.dtsi @@ -40,7 +40,7 @@ mmc@700b0600 { non-removable; }; - clk32k_in: clock@0 { + clk32k_in: clock-32k { compatible = "fixed-clock"; clock-frequency = <32768>; #clock-cells = <0>; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2597.dtsi b/arch/arm64/boot/dts/nvidia/tegra210-p2597.dtsi index d8409c1b43..4b43b89a96 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-p2597.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2597.dtsi @@ -1361,11 +1361,6 @@ usb@70090000 { dvddio-pex-supply = <&vdd_pex_1v05>; hvddio-pex-supply = <&vdd_1v8>; avdd-usb-supply = <&vdd_3v3_sys>; - /* XXX what are these? */ - avdd-pll-utmip-supply = <&vdd_1v8>; - avdd-pll-uerefe-supply = <&vdd_pex_1v05>; - dvdd-usb-ss-pll-supply = <&vdd_pex_1v05>; - hvdd-usb-ss-pll-e-supply = <&vdd_1v8>; status = "okay"; @@ -1555,7 +1550,7 @@ volume_up { }; }; - vdd_sys_mux: regulator@0 { + vdd_sys_mux: regulator-vdd-sys-mux { compatible = "regulator-fixed"; regulator-name = "VDD_SYS_MUX"; regulator-min-microvolt = <5000000>; @@ -1564,7 +1559,7 @@ vdd_sys_mux: regulator@0 { regulator-boot-on; }; - vdd_5v0_sys: regulator@1 { + vdd_5v0_sys: regulator-vdd-5v0-sys { compatible = "regulator-fixed"; regulator-name = "VDD_5V0_SYS"; regulator-min-microvolt = <5000000>; @@ -1576,7 +1571,7 @@ vdd_5v0_sys: regulator@1 { vin-supply = <&vdd_sys_mux>; }; - vdd_3v3_sys: regulator@2 { + vdd_3v3_sys: regulator-vdd-3v3-sys { compatible = "regulator-fixed"; regulator-name = "VDD_3V3_SYS"; regulator-min-microvolt = <3300000>; @@ -1588,10 +1583,9 @@ vdd_3v3_sys: regulator@2 { vin-supply = <&vdd_sys_mux>; regulator-enable-ramp-delay = <160>; - regulator-disable-ramp-delay = <10000>; }; - vdd_5v0_io: regulator@3 { + vdd_5v0_io: regulator-vdd-5v0-io { compatible = "regulator-fixed"; regulator-name = "VDD_5V0_IO_SYS"; regulator-min-microvolt = <5000000>; @@ -1600,7 +1594,7 @@ vdd_5v0_io: regulator@3 { regulator-boot-on; }; - vdd_3v3_sd: regulator@4 { + vdd_3v3_sd: regulator-vdd-3v3-sd { compatible = "regulator-fixed"; regulator-name = "VDD_3V3_SD"; regulator-min-microvolt = <3300000>; @@ -1610,10 +1604,9 @@ vdd_3v3_sd: regulator@4 { vin-supply = <&vdd_3v3_sys>; regulator-enable-ramp-delay = <472>; - regulator-disable-ramp-delay = <4880>; }; - vdd_dsi_csi: regulator@5 { + vdd_dsi_csi: regulator-vdd-dsi-csi { compatible = "regulator-fixed"; regulator-name = "AVDD_DSI_CSI_1V2"; regulator-min-microvolt = <1200000>; @@ -1621,7 +1614,7 @@ vdd_dsi_csi: regulator@5 { vin-supply = <&vdd_sys_1v2>; }; - vdd_3v3_dis: regulator@6 { + vdd_3v3_dis: regulator-vdd-3v3-dis { compatible = "regulator-fixed"; regulator-name = "VDD_DIS_3V3_LCD"; regulator-min-microvolt = <3300000>; @@ -1632,7 +1625,7 @@ vdd_3v3_dis: regulator@6 { vin-supply = <&vdd_3v3_sys>; }; - vdd_1v8_dis: regulator@7 { + vdd_1v8_dis: regulator-vdd-1v8-dis { compatible = "regulator-fixed"; regulator-name = "VDD_LCD_1V8_DIS"; regulator-min-microvolt = <1800000>; @@ -1643,7 +1636,7 @@ vdd_1v8_dis: regulator@7 { vin-supply = <&vdd_1v8>; }; - vdd_5v0_rtl: regulator@8 { + vdd_5v0_rtl: regulator-vdd-5v0-rtl { compatible = "regulator-fixed"; regulator-name = "RTL_5V"; regulator-min-microvolt = <5000000>; @@ -1653,7 +1646,7 @@ vdd_5v0_rtl: regulator@8 { vin-supply = <&vdd_5v0_sys>; }; - vdd_usb_vbus: regulator@9 { + vdd_usb_vbus: regulator-vdd-usb-vbus { compatible = "regulator-fixed"; regulator-name = "USB_VBUS_EN1"; regulator-min-microvolt = <5000000>; @@ -1663,7 +1656,7 @@ vdd_usb_vbus: regulator@9 { vin-supply = <&vdd_5v0_sys>; }; - vdd_hdmi: regulator@10 { + vdd_hdmi: regulator-vdd-hdmi { compatible = "regulator-fixed"; regulator-name = "VDD_HDMI_5V0"; regulator-min-microvolt = <5000000>; @@ -1673,7 +1666,7 @@ vdd_hdmi: regulator@10 { vin-supply = <&vdd_5v0_sys>; }; - vdd_cam_1v2: regulator@11 { + vdd_cam_1v2: regulator-vdd-cam-1v8 { compatible = "regulator-fixed"; regulator-name = "vdd-cam-1v2"; regulator-min-microvolt = <1200000>; @@ -1683,7 +1676,7 @@ vdd_cam_1v2: regulator@11 { vin-supply = <&vdd_3v3_sys>; }; - vdd_cam_2v8: regulator@12 { + vdd_cam_2v8: regulator-vdd-cam-2v8 { compatible = "regulator-fixed"; regulator-name = "vdd-cam-2v8"; regulator-min-microvolt = <2800000>; @@ -1693,7 +1686,7 @@ vdd_cam_2v8: regulator@12 { vin-supply = <&vdd_3v3_sys>; }; - vdd_cam_1v8: regulator@13 { + vdd_cam_1v8: regulator-vdd-cam-1v8 { compatible = "regulator-fixed"; regulator-name = "vdd-cam-1v8"; regulator-min-microvolt = <1800000>; @@ -1703,7 +1696,7 @@ vdd_cam_1v8: regulator@13 { vin-supply = <&vdd_3v3_sys>; }; - vdd_usb_vbus_otg: regulator@14 { + vdd_usb_vbus_otg: regulator-vdd-usb-vbus-otg { compatible = "regulator-fixed"; regulator-name = "USB_VBUS_EN0"; regulator-min-microvolt = <5000000>; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2894.dtsi b/arch/arm64/boot/dts/nvidia/tegra210-p2894.dtsi index 41beab626d..10347b6e6e 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-p2894.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2894.dtsi @@ -1383,7 +1383,7 @@ gpio5_6_7 { }; }; - gpio@0 { + hog-0 { gpio-hog; output-high; gpios = <2 GPIO_ACTIVE_HIGH>, @@ -1586,7 +1586,7 @@ mmc@700b0600 { status = "okay"; }; - clk32k_in: clock@0 { + clk32k_in: clock-32k { compatible = "fixed-clock"; clock-frequency = <32768>; #clock-cells = <0>; @@ -1635,7 +1635,7 @@ psci { method = "smc"; }; - battery_reg: regulator@0 { + battery_reg: regulator-vdd-ac-bat { compatible = "regulator-fixed"; regulator-name = "vdd-ac-bat"; regulator-min-microvolt = <5000000>; @@ -1643,7 +1643,7 @@ battery_reg: regulator@0 { regulator-always-on; }; - vdd_3v3: regulator@1 { + vdd_3v3: regulator-vdd-3v3 { compatible = "regulator-fixed"; regulator-name = "vdd-3v3"; regulator-enable-ramp-delay = <160>; @@ -1655,7 +1655,7 @@ vdd_3v3: regulator@1 { enable-active-high; }; - max77620_gpio7: regulator@2 { + max77620_gpio7: regulator-max77620-gpio7 { compatible = "regulator-fixed"; regulator-name = "max77620-gpio7"; regulator-enable-ramp-delay = <240>; @@ -1669,7 +1669,7 @@ max77620_gpio7: regulator@2 { enable-active-high; }; - lcd_bl_en: regulator@3 { + lcd_bl_en: regulator-lcd-bl-en { compatible = "regulator-fixed"; regulator-name = "lcd-bl-en"; regulator-min-microvolt = <1800000>; @@ -1680,7 +1680,7 @@ lcd_bl_en: regulator@3 { enable-active-high; }; - en_vdd_sd: regulator@4 { + en_vdd_sd: regulator-vdd-sd { compatible = "regulator-fixed"; regulator-name = "en-vdd-sd"; regulator-enable-ramp-delay = <472>; @@ -1692,7 +1692,7 @@ en_vdd_sd: regulator@4 { enable-active-high; }; - en_vdd_cam: regulator@5 { + en_vdd_cam: regulator-vdd-cam { compatible = "regulator-fixed"; regulator-name = "en-vdd-cam"; regulator-min-microvolt = <1800000>; @@ -1702,7 +1702,7 @@ en_vdd_cam: regulator@5 { enable-active-high; }; - vdd_sys_boost: regulator@6 { + vdd_sys_boost: regulator-vdd-sys-boost { compatible = "regulator-fixed"; regulator-name = "vdd-sys-boost"; regulator-enable-ramp-delay = <3090>; @@ -1714,7 +1714,7 @@ vdd_sys_boost: regulator@6 { enable-active-high; }; - vdd_hdmi: regulator@7 { + vdd_hdmi: regulator-vdd-hdmi { compatible = "regulator-fixed"; regulator-name = "vdd-hdmi"; regulator-enable-ramp-delay = <468>; @@ -1727,21 +1727,21 @@ vdd_hdmi: regulator@7 { enable-active-high; }; - en_vdd_cpu_fixed: regulator@8 { + en_vdd_cpu_fixed: regulator-vdd-cpu-fixed { compatible = "regulator-fixed"; regulator-name = "vdd-cpu-fixed"; regulator-min-microvolt = <1000000>; regulator-max-microvolt = <1000000>; }; - vdd_aux_3v3: regulator@9 { + vdd_aux_3v3: regulator-vdd-aux-3v3 { compatible = "regulator-fixed"; regulator-name = "aux-3v3"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; }; - vdd_snsr_pm: regulator@10 { + vdd_snsr_pm: regulator-vdd-snsr-pm { compatible = "regulator-fixed"; regulator-name = "snsr_pm"; regulator-min-microvolt = <3300000>; @@ -1750,7 +1750,7 @@ vdd_snsr_pm: regulator@10 { enable-active-high; }; - vdd_usb_5v0: regulator@11 { + vdd_usb_5v0: regulator-vdd-usb-5v0 { compatible = "regulator-fixed"; status = "disabled"; regulator-name = "vdd-usb-5v0"; @@ -1761,7 +1761,7 @@ vdd_usb_5v0: regulator@11 { enable-active-high; }; - vdd_cdc_1v2_aud: regulator@101 { + vdd_cdc_1v2_aud: regulator-vdd-cdc-1v2-aud { compatible = "regulator-fixed"; status = "disabled"; regulator-name = "vdd_cdc_1v2_aud"; @@ -1772,7 +1772,7 @@ vdd_cdc_1v2_aud: regulator@101 { enable-active-high; }; - vdd_disp_3v0: regulator@12 { + vdd_disp_3v0: regulator-vdd-disp-3v0 { compatible = "regulator-fixed"; regulator-name = "vdd-disp-3v0"; regulator-enable-ramp-delay = <232>; @@ -1784,7 +1784,7 @@ vdd_disp_3v0: regulator@12 { enable-active-high; }; - vdd_fan: regulator@13 { + vdd_fan: regulator-vdd-fan { compatible = "regulator-fixed"; regulator-name = "vdd-fan"; regulator-enable-ramp-delay = <284>; @@ -1795,7 +1795,7 @@ vdd_fan: regulator@13 { enable-active-high; }; - usb_vbus1: regulator@14 { + usb_vbus1: regulator-usb-vbus1 { compatible = "regulator-fixed"; regulator-name = "usb-vbus1"; regulator-min-microvolt = <5000000>; @@ -1806,7 +1806,7 @@ usb_vbus1: regulator@14 { gpio-open-drain; }; - usb_vbus2: regulator@15 { + usb_vbus2: regulator-usb-vbus2 { compatible = "regulator-fixed"; regulator-name = "usb-vbus2"; regulator-min-microvolt = <5000000>; @@ -1817,7 +1817,7 @@ usb_vbus2: regulator@15 { gpio-open-drain; }; - vdd_3v3_eth: regulator@16 { + vdd_3v3_eth: regulator-vdd-3v3-eth { compatible = "regulator-fixed"; regulator-name = "vdd-3v3-eth-a02"; regulator-min-microvolt = <3300000>; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dts b/arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dts index 7dbb13f20d..72c2dc3c14 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dts +++ b/arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dts @@ -30,11 +30,8 @@ memory@80000000 { pcie@1003000 { status = "okay"; - avdd-pll-uerefe-supply = <&vdd_pex_1v05>; hvddio-pex-supply = <&vdd_1v8>; dvddio-pex-supply = <&vdd_pex_1v05>; - dvdd-pex-pll-supply = <&vdd_pex_1v05>; - hvdd-pex-pll-e-supply = <&vdd_1v8>; vddio-pex-ctl-supply = <&vdd_1v8>; pci@1,0 { @@ -266,7 +263,6 @@ vdd_soc: sd0 { regulator-min-microvolt = <1000000>; regulator-max-microvolt = <1170000>; regulator-enable-ramp-delay = <146>; - regulator-disable-ramp-delay = <4080>; regulator-ramp-delay = <27500>; regulator-ramp-delay-scale = <300>; regulator-always-on; @@ -282,7 +278,6 @@ vdd_ddr: sd1 { regulator-min-microvolt = <1150000>; regulator-max-microvolt = <1150000>; regulator-enable-ramp-delay = <176>; - regulator-disable-ramp-delay = <145800>; regulator-ramp-delay = <27500>; regulator-ramp-delay-scale = <300>; regulator-always-on; @@ -298,7 +293,6 @@ vdd_pre: sd2 { regulator-min-microvolt = <1350000>; regulator-max-microvolt = <1350000>; regulator-enable-ramp-delay = <176>; - regulator-disable-ramp-delay = <32000>; regulator-ramp-delay = <27500>; regulator-ramp-delay-scale = <350>; regulator-always-on; @@ -314,7 +308,6 @@ vdd_1v8: sd3 { regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-enable-ramp-delay = <242>; - regulator-disable-ramp-delay = <118000>; regulator-ramp-delay = <27500>; regulator-ramp-delay-scale = <360>; regulator-always-on; @@ -330,7 +323,6 @@ vdd_sys_1v2: ldo0 { regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-enable-ramp-delay = <26>; - regulator-disable-ramp-delay = <626>; regulator-ramp-delay = <100000>; regulator-ramp-delay-scale = <200>; regulator-always-on; @@ -346,7 +338,6 @@ vdd_pex_1v05: ldo1 { regulator-min-microvolt = <1050000>; regulator-max-microvolt = <1050000>; regulator-enable-ramp-delay = <22>; - regulator-disable-ramp-delay = <650>; regulator-ramp-delay = <100000>; regulator-ramp-delay-scale = <200>; @@ -360,7 +351,6 @@ vddio_sdmmc: ldo2 { regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; regulator-enable-ramp-delay = <62>; - regulator-disable-ramp-delay = <650>; regulator-ramp-delay = <100000>; regulator-ramp-delay-scale = <200>; @@ -378,7 +368,6 @@ vdd_rtc: ldo4 { regulator-min-microvolt = <850000>; regulator-max-microvolt = <1100000>; regulator-enable-ramp-delay = <22>; - regulator-disable-ramp-delay = <610>; regulator-ramp-delay = <100000>; regulator-ramp-delay-scale = <200>; regulator-disable-active-discharge; @@ -403,7 +392,6 @@ avdd_1v05_pll: ldo7 { regulator-min-microvolt = <1050000>; regulator-max-microvolt = <1050000>; regulator-enable-ramp-delay = <24>; - regulator-disable-ramp-delay = <2768>; regulator-ramp-delay = <100000>; regulator-ramp-delay-scale = <200>; @@ -417,7 +405,6 @@ avdd_1v05: ldo8 { regulator-min-microvolt = <1050000>; regulator-max-microvolt = <1050000>; regulator-enable-ramp-delay = <22>; - regulator-disable-ramp-delay = <1160>; regulator-ramp-delay = <100000>; regulator-ramp-delay-scale = <200>; @@ -456,11 +443,6 @@ usb@70090000 { avdd-usb-supply = <&vdd_3v3_sys>; dvddio-pex-supply = <&vdd_pex_1v05>; hvddio-pex-supply = <&vdd_1v8>; - /* these really belong to the XUSB pad controller */ - avdd-pll-utmip-supply = <&vdd_1v8>; - avdd-pll-uerefe-supply = <&vdd_pex_1v05>; - dvdd-usb-ss-pll-supply = <&vdd_pex_1v05>; - hvdd-usb-ss-pll-e-supply = <&vdd_1v8>; status = "okay"; }; @@ -688,7 +670,7 @@ i2s4_cif_ep: endpoint { i2s4_port: port@1 { reg = <1>; - i2s4_dap_ep: endpoint@0 { + i2s4_dap_ep: endpoint { dai-format = "i2s"; /* Placeholder for external Codec */ }; @@ -706,7 +688,7 @@ ports { port@0 { reg = <0>; - dmic1_cif_ep: endpoint@0 { + dmic1_cif_ep: endpoint { remote-endpoint = <&xbar_dmic1_ep>; }; }; @@ -714,7 +696,7 @@ dmic1_cif_ep: endpoint@0 { dmic1_port: port@1 { reg = <1>; - dmic1_dap_ep: endpoint@0 { + dmic1_dap_ep: endpoint { /* Placeholder for external Codec */ }; }; @@ -731,7 +713,7 @@ ports { port@0 { reg = <0>; - dmic2_cif_ep: endpoint@0 { + dmic2_cif_ep: endpoint { remote-endpoint = <&xbar_dmic2_ep>; }; }; @@ -739,13 +721,488 @@ dmic2_cif_ep: endpoint@0 { dmic2_port: port@1 { reg = <1>; - dmic2_dap_ep: endpoint@0 { + dmic2_dap_ep: endpoint { /* Placeholder for external Codec */ }; }; }; }; + sfc@702d2000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc1_in_ep>; + }; + }; + + sfc1_out_port: port@1 { + reg = <1>; + + sfc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc1_out_ep>; + }; + }; + }; + }; + + sfc@702d2200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc2_in_ep>; + }; + }; + + sfc2_out_port: port@1 { + reg = <1>; + + sfc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc2_out_ep>; + }; + }; + }; + }; + + sfc@702d2400 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc3_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc3_in_ep>; + }; + }; + + sfc3_out_port: port@1 { + reg = <1>; + + sfc3_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc3_out_ep>; + }; + }; + }; + }; + + sfc@702d2600 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sfc4_cif_in_ep: endpoint { + remote-endpoint = <&xbar_sfc4_in_ep>; + }; + }; + + sfc4_out_port: port@1 { + reg = <1>; + + sfc4_cif_out_ep: endpoint { + remote-endpoint = <&xbar_sfc4_out_ep>; + }; + }; + }; + }; + + mvc@702da000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc1_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc1_in_ep>; + }; + }; + + mvc1_out_port: port@1 { + reg = <1>; + + mvc1_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc1_out_ep>; + }; + }; + }; + }; + + mvc@702da200 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + mvc2_cif_in_ep: endpoint { + remote-endpoint = <&xbar_mvc2_in_ep>; + }; + }; + + mvc2_out_port: port@1 { + reg = <1>; + + mvc2_cif_out_ep: endpoint { + remote-endpoint = <&xbar_mvc2_out_ep>; + }; + }; + }; + }; + + amx@702d3000 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx1_in1_ep: endpoint { + remote-endpoint = <&xbar_amx1_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx1_in2_ep: endpoint { + remote-endpoint = <&xbar_amx1_in2_ep>; + }; + }; + + port@2 { + reg = <2>; + + amx1_in3_ep: endpoint { + remote-endpoint = <&xbar_amx1_in3_ep>; + }; + }; + + port@3 { + reg = <3>; + + amx1_in4_ep: endpoint { + remote-endpoint = <&xbar_amx1_in4_ep>; + }; + }; + + amx1_out_port: port@4 { + reg = <4>; + + amx1_out_ep: endpoint { + remote-endpoint = <&xbar_amx1_out_ep>; + }; + }; + }; + }; + + amx@702d3100 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + amx2_in1_ep: endpoint { + remote-endpoint = <&xbar_amx2_in1_ep>; + }; + }; + + port@1 { + reg = <1>; + + amx2_in2_ep: endpoint { + remote-endpoint = <&xbar_amx2_in2_ep>; + }; + }; + + amx2_in3_port: port@2 { + reg = <2>; + + amx2_in3_ep: endpoint { + remote-endpoint = <&xbar_amx2_in3_ep>; + }; + }; + + amx2_in4_port: port@3 { + reg = <3>; + + amx2_in4_ep: endpoint { + remote-endpoint = <&xbar_amx2_in4_ep>; + }; + }; + + amx2_out_port: port@4 { + reg = <4>; + + amx2_out_ep: endpoint { + remote-endpoint = <&xbar_amx2_out_ep>; + }; + }; + }; + }; + + adx@702d3800 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx1_in_ep: endpoint { + remote-endpoint = <&xbar_adx1_in_ep>; + }; + }; + + adx1_out1_port: port@1 { + reg = <1>; + + adx1_out1_ep: endpoint { + remote-endpoint = <&xbar_adx1_out1_ep>; + }; + }; + + adx1_out2_port: port@2 { + reg = <2>; + + adx1_out2_ep: endpoint { + remote-endpoint = <&xbar_adx1_out2_ep>; + }; + }; + + adx1_out3_port: port@3 { + reg = <3>; + + adx1_out3_ep: endpoint { + remote-endpoint = <&xbar_adx1_out3_ep>; + }; + }; + + adx1_out4_port: port@4 { + reg = <4>; + + adx1_out4_ep: endpoint { + remote-endpoint = <&xbar_adx1_out4_ep>; + }; + }; + }; + }; + + adx@702d3900 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + adx2_in_ep: endpoint { + remote-endpoint = <&xbar_adx2_in_ep>; + }; + }; + + adx2_out1_port: port@1 { + reg = <1>; + + adx2_out1_ep: endpoint { + remote-endpoint = <&xbar_adx2_out1_ep>; + }; + }; + + adx2_out2_port: port@2 { + reg = <2>; + + adx2_out2_ep: endpoint { + remote-endpoint = <&xbar_adx2_out2_ep>; + }; + }; + + adx2_out3_port: port@3 { + reg = <3>; + + adx2_out3_ep: endpoint { + remote-endpoint = <&xbar_adx2_out3_ep>; + }; + }; + + adx2_out4_port: port@4 { + reg = <4>; + + adx2_out4_ep: endpoint { + remote-endpoint = <&xbar_adx2_out4_ep>; + }; + }; + }; + }; + + amixer@702dbb00 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0x0>; + + mixer_in1_ep: endpoint { + remote-endpoint = <&xbar_mixer_in1_ep>; + }; + }; + + port@1 { + reg = <0x1>; + + mixer_in2_ep: endpoint { + remote-endpoint = <&xbar_mixer_in2_ep>; + }; + }; + + port@2 { + reg = <0x2>; + + mixer_in3_ep: endpoint { + remote-endpoint = <&xbar_mixer_in3_ep>; + }; + }; + + port@3 { + reg = <0x3>; + + mixer_in4_ep: endpoint { + remote-endpoint = <&xbar_mixer_in4_ep>; + }; + }; + + port@4 { + reg = <0x4>; + + mixer_in5_ep: endpoint { + remote-endpoint = <&xbar_mixer_in5_ep>; + }; + }; + + port@5 { + reg = <0x5>; + + mixer_in6_ep: endpoint { + remote-endpoint = <&xbar_mixer_in6_ep>; + }; + }; + + port@6 { + reg = <0x6>; + + mixer_in7_ep: endpoint { + remote-endpoint = <&xbar_mixer_in7_ep>; + }; + }; + + port@7 { + reg = <0x7>; + + mixer_in8_ep: endpoint { + remote-endpoint = <&xbar_mixer_in8_ep>; + }; + }; + + port@8 { + reg = <0x8>; + + mixer_in9_ep: endpoint { + remote-endpoint = <&xbar_mixer_in9_ep>; + }; + }; + + port@9 { + reg = <0x9>; + + mixer_in10_ep: endpoint { + remote-endpoint = <&xbar_mixer_in10_ep>; + }; + }; + + mixer_out1_port: port@a { + reg = <0xa>; + + mixer_out1_ep: endpoint { + remote-endpoint = <&xbar_mixer_out1_ep>; + }; + }; + + mixer_out2_port: port@b { + reg = <0xb>; + + mixer_out2_ep: endpoint { + remote-endpoint = <&xbar_mixer_out2_ep>; + }; + }; + + mixer_out3_port: port@c { + reg = <0xc>; + + mixer_out3_ep: endpoint { + remote-endpoint = <&xbar_mixer_out3_ep>; + }; + }; + + mixer_out4_port: port@d { + reg = <0xd>; + + mixer_out4_ep: endpoint { + remote-endpoint = <&xbar_mixer_out4_ep>; + }; + }; + + mixer_out5_port: port@e { + reg = <0xe>; + + mixer_out5_ep: endpoint { + remote-endpoint = <&xbar_mixer_out5_ep>; + }; + }; + }; + }; + ports { xbar_i2s3_port: port@c { reg = <0xc>; @@ -778,6 +1235,382 @@ xbar_dmic2_ep: endpoint { remote-endpoint = <&dmic2_cif_ep>; }; }; + + xbar_sfc1_in_port: port@12 { + reg = <0x12>; + + xbar_sfc1_in_ep: endpoint { + remote-endpoint = <&sfc1_cif_in_ep>; + }; + }; + + port@13 { + reg = <0x13>; + + xbar_sfc1_out_ep: endpoint { + remote-endpoint = <&sfc1_cif_out_ep>; + }; + }; + + xbar_sfc2_in_port: port@14 { + reg = <0x14>; + + xbar_sfc2_in_ep: endpoint { + remote-endpoint = <&sfc2_cif_in_ep>; + }; + }; + + port@15 { + reg = <0x15>; + + xbar_sfc2_out_ep: endpoint { + remote-endpoint = <&sfc2_cif_out_ep>; + }; + }; + + xbar_sfc3_in_port: port@16 { + reg = <0x16>; + + xbar_sfc3_in_ep: endpoint { + remote-endpoint = <&sfc3_cif_in_ep>; + }; + }; + + port@17 { + reg = <0x17>; + + xbar_sfc3_out_ep: endpoint { + remote-endpoint = <&sfc3_cif_out_ep>; + }; + }; + + xbar_sfc4_in_port: port@18 { + reg = <0x18>; + + xbar_sfc4_in_ep: endpoint { + remote-endpoint = <&sfc4_cif_in_ep>; + }; + }; + + port@19 { + reg = <0x19>; + + xbar_sfc4_out_ep: endpoint { + remote-endpoint = <&sfc4_cif_out_ep>; + }; + }; + + xbar_mvc1_in_port: port@1a { + reg = <0x1a>; + + xbar_mvc1_in_ep: endpoint { + remote-endpoint = <&mvc1_cif_in_ep>; + }; + }; + + port@1b { + reg = <0x1b>; + + xbar_mvc1_out_ep: endpoint { + remote-endpoint = <&mvc1_cif_out_ep>; + }; + }; + + xbar_mvc2_in_port: port@1c { + reg = <0x1c>; + + xbar_mvc2_in_ep: endpoint { + remote-endpoint = <&mvc2_cif_in_ep>; + }; + }; + + port@1d { + reg = <0x1d>; + + xbar_mvc2_out_ep: endpoint { + remote-endpoint = <&mvc2_cif_out_ep>; + }; + }; + + xbar_amx1_in1_port: port@1e { + reg = <0x1e>; + + xbar_amx1_in1_ep: endpoint { + remote-endpoint = <&amx1_in1_ep>; + }; + }; + + xbar_amx1_in2_port: port@1f { + reg = <0x1f>; + + xbar_amx1_in2_ep: endpoint { + remote-endpoint = <&amx1_in2_ep>; + }; + }; + + xbar_amx1_in3_port: port@20 { + reg = <0x20>; + + xbar_amx1_in3_ep: endpoint { + remote-endpoint = <&amx1_in3_ep>; + }; + }; + + xbar_amx1_in4_port: port@21 { + reg = <0x21>; + + xbar_amx1_in4_ep: endpoint { + remote-endpoint = <&amx1_in4_ep>; + }; + }; + + port@22 { + reg = <0x22>; + + xbar_amx1_out_ep: endpoint { + remote-endpoint = <&amx1_out_ep>; + }; + }; + + xbar_amx2_in1_port: port@23 { + reg = <0x23>; + + xbar_amx2_in1_ep: endpoint { + remote-endpoint = <&amx2_in1_ep>; + }; + }; + + xbar_amx2_in2_port: port@24 { + reg = <0x24>; + + xbar_amx2_in2_ep: endpoint { + remote-endpoint = <&amx2_in2_ep>; + }; + }; + + xbar_amx2_in3_port: port@25 { + reg = <0x25>; + + xbar_amx2_in3_ep: endpoint { + remote-endpoint = <&amx2_in3_ep>; + }; + }; + + xbar_amx2_in4_port: port@26 { + reg = <0x26>; + + xbar_amx2_in4_ep: endpoint { + remote-endpoint = <&amx2_in4_ep>; + }; + }; + + port@27 { + reg = <0x27>; + + xbar_amx2_out_ep: endpoint { + remote-endpoint = <&amx2_out_ep>; + }; + }; + + xbar_adx1_in_port: port@28 { + reg = <0x28>; + + xbar_adx1_in_ep: endpoint { + remote-endpoint = <&adx1_in_ep>; + }; + }; + + port@29 { + reg = <0x29>; + + xbar_adx1_out1_ep: endpoint { + remote-endpoint = <&adx1_out1_ep>; + }; + }; + + port@2a { + reg = <0x2a>; + + xbar_adx1_out2_ep: endpoint { + remote-endpoint = <&adx1_out2_ep>; + }; + }; + + port@2b { + reg = <0x2b>; + + xbar_adx1_out3_ep: endpoint { + remote-endpoint = <&adx1_out3_ep>; + }; + }; + + port@2c { + reg = <0x2c>; + + xbar_adx1_out4_ep: endpoint { + remote-endpoint = <&adx1_out4_ep>; + }; + }; + + xbar_adx2_in_port: port@2d { + reg = <0x2d>; + + xbar_adx2_in_ep: endpoint { + remote-endpoint = <&adx2_in_ep>; + }; + }; + + port@2e { + reg = <0x2e>; + + xbar_adx2_out1_ep: endpoint { + remote-endpoint = <&adx2_out1_ep>; + }; + }; + + port@2f { + reg = <0x2f>; + + xbar_adx2_out2_ep: endpoint { + remote-endpoint = <&adx2_out2_ep>; + }; + }; + + port@30 { + reg = <0x30>; + + xbar_adx2_out3_ep: endpoint { + remote-endpoint = <&adx2_out3_ep>; + }; + }; + + port@31 { + reg = <0x31>; + + xbar_adx2_out4_ep: endpoint { + remote-endpoint = <&adx2_out4_ep>; + }; + }; + + xbar_mixer_in1_port: port@32 { + reg = <0x32>; + + xbar_mixer_in1_ep: endpoint { + remote-endpoint = <&mixer_in1_ep>; + }; + }; + + xbar_mixer_in2_port: port@33 { + reg = <0x33>; + + xbar_mixer_in2_ep: endpoint { + remote-endpoint = <&mixer_in2_ep>; + }; + }; + + xbar_mixer_in3_port: port@34 { + reg = <0x34>; + + xbar_mixer_in3_ep: endpoint { + remote-endpoint = <&mixer_in3_ep>; + }; + }; + + xbar_mixer_in4_port: port@35 { + reg = <0x35>; + + xbar_mixer_in4_ep: endpoint { + remote-endpoint = <&mixer_in4_ep>; + }; + }; + + xbar_mixer_in5_port: port@36 { + reg = <0x36>; + + xbar_mixer_in5_ep: endpoint { + remote-endpoint = <&mixer_in5_ep>; + }; + }; + + xbar_mixer_in6_port: port@37 { + reg = <0x37>; + + xbar_mixer_in6_ep: endpoint { + remote-endpoint = <&mixer_in6_ep>; + }; + }; + + xbar_mixer_in7_port: port@38 { + reg = <0x38>; + + xbar_mixer_in7_ep: endpoint { + remote-endpoint = <&mixer_in7_ep>; + }; + }; + + xbar_mixer_in8_port: port@39 { + reg = <0x39>; + + xbar_mixer_in8_ep: endpoint { + remote-endpoint = <&mixer_in8_ep>; + }; + }; + + xbar_mixer_in9_port: port@3a { + reg = <0x3a>; + + xbar_mixer_in9_ep: endpoint { + remote-endpoint = <&mixer_in9_ep>; + }; + }; + + xbar_mixer_in10_port: port@3b { + reg = <0x3b>; + + xbar_mixer_in10_ep: endpoint { + remote-endpoint = <&mixer_in10_ep>; + }; + }; + + port@3c { + reg = <0x3c>; + + xbar_mixer_out1_ep: endpoint { + remote-endpoint = <&mixer_out1_ep>; + }; + }; + + port@3d { + reg = <0x3d>; + + xbar_mixer_out2_ep: endpoint { + remote-endpoint = <&mixer_out2_ep>; + }; + }; + + port@3e { + reg = <0x3e>; + + xbar_mixer_out3_ep: endpoint { + remote-endpoint = <&mixer_out3_ep>; + }; + }; + + port@3f { + reg = <0x3f>; + + xbar_mixer_out4_ep: endpoint { + remote-endpoint = <&mixer_out4_ep>; + }; + }; + + port@40 { + reg = <0x40>; + + xbar_mixer_out5_ep: endpoint { + remote-endpoint = <&mixer_out5_ep>; + }; + }; }; }; }; @@ -786,7 +1619,7 @@ spi@70410000 { status = "okay"; flash@0 { - compatible = "spi-nor"; + compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <104000000>; spi-tx-bus-width = <2>; @@ -794,7 +1627,7 @@ flash@0 { }; }; - clk32k_in: clock@0 { + clk32k_in: clock-32k { compatible = "fixed-clock"; clock-frequency = <32768>; #clock-cells = <0>; @@ -833,7 +1666,7 @@ fan: fan { }; thermal-zones { - cpu { + cpu-thermal { trips { cpu_trip_critical: critical { temperature = <96500>; @@ -911,7 +1744,7 @@ psci { method = "smc"; }; - vdd_5v0_sys: regulator@0 { + vdd_5v0_sys: regulator-vdd-5v0-sys { compatible = "regulator-fixed"; regulator-name = "VDD_5V0_SYS"; @@ -921,14 +1754,13 @@ vdd_5v0_sys: regulator@0 { regulator-boot-on; }; - vdd_3v3_sys: regulator@1 { + vdd_3v3_sys: regulator-vdd-3v3-sys { compatible = "regulator-fixed"; regulator-name = "VDD_3V3_SYS"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; regulator-enable-ramp-delay = <240>; - regulator-disable-ramp-delay = <11340>; regulator-always-on; regulator-boot-on; @@ -938,7 +1770,7 @@ vdd_3v3_sys: regulator@1 { vin-supply = <&vdd_5v0_sys>; }; - vdd_3v3_sd: regulator@2 { + vdd_3v3_sd: regulator-vdd-3v3-sd { compatible = "regulator-fixed"; regulator-name = "VDD_3V3_SD"; @@ -951,7 +1783,7 @@ vdd_3v3_sd: regulator@2 { vin-supply = <&vdd_3v3_sys>; }; - vdd_hdmi: regulator@3 { + vdd_hdmi: regulator-vdd-hdmi-5v0 { compatible = "regulator-fixed"; regulator-name = "VDD_HDMI_5V0"; @@ -961,7 +1793,7 @@ vdd_hdmi: regulator@3 { vin-supply = <&vdd_5v0_sys>; }; - vdd_hub_3v3: regulator@4 { + vdd_hub_3v3: regulator-vdd-hub-3v3 { compatible = "regulator-fixed"; regulator-name = "VDD_HUB_3V3"; @@ -974,7 +1806,7 @@ vdd_hub_3v3: regulator@4 { vin-supply = <&vdd_5v0_sys>; }; - vdd_cpu: regulator@5 { + vdd_cpu: regulator-vdd-cpu { compatible = "regulator-fixed"; regulator-name = "VDD_CPU"; @@ -989,7 +1821,7 @@ vdd_cpu: regulator@5 { vin-supply = <&vdd_5v0_sys>; }; - vdd_gpu: regulator@6 { + vdd_gpu: regulator-vdd-gpu { compatible = "pwm-regulator"; pwms = <&pwm 1 8000>; @@ -1004,7 +1836,7 @@ vdd_gpu: regulator@6 { vin-supply = <&vdd_5v0_sys>; }; - avdd_io_edp_1v05: regulator@7 { + avdd_io_edp_1v05: regulator-avdd-io-epd-1v05 { compatible = "regulator-fixed"; regulator-name = "AVDD_IO_EDP_1V05"; @@ -1017,7 +1849,7 @@ avdd_io_edp_1v05: regulator@7 { vin-supply = <&avdd_1v05_pll>; }; - vdd_5v0_usb: regulator@8 { + vdd_5v0_usb: regulator-vdd-5v-usb { compatible = "regulator-fixed"; regulator-name = "VDD_5V_USB"; @@ -1039,6 +1871,31 @@ sound { /* Router */ <&xbar_i2s3_port>, <&xbar_i2s4_port>, <&xbar_dmic1_port>, <&xbar_dmic2_port>, + <&xbar_sfc1_in_port>, <&xbar_sfc2_in_port>, + <&xbar_sfc3_in_port>, <&xbar_sfc4_in_port>, + <&xbar_mvc1_in_port>, <&xbar_mvc2_in_port>, + <&xbar_amx1_in1_port>, <&xbar_amx1_in2_port>, + <&xbar_amx1_in3_port>, <&xbar_amx1_in4_port>, + <&xbar_amx2_in1_port>, <&xbar_amx2_in2_port>, + <&xbar_amx2_in3_port>, <&xbar_amx2_in4_port>, + <&xbar_adx1_in_port>, <&xbar_adx2_in_port>, + <&xbar_mixer_in1_port>, <&xbar_mixer_in2_port>, + <&xbar_mixer_in3_port>, <&xbar_mixer_in4_port>, + <&xbar_mixer_in5_port>, <&xbar_mixer_in6_port>, + <&xbar_mixer_in7_port>, <&xbar_mixer_in8_port>, + <&xbar_mixer_in9_port>, <&xbar_mixer_in10_port>, + /* HW accelerators */ + <&sfc1_out_port>, <&sfc2_out_port>, + <&sfc3_out_port>, <&sfc4_out_port>, + <&mvc1_out_port>, <&mvc2_out_port>, + <&amx1_out_port>, <&amx2_out_port>, + <&adx1_out1_port>, <&adx1_out2_port>, + <&adx1_out3_port>, <&adx1_out4_port>, + <&adx2_out1_port>, <&adx2_out2_port>, + <&adx2_out3_port>, <&adx2_out4_port>, + <&mixer_out1_port>, <&mixer_out2_port>, + <&mixer_out3_port>, <&mixer_out4_port>, + <&mixer_out5_port>, /* I/O DAP Ports */ <&i2s3_port>, <&i2s4_port>, <&dmic1_port>, <&dmic2_port>; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts index 131c064d69..a263d51882 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts +++ b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts @@ -1642,10 +1642,6 @@ usb@70090000 { dvddio-pex-supply = <&avddio_1v05>; hvddio-pex-supply = <&pp1800>; avdd-usb-supply = <&pp3300>; - avdd-pll-utmip-supply = <&pp1800>; - avdd-pll-uerefe-supply = <&pp1050_avdd>; - dvdd-pex-pll-supply = <&avddio_1v05>; - hvdd-pex-pll-e-supply = <&pp1800>; status = "okay"; }; @@ -1726,7 +1722,7 @@ interrupt-controller@702f9000 { }; }; - clk32k_in: clock@0 { + clk32k_in: clock-32k { compatible = "fixed-clock"; clock-frequency = <32768>; #clock-cells = <0>; @@ -1807,7 +1803,7 @@ psci { method = "smc"; }; - ppvar_sys: regulator@0 { + ppvar_sys: regulator-ppvar-sys { compatible = "regulator-fixed"; regulator-name = "PPVAR_SYS"; regulator-min-microvolt = <4400000>; @@ -1815,7 +1811,7 @@ ppvar_sys: regulator@0 { regulator-always-on; }; - pplcd_vdd: regulator@1 { + pplcd_vdd: regulator-pplcd-vdd { compatible = "regulator-fixed"; regulator-name = "PPLCD_VDD"; regulator-min-microvolt = <4400000>; @@ -1825,7 +1821,7 @@ pplcd_vdd: regulator@1 { regulator-boot-on; }; - pp3000_always: regulator@2 { + pp3000_always: regulator-pp3000-always { compatible = "regulator-fixed"; regulator-name = "PP3000_ALWAYS"; regulator-min-microvolt = <3000000>; @@ -1833,7 +1829,7 @@ pp3000_always: regulator@2 { regulator-always-on; }; - pp3300: regulator@3 { + pp3300: regulator-pp3000 { compatible = "regulator-fixed"; regulator-name = "PP3300"; regulator-min-microvolt = <3300000>; @@ -1843,7 +1839,7 @@ pp3300: regulator@3 { enable-active-high; }; - pp5000: regulator@4 { + pp5000: regulator-pp5000 { compatible = "regulator-fixed"; regulator-name = "PP5000"; regulator-min-microvolt = <5000000>; @@ -1851,7 +1847,7 @@ pp5000: regulator@4 { regulator-always-on; }; - pp1800_lcdio: regulator@5 { + pp1800_lcdio: regulator-pp1800-lcdio { compatible = "regulator-fixed"; regulator-name = "PP1800_LCDIO"; regulator-min-microvolt = <1800000>; @@ -1861,7 +1857,7 @@ pp1800_lcdio: regulator@5 { regulator-boot-on; }; - pp1800_cam: regulator@6 { + pp1800_cam: regulator-pp1800-cam { compatible = "regulator-fixed"; regulator-name = "PP1800_CAM"; regulator-min-microvolt = <1800000>; @@ -1870,7 +1866,7 @@ pp1800_cam: regulator@6 { enable-active-high; }; - usbc_vbus: regulator@7 { + usbc_vbus: regulator-usbc-vbus { compatible = "regulator-fixed"; regulator-name = "USBC_VBUS"; regulator-min-microvolt = <5000000>; diff --git a/arch/arm64/boot/dts/nvidia/tegra210.dtsi b/arch/arm64/boot/dts/nvidia/tegra210.dtsi index 26b3f98a21..218a2b3220 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra210.dtsi @@ -93,8 +93,8 @@ host1x@50000000 { interrupt-names = "syncpt", "host1x"; clocks = <&tegra_car TEGRA210_CLK_HOST1X>; clock-names = "host1x"; - resets = <&tegra_car 28>; - reset-names = "host1x"; + resets = <&tegra_car 28>, <&mc TEGRA210_MC_RESET_HC>; + reset-names = "host1x", "mc"; #address-cells = <2>; #size-cells = <2>; @@ -181,6 +181,12 @@ csi@838 { tsec@54100000 { compatible = "nvidia,tegra210-tsec"; reg = <0x0 0x54100000 0x0 0x00040000>; + interrupts = ; + clocks = <&tegra_car TEGRA210_CLK_TSEC>; + clock-names = "tsec"; + resets = <&tegra_car 83>; + reset-names = "tsec"; + status = "disabled"; }; dc@54200000 { @@ -283,6 +289,11 @@ nvenc@544c0000 { tsec@54500000 { compatible = "nvidia,tegra210-tsec"; reg = <0x0 0x54500000 0x0 0x00040000>; + interrupts = ; + clocks = <&tegra_car TEGRA210_CLK_TSECB>; + clock-names = "tsec"; + resets = <&tegra_car 206>; + reset-names = "tsec"; status = "disabled"; }; @@ -1015,8 +1026,8 @@ usb@70090000 { <&tegra_car TEGRA210_CLK_XUSB_HOST_SRC>, <&tegra_car TEGRA210_CLK_XUSB_FALCON_SRC>, <&tegra_car TEGRA210_CLK_XUSB_SS>, - <&tegra_car TEGRA210_CLK_XUSB_SS_SRC>, <&tegra_car TEGRA210_CLK_XUSB_SS_DIV2>, + <&tegra_car TEGRA210_CLK_XUSB_SS_SRC>, <&tegra_car TEGRA210_CLK_XUSB_HS_SRC>, <&tegra_car TEGRA210_CLK_XUSB_FS_SRC>, <&tegra_car TEGRA210_CLK_PLL_U_480M>, @@ -1024,7 +1035,7 @@ usb@70090000 { <&tegra_car TEGRA210_CLK_PLL_E>; clock-names = "xusb_host", "xusb_host_src", "xusb_falcon_src", "xusb_ss", - "xusb_ss_src", "xusb_ss_div2", + "xusb_ss_div2", "xusb_ss_src", "xusb_hs_src", "xusb_fs_src", "pll_u_480m", "clk_m", "pll_e"; resets = <&tegra_car 89>, <&tegra_car 156>, @@ -1641,6 +1652,83 @@ tegra_dmic3: dmic@702d4200 { status = "disabled"; }; + tegra_sfc1: sfc@702d2000 { + compatible = "nvidia,tegra210-sfc"; + reg = <0x702d2000 0x200>; + sound-name-prefix = "SFC1"; + status = "disabled"; + }; + + tegra_sfc2: sfc@702d2200 { + compatible = "nvidia,tegra210-sfc"; + reg = <0x702d2200 0x200>; + sound-name-prefix = "SFC2"; + status = "disabled"; + }; + + tegra_sfc3: sfc@702d2400 { + compatible = "nvidia,tegra210-sfc"; + reg = <0x702d2400 0x200>; + sound-name-prefix = "SFC3"; + status = "disabled"; + }; + + tegra_sfc4: sfc@702d2600 { + compatible = "nvidia,tegra210-sfc"; + reg = <0x702d2600 0x200>; + sound-name-prefix = "SFC4"; + status = "disabled"; + }; + + tegra_mvc1: mvc@702da000 { + compatible = "nvidia,tegra210-mvc"; + reg = <0x702da000 0x200>; + sound-name-prefix = "MVC1"; + status = "disabled"; + }; + + tegra_mvc2: mvc@702da200 { + compatible = "nvidia,tegra210-mvc"; + reg = <0x702da200 0x200>; + sound-name-prefix = "MVC2"; + status = "disabled"; + }; + + tegra_amx1: amx@702d3000 { + compatible = "nvidia,tegra210-amx"; + reg = <0x702d3000 0x100>; + sound-name-prefix = "AMX1"; + status = "disabled"; + }; + + tegra_amx2: amx@702d3100 { + compatible = "nvidia,tegra210-amx"; + reg = <0x702d3100 0x100>; + sound-name-prefix = "AMX2"; + status = "disabled"; + }; + + tegra_adx1: adx@702d3800 { + compatible = "nvidia,tegra210-adx"; + reg = <0x702d3800 0x100>; + sound-name-prefix = "ADX1"; + status = "disabled"; + }; + + tegra_adx2: adx@702d3900 { + compatible = "nvidia,tegra210-adx"; + reg = <0x702d3900 0x100>; + sound-name-prefix = "ADX2"; + status = "disabled"; + }; + + tegra_amixer: amixer@702dbb00 { + compatible = "nvidia,tegra210-amixer"; + reg = <0x702dbb00 0x800>; + sound-name-prefix = "MIXER1"; + status = "disabled"; + }; + ports { #address-cells = <1>; #size-cells = <0>; @@ -1743,7 +1831,7 @@ spi@70410000 { }; usb@7d000000 { - compatible = "nvidia,tegra210-ehci", "nvidia,tegra30-ehci", "usb-ehci"; + compatible = "nvidia,tegra210-ehci", "nvidia,tegra30-ehci"; reg = <0x0 0x7d000000 0x0 0x4000>; interrupts = ; phy_type = "utmi"; @@ -1781,7 +1869,7 @@ phy1: usb-phy@7d000000 { }; usb@7d004000 { - compatible = "nvidia,tegra210-ehci", "nvidia,tegra30-ehci", "usb-ehci"; + compatible = "nvidia,tegra210-ehci", "nvidia,tegra30-ehci"; reg = <0x0 0x7d004000 0x0 0x4000>; interrupts = ; phy_type = "utmi"; @@ -1904,7 +1992,7 @@ sound { }; thermal-zones { - cpu { + cpu-thermal { polling-delay-passive = <1000>; polling-delay = <0>; @@ -1933,7 +2021,7 @@ map0 { }; }; - mem { + mem-thermal { polling-delay-passive = <0>; polling-delay = <0>; @@ -1979,7 +2067,7 @@ dram-active { }; }; - gpu { + gpu-thermal { polling-delay-passive = <1000>; polling-delay = <0>; @@ -2008,7 +2096,7 @@ map0 { }; }; - pllx { + pllx-thermal { polling-delay-passive = <0>; polling-delay = <0>; diff --git a/arch/arm64/boot/dts/nvidia/tegra234-sim-vdk.dts b/arch/arm64/boot/dts/nvidia/tegra234-sim-vdk.dts index b5d9a55262..5804acfc42 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234-sim-vdk.dts +++ b/arch/arm64/boot/dts/nvidia/tegra234-sim-vdk.dts @@ -26,7 +26,6 @@ mmc@3460000 { status = "okay"; bus-width = <8>; non-removable; - only-1-8-v; }; rtc@c2a0000 { diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi index f0efb3a628..6b6f15804a 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi @@ -3,6 +3,7 @@ #include #include #include +#include #include / { @@ -25,6 +26,113 @@ misc@100000 { status = "okay"; }; + gpio: gpio@2200000 { + compatible = "nvidia,tegra234-gpio"; + reg-names = "security", "gpio"; + reg = <0x02200000 0x10000>, + <0x02210000 0x10000>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + #interrupt-cells = <2>; + interrupt-controller; + #gpio-cells = <2>; + gpio-controller; + }; + + mc: memory-controller@2c00000 { + compatible = "nvidia,tegra234-mc"; + reg = <0x02c00000 0x100000>, + <0x02b80000 0x040000>, + <0x01700000 0x100000>; + interrupts = ; + #interconnect-cells = <1>; + status = "okay"; + + #address-cells = <2>; + #size-cells = <2>; + + ranges = <0x01700000 0x0 0x01700000 0x0 0x100000>, + <0x02b80000 0x0 0x02b80000 0x0 0x040000>, + <0x02c00000 0x0 0x02c00000 0x0 0x100000>; + + /* + * Bit 39 of addresses passing through the memory + * controller selects the XBAR format used when memory + * is accessed. This is used to transparently access + * memory in the XBAR format used by the discrete GPU + * (bit 39 set) or Tegra (bit 39 clear). + * + * As a consequence, the operating system must ensure + * that bit 39 is never used implicitly, for example + * via an I/O virtual address mapping of an IOMMU. If + * devices require access to the XBAR switch, their + * drivers must set this bit explicitly. + * + * Limit the DMA range for memory clients to [38:0]. + */ + dma-ranges = <0x0 0x0 0x0 0x80 0x0>; + + emc: external-memory-controller@2c60000 { + compatible = "nvidia,tegra234-emc"; + reg = <0x0 0x02c60000 0x0 0x90000>, + <0x0 0x01780000 0x0 0x80000>; + interrupts = ; + clocks = <&bpmp TEGRA234_CLK_EMC>; + clock-names = "emc"; + status = "okay"; + + #interconnect-cells = <0>; + + nvidia,bpmp = <&bpmp>; + }; + }; + uarta: serial@3100000 { compatible = "nvidia,tegra234-uart", "nvidia,tegra20-uart"; reg = <0x03100000 0x10000>; @@ -40,11 +148,27 @@ mmc@3460000 { compatible = "nvidia,tegra234-sdhci", "nvidia,tegra186-sdhci"; reg = <0x03460000 0x20000>; interrupts = ; - clocks = <&bpmp TEGRA234_CLK_SDMMC4>; - clock-names = "sdhci"; + clocks = <&bpmp TEGRA234_CLK_SDMMC4>, + <&bpmp TEGRA234_CLK_SDMMC_LEGACY_TM>; + clock-names = "sdhci", "tmclk"; + assigned-clocks = <&bpmp TEGRA234_CLK_SDMMC4>, + <&bpmp TEGRA234_CLK_PLLC4>; + assigned-clock-parents = <&bpmp TEGRA234_CLK_PLLC4>; resets = <&bpmp TEGRA234_RESET_SDMMC4>; reset-names = "sdhci"; - dma-coherent; + interconnects = <&mc TEGRA234_MEMORY_CLIENT_SDMMCRAB &emc>, + <&mc TEGRA234_MEMORY_CLIENT_SDMMCWAB &emc>; + interconnect-names = "dma-mem", "write"; + nvidia,pad-autocal-pull-up-offset-hs400 = <0x00>; + nvidia,pad-autocal-pull-down-offset-hs400 = <0x00>; + nvidia,pad-autocal-pull-up-offset-1v8-timeout = <0x0a>; + nvidia,pad-autocal-pull-down-offset-1v8-timeout = <0x0a>; + nvidia,pad-autocal-pull-up-offset-3v3-timeout = <0x0a>; + nvidia,pad-autocal-pull-down-offset-3v3-timeout = <0x0a>; + nvidia,default-tap = <0x8>; + nvidia,default-trim = <0x14>; + nvidia,dqs-trim = <40>; + supports-cqe; status = "disabled"; }; @@ -93,9 +217,26 @@ rtc@c2a0000 { reg = <0x0c2a0000 0x10000>; interrupt-parent = <&pmc>; interrupts = <73 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&bpmp TEGRA234_CLK_CLK_32K>; + clock-names = "rtc"; status = "disabled"; }; + gpio_aon: gpio@c2f0000 { + compatible = "nvidia,tegra234-gpio-aon"; + reg-names = "security", "gpio"; + reg = <0x0c2f0000 0x1000>, + <0x0c2f1000 0x1000>; + interrupts = , + , + , + ; + #interrupt-cells = <2>; + interrupt-controller; + #gpio-cells = <2>; + gpio-controller; + }; + pmc: pmc@c360000 { compatible = "nvidia,tegra234-pmc"; reg = <0x0c360000 0x10000>, @@ -122,21 +263,21 @@ gic: interrupt-controller@f400000 { }; }; - sysram@40000000 { + sram@40000000 { compatible = "nvidia,tegra234-sysram", "mmio-sram"; - reg = <0x0 0x40000000 0x0 0x50000>; + reg = <0x0 0x40000000 0x0 0x80000>; #address-cells = <1>; #size-cells = <1>; - ranges = <0x0 0x0 0x40000000 0x50000>; + ranges = <0x0 0x0 0x40000000 0x80000>; - cpu_bpmp_tx: shmem@4e000 { - reg = <0x4e000 0x1000>; + cpu_bpmp_tx: sram@70000 { + reg = <0x70000 0x1000>; label = "cpu-bpmp-tx"; pool; }; - cpu_bpmp_rx: shmem@4f000 { - reg = <0x4f000 0x1000>; + cpu_bpmp_rx: sram@71000 { + reg = <0x71000 0x1000>; label = "cpu-bpmp-rx"; pool; }; @@ -146,10 +287,15 @@ bpmp: bpmp { compatible = "nvidia,tegra234-bpmp", "nvidia,tegra186-bpmp"; mboxes = <&hsp_top0 TEGRA_HSP_MBOX_TYPE_DB TEGRA_HSP_DB_MASTER_BPMP>; - shmem = <&cpu_bpmp_tx &cpu_bpmp_rx>; + shmem = <&cpu_bpmp_tx>, <&cpu_bpmp_rx>; #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; + interconnects = <&mc TEGRA234_MEMORY_CLIENT_BPMPR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_BPMPW &emc>, + <&mc TEGRA234_MEMORY_CLIENT_BPMPDMAR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_BPMPDMAW &emc>; + interconnect-names = "read", "write", "dma-mem", "dma-write"; bpmp_i2c: i2c { compatible = "nvidia,tegra186-bpmp-i2c"; @@ -163,12 +309,373 @@ cpus { #address-cells = <1>; #size-cells = <0>; - cpu@0 { + cpu0_0: cpu@0 { + compatible = "arm,cortex-a78"; device_type = "cpu"; - reg = <0x000>; + reg = <0x00000>; enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c0_0>; }; + + cpu0_1: cpu@100 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x00100>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c0_1>; + }; + + cpu0_2: cpu@200 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x00200>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c0_2>; + }; + + cpu0_3: cpu@300 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x00300>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c0_3>; + }; + + cpu1_0: cpu@10000 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x10000>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c1_0>; + }; + + cpu1_1: cpu@10100 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x10100>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c1_1>; + }; + + cpu1_2: cpu@10200 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x10200>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c1_2>; + }; + + cpu1_3: cpu@10300 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x10300>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c1_3>; + }; + + cpu2_0: cpu@20000 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x20000>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c2_0>; + }; + + cpu2_1: cpu@20100 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x20100>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c2_1>; + }; + + cpu2_2: cpu@20200 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x20200>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c2_2>; + }; + + cpu2_3: cpu@20300 { + compatible = "arm,cortex-a78"; + device_type = "cpu"; + reg = <0x20300>; + + enable-method = "psci"; + + i-cache-size = <65536>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <65536>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + next-level-cache = <&l2c2_3>; + }; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu0_0>; + }; + + core1 { + cpu = <&cpu0_1>; + }; + + core2 { + cpu = <&cpu0_2>; + }; + + core3 { + cpu = <&cpu0_3>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu1_0>; + }; + + core1 { + cpu = <&cpu1_1>; + }; + + core2 { + cpu = <&cpu1_2>; + }; + + core3 { + cpu = <&cpu1_3>; + }; + }; + + cluster2 { + core0 { + cpu = <&cpu2_0>; + }; + + core1 { + cpu = <&cpu2_1>; + }; + + core2 { + cpu = <&cpu2_2>; + }; + + core3 { + cpu = <&cpu2_3>; + }; + }; + }; + + l2c0_0: l2-cache00 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c0>; + }; + + l2c0_1: l2-cache01 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c0>; + }; + + l2c0_2: l2-cache02 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c0>; + }; + + l2c0_3: l2-cache03 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c0>; + }; + + l2c1_0: l2-cache10 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c1>; + }; + + l2c1_1: l2-cache11 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c1>; + }; + + l2c1_2: l2-cache12 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c1>; + }; + + l2c1_3: l2-cache13 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c1>; + }; + + l2c2_0: l2-cache20 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c2>; + }; + + l2c2_1: l2-cache21 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c2>; + }; + + l2c2_2: l2-cache22 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c2>; + }; + + l2c2_3: l2-cache23 { + cache-size = <262144>; + cache-line-size = <64>; + cache-sets = <512>; + cache-unified; + next-level-cache = <&l3c2>; + }; + + l3c0: l3-cache0 { + cache-size = <2097152>; + cache-line-size = <64>; + cache-sets = <2048>; + }; + + l3c1: l3-cache1 { + cache-size = <2097152>; + cache-line-size = <64>; + cache-sets = <2048>; + }; + + l3c2: l3-cache2 { + cache-size = <2097152>; + cache-line-size = <64>; + cache-sets = <2048>; + }; + }; + + pmu { + compatible = "arm,cortex-a78-pmu"; + interrupts = ; + status = "okay"; }; psci { @@ -177,6 +684,14 @@ psci { method = "smc"; }; + tcu: serial { + compatible = "nvidia,tegra234-tcu", "nvidia,tegra194-tcu"; + mboxes = <&hsp_top0 TEGRA_HSP_MBOX_TYPE_SM TEGRA_HSP_SM_RX(0)>, + <&hsp_aon TEGRA_HSP_MBOX_TYPE_SM TEGRA_HSP_SM_TX(1)>; + mbox-names = "rx", "tx"; + status = "disabled"; + }; + timer { compatible = "arm,armv8-timer"; interrupts = , diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile index 70516508be..f7232052d2 100644 --- a/arch/arm64/boot/dts/qcom/Makefile +++ b/arch/arm64/boot/dts/qcom/Makefile @@ -15,6 +15,8 @@ dtb-$(CONFIG_ARCH_QCOM) += msm8916-longcheer-l8910.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-mtp.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-a3u-eur.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-a5u-eur.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-j5.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-serranove.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-wingtech-wt88047.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8992-bullhead-rev-101.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8992-msft-lumia-octagon-talkman.dtb @@ -33,12 +35,18 @@ dtb-$(CONFIG_ARCH_QCOM) += msm8996-pmi8996-sony-xperia-tone-keyaki.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8996-sony-xperia-tone-dora.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8996-sony-xperia-tone-kagura.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8996-sony-xperia-tone-keyaki.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8996-xiaomi-gemini.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8996-xiaomi-scorpio.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8998-asus-novago-tp370ql.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8998-fxtec-pro1.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8998-hp-envy-x2.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8998-lenovo-miix-630.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8998-mtp.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8998-oneplus-cheeseburger.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8998-oneplus-dumpling.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8998-sony-xperia-yoshino-lilac.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8998-sony-xperia-yoshino-maple.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8998-sony-xperia-yoshino-poplar.dtb dtb-$(CONFIG_ARCH_QCOM) += qcs404-evb-1000.dtb dtb-$(CONFIG_ARCH_QCOM) += qcs404-evb-4000.dtb dtb-$(CONFIG_ARCH_QCOM) += qrb5165-rb5.dtb @@ -48,6 +56,9 @@ dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-coachz-r1.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-coachz-r1-lte.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-coachz-r3.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-coachz-r3-lte.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-homestar-r2.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-homestar-r3.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-homestar-r4.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r0.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r1.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r1-kb.dtb @@ -55,9 +66,14 @@ dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r1-lte.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r3.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r3-kb.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r3-lte.dtb -dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-limozeen.dtb -dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-limozeen-nots.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r9.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r9-kb.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-r9-lte.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-limozeen-r4.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-limozeen-r9.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-limozeen-nots-r4.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-limozeen-nots-r5.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-lazor-limozeen-nots-r9.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-pompom-r1.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-pompom-r1-lte.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-pompom-r2.dtb @@ -66,8 +82,10 @@ dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-pompom-r3.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-pompom-r3-lte.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-r1.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-r1-lte.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7280-herobrine.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7280-idp.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7280-idp2.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7280-crd.dtb dtb-$(CONFIG_ARCH_QCOM) += sdm630-sony-xperia-ganges-kirin.dtb dtb-$(CONFIG_ARCH_QCOM) += sdm630-sony-xperia-nile-discovery.dtb dtb-$(CONFIG_ARCH_QCOM) += sdm630-sony-xperia-nile-pioneer.dtb @@ -81,9 +99,14 @@ dtb-$(CONFIG_ARCH_QCOM) += sdm845-db845c.dtb dtb-$(CONFIG_ARCH_QCOM) += sdm845-mtp.dtb dtb-$(CONFIG_ARCH_QCOM) += sdm845-oneplus-enchilada.dtb dtb-$(CONFIG_ARCH_QCOM) += sdm845-oneplus-fajita.dtb +dtb-$(CONFIG_ARCH_QCOM) += sdm845-sony-xperia-tama-akari.dtb +dtb-$(CONFIG_ARCH_QCOM) += sdm845-sony-xperia-tama-akatsuki.dtb +dtb-$(CONFIG_ARCH_QCOM) += sdm845-sony-xperia-tama-apollo.dtb dtb-$(CONFIG_ARCH_QCOM) += sdm845-xiaomi-beryllium.dtb dtb-$(CONFIG_ARCH_QCOM) += sdm850-lenovo-yoga-c630.dtb dtb-$(CONFIG_ARCH_QCOM) += sm6125-sony-xperia-seine-pdx201.dtb +dtb-$(CONFIG_ARCH_QCOM) += sm6350-sony-xperia-lena-pdx213.dtb +dtb-$(CONFIG_ARCH_QCOM) += sm7225-fairphone-fp4.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8150-hdk.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8150-microsoft-surface-duo.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8150-mtp.dtb @@ -94,4 +117,8 @@ dtb-$(CONFIG_ARCH_QCOM) += sm8250-mtp.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8250-sony-xperia-edo-pdx203.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8250-sony-xperia-edo-pdx206.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8350-hdk.dtb +dtb-$(CONFIG_ARCH_QCOM) += sm8350-microsoft-surface-duo2.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8350-mtp.dtb +dtb-$(CONFIG_ARCH_QCOM) += sm8350-sony-xperia-sagami-pdx214.dtb +dtb-$(CONFIG_ARCH_QCOM) += sm8350-sony-xperia-sagami-pdx215.dtb +dtb-$(CONFIG_ARCH_QCOM) += sm8450-qrd.dtb diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc.dts b/arch/arm64/boot/dts/qcom/apq8016-sbc.dts index f3c0dbfd0a..a5320d6d30 100644 --- a/arch/arm64/boot/dts/qcom/apq8016-sbc.dts +++ b/arch/arm64/boot/dts/qcom/apq8016-sbc.dts @@ -5,9 +5,839 @@ /dts-v1/; -#include "apq8016-sbc.dtsi" +#include "msm8916-pm8916.dtsi" +#include +#include +#include +#include +#include / { model = "Qualcomm Technologies, Inc. APQ 8016 SBC"; compatible = "qcom,apq8016-sbc", "qcom,apq8016"; + + aliases { + serial0 = &blsp1_uart2; + serial1 = &blsp1_uart1; + usid0 = &pm8916_0; + i2c0 = &blsp_i2c2; + i2c1 = &blsp_i2c6; + i2c3 = &blsp_i2c4; + spi0 = &blsp_spi5; + spi1 = &blsp_spi3; + }; + + chosen { + stdout-path = "serial0"; + }; + + camera_vdddo_1v8: camera-vdddo-1v8 { + compatible = "regulator-fixed"; + regulator-name = "camera_vdddo"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + + camera_vdda_2v8: camera-vdda-2v8 { + compatible = "regulator-fixed"; + regulator-name = "camera_vdda"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-always-on; + }; + + camera_vddd_1v5: camera-vddd-1v5 { + compatible = "regulator-fixed"; + regulator-name = "camera_vddd"; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-always-on; + }; + + reserved-memory { + ramoops@bff00000 { + compatible = "ramoops"; + reg = <0x0 0xbff00000 0x0 0x100000>; + + record-size = <0x20000>; + console-size = <0x20000>; + ftrace-size = <0x20000>; + }; + }; + + usb2513 { + compatible = "smsc,usb3503"; + reset-gpios = <&pm8916_gpios 3 GPIO_ACTIVE_LOW>; + initial-mode = <1>; + }; + + usb_id: usb-id { + compatible = "linux,extcon-usb-gpio"; + id-gpio = <&msmgpio 121 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb_id_default>; + }; + + hdmi-out { + compatible = "hdmi-connector"; + type = "a"; + + port { + hdmi_con: endpoint { + remote-endpoint = <&adv7533_out>; + }; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&msm_key_volp_n_default>; + + button@0 { + label = "Volume Up"; + linux,code = ; + gpios = <&msmgpio 107 GPIO_ACTIVE_LOW>; + }; + }; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&msmgpio_leds>, + <&pm8916_gpios_leds>, + <&pm8916_mpps_leds>; + + compatible = "gpio-leds"; + + led@1 { + label = "apq8016-sbc:green:user1"; + gpios = <&msmgpio 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + led@2 { + label = "apq8016-sbc:green:user2"; + gpios = <&msmgpio 120 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + + led@3 { + label = "apq8016-sbc:green:user3"; + gpios = <&pm8916_gpios 1 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + + led@4 { + label = "apq8016-sbc:green:user4"; + gpios = <&pm8916_gpios 2 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "none"; + panic-indicator; + default-state = "off"; + }; + + led@5 { + label = "apq8016-sbc:yellow:wlan"; + gpios = <&pm8916_mpps 2 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "phy0tx"; + default-state = "off"; + }; + + led@6 { + label = "apq8016-sbc:blue:bt"; + gpios = <&pm8916_mpps 3 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "bluetooth-power"; + default-state = "off"; + }; + }; +}; + +&blsp_dma { + status = "okay"; +}; + +&blsp_i2c2 { + /* On Low speed expansion */ + status = "okay"; + label = "LS-I2C0"; +}; + +&blsp_i2c4 { + /* On High speed expansion */ + status = "okay"; + label = "HS-I2C2"; + + adv_bridge: bridge@39 { + status = "okay"; + + compatible = "adi,adv7533"; + reg = <0x39>; + + interrupt-parent = <&msmgpio>; + interrupts = <31 IRQ_TYPE_EDGE_FALLING>; + + adi,dsi-lanes = <4>; + clocks = <&rpmcc RPM_SMD_BB_CLK2>; + clock-names = "cec"; + + pd-gpios = <&msmgpio 32 GPIO_ACTIVE_HIGH>; + + avdd-supply = <&pm8916_l6>; + v1p2-supply = <&pm8916_l6>; + v3p3-supply = <&pm8916_l17>; + + pinctrl-names = "default","sleep"; + pinctrl-0 = <&adv7533_int_active &adv7533_switch_active>; + pinctrl-1 = <&adv7533_int_suspend &adv7533_switch_suspend>; + #sound-dai-cells = <1>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + adv7533_in: endpoint { + remote-endpoint = <&dsi0_out>; + }; + }; + + port@1 { + reg = <1>; + adv7533_out: endpoint { + remote-endpoint = <&hdmi_con>; + }; + }; + }; + }; +}; + +&blsp_i2c6 { + /* On Low speed expansion */ + status = "okay"; + label = "LS-I2C1"; +}; + +&blsp_spi3 { + /* On High speed expansion */ + status = "okay"; + label = "HS-SPI1"; +}; + +&blsp_spi5 { + /* On Low speed expansion */ + status = "okay"; + label = "LS-SPI0"; +}; + +&blsp1_uart1 { + status = "okay"; + label = "LS-UART0"; +}; + +&blsp1_uart2 { + status = "okay"; + label = "LS-UART1"; +}; + +&camss { + status = "okay"; + ports { + port@0 { + reg = <0>; + csiphy0_ep: endpoint { + clock-lanes = <1>; + data-lanes = <0 2>; + remote-endpoint = <&ov5640_ep>; + status = "okay"; + }; + }; + }; +}; + +&cci { + status = "okay"; +}; + +&cci_i2c0 { + camera_rear@3b { + compatible = "ovti,ov5640"; + reg = <0x3b>; + + enable-gpios = <&msmgpio 34 GPIO_ACTIVE_HIGH>; + reset-gpios = <&msmgpio 35 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_rear_default>; + + clocks = <&gcc GCC_CAMSS_MCLK0_CLK>; + clock-names = "xclk"; + clock-frequency = <23880000>; + + vdddo-supply = <&camera_vdddo_1v8>; + vdda-supply = <&camera_vdda_2v8>; + vddd-supply = <&camera_vddd_1v5>; + + /* No camera mezzanine by default */ + status = "disabled"; + + port { + ov5640_ep: endpoint { + clock-lanes = <1>; + data-lanes = <0 2>; + remote-endpoint = <&csiphy0_ep>; + }; + }; + }; +}; + +&dsi0_out { + data-lanes = <0 1 2 3>; + remote-endpoint = <&adv7533_in>; +}; + +&lpass { + status = "okay"; +}; + +&mdss { + status = "okay"; +}; + +&mpss { + status = "okay"; + + firmware-name = "qcom/apq8016/mba.mbn", "qcom/apq8016/modem.mbn"; +}; + +&pm8916_resin { + status = "okay"; + linux,code = ; +}; + +&pronto { + status = "okay"; + + firmware-name = "qcom/apq8016/wcnss.mbn"; +}; + +&sdhc_1 { + status = "okay"; + + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&sdc1_clk_on &sdc1_cmd_on &sdc1_data_on>; + pinctrl-1 = <&sdc1_clk_off &sdc1_cmd_off &sdc1_data_off>; +}; + +&sdhc_2 { + status = "okay"; + + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&sdc2_clk_on &sdc2_cmd_on &sdc2_data_on &sdc2_cd_on>; + pinctrl-1 = <&sdc2_clk_off &sdc2_cmd_off &sdc2_data_off &sdc2_cd_off>; + + cd-gpios = <&msmgpio 38 GPIO_ACTIVE_LOW>; +}; + +&sound { + status = "okay"; + + pinctrl-0 = <&cdc_pdm_lines_act &ext_sec_tlmm_lines_act &ext_mclk_tlmm_lines_act>; + pinctrl-1 = <&cdc_pdm_lines_sus &ext_sec_tlmm_lines_sus &ext_mclk_tlmm_lines_sus>; + pinctrl-names = "default", "sleep"; + qcom,model = "DB410c"; + qcom,audio-routing = + "AMIC2", "MIC BIAS Internal2", + "AMIC3", "MIC BIAS External1"; + + external-dai-link@0 { + link-name = "ADV7533"; + cpu { + sound-dai = <&lpass MI2S_QUATERNARY>; + }; + codec { + sound-dai = <&adv_bridge 0>; + }; + }; + + internal-codec-playback-dai-link@0 { + link-name = "WCD"; + cpu { + sound-dai = <&lpass MI2S_PRIMARY>; + }; + codec { + sound-dai = <&lpass_codec 0>, <&wcd_codec 0>; + }; + }; + + internal-codec-capture-dai-link@0 { + link-name = "WCD-Capture"; + cpu { + sound-dai = <&lpass MI2S_TERTIARY>; + }; + codec { + sound-dai = <&lpass_codec 1>, <&wcd_codec 1>; + }; + }; +}; + +&usb { + status = "okay"; + extcon = <&usb_id>, <&usb_id>; + + pinctrl-names = "default", "device"; + pinctrl-0 = <&usb_sw_sel_pm &usb_hub_reset_pm>; + pinctrl-1 = <&usb_sw_sel_pm_device &usb_hub_reset_pm_device>; +}; + +&usb_hs_phy { + extcon = <&usb_id>; +}; + +&wcd_codec { + clocks = <&gcc GCC_CODEC_DIGCODEC_CLK>; + clock-names = "mclk"; + qcom,mbhc-vthreshold-low = <75 150 237 450 500>; + qcom,mbhc-vthreshold-high = <75 150 237 450 500>; +}; + +&wcnss_ctrl { + firmware-name = "qcom/apq8016/WCNSS_qcom_wlan_nv_sbc.bin"; +}; + +/* Enable CoreSight */ +&cti0 { status = "okay"; }; +&cti1 { status = "okay"; }; +&cti12 { status = "okay"; }; +&cti13 { status = "okay"; }; +&cti14 { status = "okay"; }; +&cti15 { status = "okay"; }; +&debug0 { status = "okay"; }; +&debug1 { status = "okay"; }; +&debug2 { status = "okay"; }; +&debug3 { status = "okay"; }; +&etf { status = "okay"; }; +&etm0 { status = "okay"; }; +&etm1 { status = "okay"; }; +&etm2 { status = "okay"; }; +&etm3 { status = "okay"; }; +&etr { status = "okay"; }; +&funnel0 { status = "okay"; }; +&funnel1 { status = "okay"; }; +&replicator { status = "okay"; }; +&stm { status = "okay"; }; +&tpiu { status = "okay"; }; + +&smd_rpm_regulators { + vdd_l1_l2_l3-supply = <&pm8916_s3>; + vdd_l4_l5_l6-supply = <&pm8916_s4>; + vdd_l7-supply = <&pm8916_s4>; + + s3 { + regulator-min-microvolt = <375000>; + regulator-max-microvolt = <1562000>; + }; + + s4 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-always-on; + regulator-boot-on; + }; + + l1 { + regulator-min-microvolt = <375000>; + regulator-max-microvolt = <1525000>; + }; + + l2 { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + }; + + l4 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + l5 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + l6 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + l7 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + l8 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + l9 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + l10 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + l11 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + regulator-allow-set-load; + regulator-system-load = <200000>; + }; + + l12 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + l13 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + l14 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + /** + * 1.8v required on LS expansion + * for mezzanine boards + */ + l15 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + regulator-always-on; + }; + + l16 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; + + l17 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + l18 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <3337000>; + }; +}; + +/* + * 2mA drive strength is not enough when connecting multiple + * I2C devices with different pull up resistors. + */ +&i2c2_default { + drive-strength = <16>; +}; + +&i2c4_default { + drive-strength = <16>; +}; + +&i2c6_default { + drive-strength = <16>; +}; + +/* + * GPIO name legend: proper name = the GPIO line is used as GPIO + * NC = not connected (pin out but not routed from the chip to + * anything the board) + * "[PER]" = pin is muxed for [peripheral] (not GPIO) + * LSEC = Low Speed External Connector + * HSEC = High Speed External Connector + * + * Line names are taken from the schematic "DragonBoard410c" + * dated monday, august 31, 2015. Page 5 in particular. + * + * For the lines routed to the external connectors the + * lines are named after the 96Boards CE Specification 1.0, + * Appendix "Expansion Connector Signal Description". + * + * When the 96Board naming of a line and the schematic name of + * the same line are in conflict, the 96Board specification + * takes precedence, which means that the external UART on the + * LSEC is named UART0 while the schematic and SoC names this + * UART3. This is only for the informational lines i.e. "[FOO]", + * the GPIO named lines "GPIO-A" thru "GPIO-L" are the only + * ones actually used for GPIO. + */ + +&msmgpio { + gpio-line-names = + "[UART0_TX]", /* GPIO_0, LSEC pin 5 */ + "[UART0_RX]", /* GPIO_1, LSEC pin 7 */ + "[UART0_CTS_N]", /* GPIO_2, LSEC pin 3 */ + "[UART0_RTS_N]", /* GPIO_3, LSEC pin 9 */ + "[UART1_TX]", /* GPIO_4, LSEC pin 11 */ + "[UART1_RX]", /* GPIO_5, LSEC pin 13 */ + "[I2C0_SDA]", /* GPIO_8, LSEC pin 17 */ + "[I2C0_SCL]", /* GPIO_7, LSEC pin 15 */ + "[SPI1_DOUT]", /* SPI1_MOSI, HSEC pin 1 */ + "[SPI1_DIN]", /* SPI1_MISO, HSEC pin 11 */ + "[SPI1_CS]", /* SPI1_CS_N, HSEC pin 7 */ + "[SPI1_SCLK]", /* SPI1_CLK, HSEC pin 9 */ + "GPIO-B", /* LS_EXP_GPIO_B, LSEC pin 24 */ + "GPIO-C", /* LS_EXP_GPIO_C, LSEC pin 25 */ + "[I2C3_SDA]", /* HSEC pin 38 */ + "[I2C3_SCL]", /* HSEC pin 36 */ + "[SPI0_MOSI]", /* LSEC pin 14 */ + "[SPI0_MISO]", /* LSEC pin 10 */ + "[SPI0_CS_N]", /* LSEC pin 12 */ + "[SPI0_CLK]", /* LSEC pin 8 */ + "HDMI_HPD_N", /* GPIO 20 */ + "USR_LED_1_CTRL", + "[I2C1_SDA]", /* GPIO_22, LSEC pin 21 */ + "[I2C1_SCL]", /* GPIO_23, LSEC pin 19 */ + "GPIO-G", /* LS_EXP_GPIO_G, LSEC pin 29 */ + "GPIO-H", /* LS_EXP_GPIO_H, LSEC pin 30 */ + "[CSI0_MCLK]", /* HSEC pin 15 */ + "[CSI1_MCLK]", /* HSEC pin 17 */ + "GPIO-K", /* LS_EXP_GPIO_K, LSEC pin 33 */ + "[I2C2_SDA]", /* HSEC pin 34 */ + "[I2C2_SCL]", /* HSEC pin 32 */ + "DSI2HDMI_INT_N", + "DSI_SW_SEL_APQ", + "GPIO-L", /* LS_EXP_GPIO_L, LSEC pin 34 */ + "GPIO-J", /* LS_EXP_GPIO_J, LSEC pin 32 */ + "GPIO-I", /* LS_EXP_GPIO_I, LSEC pin 31 */ + "GPIO-A", /* LS_EXP_GPIO_A, LSEC pin 23 */ + "FORCED_USB_BOOT", + "SD_CARD_DET_N", + "[WCSS_BT_SSBI]", + "[WCSS_WLAN_DATA_2]", /* GPIO 40 */ + "[WCSS_WLAN_DATA_1]", + "[WCSS_WLAN_DATA_0]", + "[WCSS_WLAN_SET]", + "[WCSS_WLAN_CLK]", + "[WCSS_FM_SSBI]", + "[WCSS_FM_SDI]", + "[WCSS_BT_DAT_CTL]", + "[WCSS_BT_DAT_STB]", + "NC", + "NC", /* GPIO 50 */ + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", /* GPIO 60 */ + "NC", + "NC", + "[CDC_PDM0_CLK]", + "[CDC_PDM0_SYNC]", + "[CDC_PDM0_TX0]", + "[CDC_PDM0_RX0]", + "[CDC_PDM0_RX1]", + "[CDC_PDM0_RX2]", + "GPIO-D", /* LS_EXP_GPIO_D, LSEC pin 26 */ + "NC", /* GPIO 70 */ + "NC", + "NC", + "NC", + "NC", /* GPIO 74 */ + "NC", + "NC", + "NC", + "NC", + "NC", + "BOOT_CONFIG_0", /* GPIO 80 */ + "BOOT_CONFIG_1", + "BOOT_CONFIG_2", + "BOOT_CONFIG_3", + "NC", + "NC", + "BOOT_CONFIG_5", + "NC", + "NC", + "NC", + "NC", /* GPIO 90 */ + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", /* GPIO 100 */ + "NC", + "NC", + "NC", + "SSBI_GPS", + "NC", + "NC", + "KEY_VOLP_N", + "NC", + "NC", + "[LS_EXP_MI2S_WS]", /* GPIO 110 */ + "NC", + "NC", + "[LS_EXP_MI2S_SCK]", + "[LS_EXP_MI2S_DATA0]", + "GPIO-E", /* LS_EXP_GPIO_E, LSEC pin 27 */ + "NC", + "[DSI2HDMI_MI2S_WS]", + "[DSI2HDMI_MI2S_SCK]", + "[DSI2HDMI_MI2S_DATA0]", + "USR_LED_2_CTRL", /* GPIO 120 */ + "SB_HS_ID"; + + msmgpio_leds: msmgpio-leds { + pins = "gpio21", "gpio120"; + function = "gpio"; + + output-low; + }; + + usb_id_default: usb-id-default { + pins = "gpio121"; + function = "gpio"; + + drive-strength = <8>; + input-enable; + bias-pull-up; + }; + + adv7533_int_active: adv533-int-active { + pins = "gpio31"; + function = "gpio"; + + drive-strength = <16>; + bias-disable; + }; + + adv7533_int_suspend: adv7533-int-suspend { + pins = "gpio31"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; + + adv7533_switch_active: adv7533-switch-active { + pins = "gpio32"; + function = "gpio"; + + drive-strength = <16>; + bias-disable; + }; + + adv7533_switch_suspend: adv7533-switch-suspend { + pins = "gpio32"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; + + msm_key_volp_n_default: msm-key-volp-n-default { + pins = "gpio107"; + function = "gpio"; + + drive-strength = <8>; + input-enable; + bias-pull-up; + }; +}; + +&pm8916_gpios { + gpio-line-names = + "USR_LED_3_CTRL", + "USR_LED_4_CTRL", + "USB_HUB_RESET_N_PM", + "USB_SW_SEL_PM"; + + usb_hub_reset_pm: usb-hub-reset-pm { + pins = "gpio3"; + function = PMIC_GPIO_FUNC_NORMAL; + + input-disable; + output-high; + }; + + usb_hub_reset_pm_device: usb-hub-reset-pm-device { + pins = "gpio3"; + function = PMIC_GPIO_FUNC_NORMAL; + + output-low; + }; + + usb_sw_sel_pm: usb-sw-sel-pm { + pins = "gpio4"; + function = PMIC_GPIO_FUNC_NORMAL; + + power-source = ; + input-disable; + output-high; + }; + + usb_sw_sel_pm_device: usb-sw-sel-pm-device { + pins = "gpio4"; + function = PMIC_GPIO_FUNC_NORMAL; + + power-source = ; + input-disable; + output-low; + }; + + pm8916_gpios_leds: pm8916-gpios-leds { + pins = "gpio1", "gpio2"; + function = PMIC_GPIO_FUNC_NORMAL; + + output-low; + }; +}; + +&pm8916_mpps { + gpio-line-names = + "VDD_PX_BIAS", + "WLAN_LED_CTRL", + "BT_LED_CTRL", + "GPIO-F"; /* LS_EXP_GPIO_F, LSEC pin 28 */ + + pinctrl-names = "default"; + pinctrl-0 = <&ls_exp_gpio_f>; + + ls_exp_gpio_f: pm8916-mpp4-state { + pins = "mpp4"; + function = "digital"; + + output-low; + power-source = ; // 1.8V + }; + + pm8916_mpps_leds: pm8916-mpps-state { + pins = "mpp2", "mpp3"; + function = "digital"; + + output-low; + }; }; diff --git a/arch/arm64/boot/dts/qcom/apq8094-sony-xperia-kitakami-karin_windy.dts b/arch/arm64/boot/dts/qcom/apq8094-sony-xperia-kitakami-karin_windy.dts index a8dffc8c64..314d2dc472 100644 --- a/arch/arm64/boot/dts/qcom/apq8094-sony-xperia-kitakami-karin_windy.dts +++ b/arch/arm64/boot/dts/qcom/apq8094-sony-xperia-kitakami-karin_windy.dts @@ -11,6 +11,7 @@ / { model = "Sony Xperia Z4 Tablet (Wi-Fi)"; compatible = "sony,karin_windy", "qcom,apq8094"; + chassis-type = "tablet"; /* * This model uses the APQ variant of MSM8994 (APQ8094). diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c.dts b/arch/arm64/boot/dts/qcom/apq8096-db820c.dts index 757afa2742..f623db8451 100644 --- a/arch/arm64/boot/dts/qcom/apq8096-db820c.dts +++ b/arch/arm64/boot/dts/qcom/apq8096-db820c.dts @@ -5,9 +5,1085 @@ /dts-v1/; -#include "apq8096-db820c.dtsi" +#include "msm8996.dtsi" +#include "pm8994.dtsi" +#include "pmi8994.dtsi" +#include +#include +#include +#include +#include + +/* + * GPIO name legend: proper name = the GPIO line is used as GPIO + * NC = not connected (pin out but not routed from the chip to + * anything the board) + * "[PER]" = pin is muxed for [peripheral] (not GPIO) + * LSEC = Low Speed External Connector + * P HSEC = Primary High Speed External Connector + * S HSEC = Secondary High Speed External Connector + * J14 = Camera Connector + * TP = Test Points + * + * Line names are taken from the schematic "DragonBoard 820c", + * drawing no: LM25-P2751-1 + * + * For the lines routed to the external connectors the + * lines are named after the 96Boards CE Specification 1.0, + * Appendix "Expansion Connector Signal Description". + * + * When the 96Board naming of a line and the schematic name of + * the same line are in conflict, the 96Board specification + * takes precedence, which means that the external UART on the + * LSEC is named UART0 while the schematic and SoC names this + * UART3. This is only for the informational lines i.e. "[FOO]", + * the GPIO named lines "GPIO-A" thru "GPIO-L" are the only + * ones actually used for GPIO. + */ / { model = "Qualcomm Technologies, Inc. DB820c"; compatible = "arrow,apq8096-db820c", "qcom,apq8096-sbc", "qcom,apq8096"; + + aliases { + serial0 = &blsp2_uart2; + serial1 = &blsp2_uart3; + serial2 = &blsp1_uart2; + i2c0 = &blsp1_i2c3; + i2c1 = &blsp2_i2c1; + i2c2 = &blsp2_i2c1; + spi0 = &blsp1_spi1; + spi1 = &blsp2_spi6; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + clocks { + compatible = "simple-bus"; + divclk4: divclk4 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "divclk4"; + + pinctrl-names = "default"; + pinctrl-0 = <&divclk4_pin_a>; + }; + + div1_mclk: divclk1 { + compatible = "gpio-gate-clock"; + pinctrl-0 = <&audio_mclk>; + pinctrl-names = "default"; + clocks = <&rpmcc RPM_SMD_DIV_CLK1>; + #clock-cells = <0>; + enable-gpios = <&pm8994_gpios 15 0>; + }; + }; + + gpio_keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&volume_up_gpio>; + + button@0 { + label = "Volume Up"; + linux,code = ; + gpios = <&pm8994_gpios 2 GPIO_ACTIVE_LOW>; + }; + }; + + usb2_id: usb2-id { + compatible = "linux,extcon-usb-gpio"; + id-gpio = <&pmi8994_gpios 6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb2_vbus_det_gpio>; + }; + + usb3_id: usb3-id { + compatible = "linux,extcon-usb-gpio"; + id-gpio = <&pm8994_gpios 22 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb3_vbus_det_gpio>; + }; + + vph_pwr: vph-pwr-regulator { + compatible = "regulator-fixed"; + regulator-name = "vph_pwr"; + regulator-always-on; + regulator-boot-on; + + regulator-min-microvolt = <3700000>; + regulator-max-microvolt = <3700000>; + }; + + wlan_en: wlan-en-1-8v { + pinctrl-names = "default"; + pinctrl-0 = <&wlan_en_gpios>; + compatible = "regulator-fixed"; + regulator-name = "wlan-en-regulator"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + gpio = <&pm8994_gpios 8 0>; + + /* WLAN card specific delay */ + startup-delay-us = <70000>; + enable-active-high; + }; +}; + +&blsp1_i2c3 { + /* On Low speed expansion */ + label = "LS-I2C0"; + status = "okay"; +}; + +&blsp1_spi1 { + /* On Low speed expansion */ + label = "LS-SPI0"; + status = "okay"; +}; + +&blsp1_uart2 { + label = "BT-UART"; + status = "okay"; + + bluetooth { + compatible = "qcom,qca6174-bt"; + + /* bt_disable_n gpio */ + enable-gpios = <&pm8994_gpios 19 GPIO_ACTIVE_HIGH>; + + clocks = <&divclk4>; + }; +}; + +&adsp_pil { + status = "okay"; + firmware-name = "qcom/apq8096/adsp.mbn"; +}; + +&blsp2_i2c1 { + /* On High speed expansion */ + label = "HS-I2C2"; + status = "okay"; +}; + +&blsp2_i2c1 { + /* On Low speed expansion */ + label = "LS-I2C1"; + status = "okay"; +}; + +&blsp2_spi6 { + /* On High speed expansion */ + label = "HS-SPI1"; + status = "okay"; +}; + +&blsp2_uart2 { + label = "LS-UART1"; + status = "okay"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&blsp2_uart2_2pins_default>; + pinctrl-1 = <&blsp2_uart2_2pins_sleep>; +}; + +&blsp2_uart3 { + label = "LS-UART0"; + status = "disabled"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&blsp2_uart3_4pins_default>; + pinctrl-1 = <&blsp2_uart3_4pins_sleep>; +}; + +&camss { + vdda-supply = <&vreg_l2a_1p25>; +}; + +&gpu { + status = "okay"; +}; + +&hdmi { + status = "okay"; + + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&hdmi_hpd_active &hdmi_ddc_active>; + pinctrl-1 = <&hdmi_hpd_suspend &hdmi_ddc_suspend>; + + core-vdda-supply = <&vreg_l12a_1p8>; + core-vcc-supply = <&vreg_s4a_1p8>; +}; + +&hdmi_phy { + status = "okay"; + + vddio-supply = <&vreg_l12a_1p8>; + vcca-supply = <&vreg_l28a_0p925>; + #phy-cells = <0>; +}; + +&hsusb_phy1 { + status = "okay"; + + vdd-supply = <&vreg_l28a_0p925>; + vdda-pll-supply = <&vreg_l12a_1p8>; + vdda-phy-dpdm-supply = <&vreg_l24a_3p075>; +}; + +&hsusb_phy2 { + status = "okay"; + + vdd-supply = <&vreg_l28a_0p925>; + vdda-pll-supply = <&vreg_l12a_1p8>; + vdda-phy-dpdm-supply = <&vreg_l24a_3p075>; +}; + +&mdp { + status = "okay"; +}; + +&mdss { + status = "okay"; +}; + +&mmcc { + vdd-gfx-supply = <&vdd_gfx>; +}; + +&pm8994_resin { + status = "okay"; + linux,code = ; +}; + +&tlmm { + gpio-line-names = + "[SPI0_DOUT]", /* GPIO_0, BLSP1_SPI_MOSI, LSEC pin 14 */ + "[SPI0_DIN]", /* GPIO_1, BLSP1_SPI_MISO, LSEC pin 10 */ + "[SPI0_CS]", /* GPIO_2, BLSP1_SPI_CS_N, LSEC pin 12 */ + "[SPI0_SCLK]", /* GPIO_3, BLSP1_SPI_CLK, LSEC pin 8 */ + "[UART1_TxD]", /* GPIO_4, BLSP8_UART_TX, LSEC pin 11 */ + "[UART1_RxD]", /* GPIO_5, BLSP8_UART_RX, LSEC pin 13 */ + "[I2C1_SDA]", /* GPIO_6, BLSP8_I2C_SDA, LSEC pin 21 */ + "[I2C1_SCL]", /* GPIO_7, BLSP8_I2C_SCL, LSEC pin 19 */ + "GPIO-H", /* GPIO_8, LCD0_RESET_N, LSEC pin 30 */ + "TP93", /* GPIO_9 */ + "GPIO-G", /* GPIO_10, MDP_VSYNC_P, LSEC pin 29 */ + "[MDP_VSYNC_S]", /* GPIO_11, S HSEC pin 55 */ + "NC", /* GPIO_12 */ + "[CSI0_MCLK]", /* GPIO_13, CAM_MCLK0, P HSEC pin 15 */ + "[CAM_MCLK1]", /* GPIO_14, J14 pin 11 */ + "[CSI1_MCLK]", /* GPIO_15, CAM_MCLK2, P HSEC pin 17 */ + "TP99", /* GPIO_16 */ + "[I2C2_SDA]", /* GPIO_17, CCI_I2C_SDA0, P HSEC pin 34 */ + "[I2C2_SCL]", /* GPIO_18, CCI_I2C_SCL0, P HSEC pin 32 */ + "[CCI_I2C_SDA1]", /* GPIO_19, S HSEC pin 38 */ + "[CCI_I2C_SCL1]", /* GPIO_20, S HSEC pin 36 */ + "FLASH_STROBE_EN", /* GPIO_21, S HSEC pin 5 */ + "FLASH_STROBE_TRIG", /* GPIO_22, S HSEC pin 1 */ + "GPIO-K", /* GPIO_23, CAM2_RST_N, LSEC pin 33 */ + "GPIO-D", /* GPIO_24, LSEC pin 26 */ + "GPIO-I", /* GPIO_25, CAM0_RST_N, LSEC pin 31 */ + "GPIO-J", /* GPIO_26, CAM0_STANDBY_N, LSEC pin 32 */ + "BLSP6_I2C_SDA", /* GPIO_27 */ + "BLSP6_I2C_SCL", /* GPIO_28 */ + "GPIO-B", /* GPIO_29, TS0_RESET_N, LSEC pin 24 */ + "GPIO30", /* GPIO_30, S HSEC pin 4 */ + "HDMI_CEC", /* GPIO_31 */ + "HDMI_DDC_CLOCK", /* GPIO_32 */ + "HDMI_DDC_DATA", /* GPIO_33 */ + "HDMI_HOT_PLUG_DETECT", /* GPIO_34 */ + "PCIE0_RST_N", /* GPIO_35 */ + "PCIE0_CLKREQ_N", /* GPIO_36 */ + "PCIE0_WAKE", /* GPIO_37 */ + "SD_CARD_DET_N", /* GPIO_38 */ + "TSIF1_SYNC", /* GPIO_39, S HSEC pin 48 */ + "W_DISABLE_N", /* GPIO_40 */ + "[BLSP9_UART_TX]", /* GPIO_41 */ + "[BLSP9_UART_RX]", /* GPIO_42 */ + "[BLSP2_UART_CTS_N]", /* GPIO_43 */ + "[BLSP2_UART_RFR_N]", /* GPIO_44 */ + "[BLSP3_UART_TX]", /* GPIO_45 */ + "[BLSP3_UART_RX]", /* GPIO_46 */ + "[I2C0_SDA]", /* GPIO_47, LS_I2C0_SDA, LSEC pin 17 */ + "[I2C0_SCL]", /* GPIO_48, LS_I2C0_SCL, LSEC pin 15 */ + "[UART0_TxD]", /* GPIO_49, BLSP9_UART_TX, LSEC pin 5 */ + "[UART0_RxD]", /* GPIO_50, BLSP9_UART_RX, LSEC pin 7 */ + "[UART0_CTS]", /* GPIO_51, BLSP9_UART_CTS_N, LSEC pin 3 */ + "[UART0_RTS]", /* GPIO_52, BLSP9_UART_RFR_N, LSEC pin 9 */ + "[CODEC_INT1_N]", /* GPIO_53 */ + "[CODEC_INT2_N]", /* GPIO_54 */ + "[BLSP7_I2C_SDA]", /* GPIO_55 */ + "[BLSP7_I2C_SCL]", /* GPIO_56 */ + "MI2S_MCLK", /* GPIO_57, S HSEC pin 3 */ + "[PCM_CLK]", /* GPIO_58, QUA_MI2S_SCK, LSEC pin 18 */ + "[PCM_FS]", /* GPIO_59, QUA_MI2S_WS, LSEC pin 16 */ + "[PCM_DO]", /* GPIO_60, QUA_MI2S_DATA0, LSEC pin 20 */ + "[PCM_DI]", /* GPIO_61, QUA_MI2S_DATA1, LSEC pin 22 */ + "GPIO-E", /* GPIO_62, LSEC pin 27 */ + "TP87", /* GPIO_63 */ + "[CODEC_RST_N]", /* GPIO_64 */ + "[PCM1_CLK]", /* GPIO_65 */ + "[PCM1_SYNC]", /* GPIO_66 */ + "[PCM1_DIN]", /* GPIO_67 */ + "[PCM1_DOUT]", /* GPIO_68 */ + "AUDIO_REF_CLK", /* GPIO_69 */ + "SLIMBUS_CLK", /* GPIO_70 */ + "SLIMBUS_DATA0", /* GPIO_71 */ + "SLIMBUS_DATA1", /* GPIO_72 */ + "NC", /* GPIO_73 */ + "NC", /* GPIO_74 */ + "NC", /* GPIO_75 */ + "NC", /* GPIO_76 */ + "TP94", /* GPIO_77 */ + "NC", /* GPIO_78 */ + "TP95", /* GPIO_79 */ + "GPIO-A", /* GPIO_80, MEMS_RESET_N, LSEC pin 23 */ + "TP88", /* GPIO_81 */ + "TP89", /* GPIO_82 */ + "TP90", /* GPIO_83 */ + "TP91", /* GPIO_84 */ + "[SD_DAT0]", /* GPIO_85, BLSP12_SPI_MOSI, P HSEC pin 1 */ + "[SD_CMD]", /* GPIO_86, BLSP12_SPI_MISO, P HSEC pin 11 */ + "[SD_DAT3]", /* GPIO_87, BLSP12_SPI_CS_N, P HSEC pin 7 */ + "[SD_SCLK]", /* GPIO_88, BLSP12_SPI_CLK, P HSEC pin 9 */ + "TSIF1_CLK", /* GPIO_89, S HSEC pin 42 */ + "TSIF1_EN", /* GPIO_90, S HSEC pin 46 */ + "TSIF1_DATA", /* GPIO_91, S HSEC pin 44 */ + "NC", /* GPIO_92 */ + "TSIF2_CLK", /* GPIO_93, S HSEC pin 52 */ + "TSIF2_EN", /* GPIO_94, S HSEC pin 56 */ + "TSIF2_DATA", /* GPIO_95, S HSEC pin 54 */ + "TSIF2_SYNC", /* GPIO_96, S HSEC pin 58 */ + "NC", /* GPIO_97 */ + "CAM1_STANDBY_N", /* GPIO_98 */ + "NC", /* GPIO_99 */ + "NC", /* GPIO_100 */ + "[LCD1_RESET_N]", /* GPIO_101, S HSEC pin 51 */ + "BOOT_CONFIG1", /* GPIO_102 */ + "USB_HUB_RESET", /* GPIO_103 */ + "CAM1_RST_N", /* GPIO_104 */ + "NC", /* GPIO_105 */ + "NC", /* GPIO_106 */ + "NC", /* GPIO_107 */ + "NC", /* GPIO_108 */ + "NC", /* GPIO_109 */ + "NC", /* GPIO_110 */ + "NC", /* GPIO_111 */ + "NC", /* GPIO_112 */ + "PMI8994_BUA", /* GPIO_113 */ + "PCIE2_RST_N", /* GPIO_114 */ + "PCIE2_CLKREQ_N", /* GPIO_115 */ + "PCIE2_WAKE", /* GPIO_116 */ + "SSC_IRQ_0", /* GPIO_117 */ + "SSC_IRQ_1", /* GPIO_118 */ + "SSC_IRQ_2", /* GPIO_119 */ + "NC", /* GPIO_120 */ + "GPIO121", /* GPIO_121, S HSEC pin 2 */ + "NC", /* GPIO_122 */ + "SSC_IRQ_6", /* GPIO_123 */ + "SSC_IRQ_7", /* GPIO_124 */ + "GPIO-C", /* GPIO_125, TS_INT0, LSEC pin 25 */ + "BOOT_CONFIG5", /* GPIO_126 */ + "NC", /* GPIO_127 */ + "NC", /* GPIO_128 */ + "BOOT_CONFIG7", /* GPIO_129 */ + "PCIE1_RST_N", /* GPIO_130 */ + "PCIE1_CLKREQ_N", /* GPIO_131 */ + "PCIE1_WAKE", /* GPIO_132 */ + "GPIO-L", /* GPIO_133, CAM2_STANDBY_N, LSEC pin 34 */ + "NC", /* GPIO_134 */ + "NC", /* GPIO_135 */ + "BOOT_CONFIG8", /* GPIO_136 */ + "NC", /* GPIO_137 */ + "NC", /* GPIO_138 */ + "GPS_SSBI2", /* GPIO_139 */ + "GPS_SSBI1", /* GPIO_140 */ + "NC", /* GPIO_141 */ + "NC", /* GPIO_142 */ + "NC", /* GPIO_143 */ + "BOOT_CONFIG6", /* GPIO_144 */ + "NC", /* GPIO_145 */ + "NC", /* GPIO_146 */ + "NC", /* GPIO_147 */ + "NC", /* GPIO_148 */ + "NC"; /* GPIO_149 */ + + sdc2_cd_on: sdc2_cd_on { + mux { + pins = "gpio38"; + function = "gpio"; + }; + + config { + pins = "gpio38"; + bias-pull-up; /* pull up */ + drive-strength = <16>; /* 16 MA */ + }; + }; + + sdc2_cd_off: sdc2_cd_off { + mux { + pins = "gpio38"; + function = "gpio"; + }; + + config { + pins = "gpio38"; + bias-pull-up; /* pull up */ + drive-strength = <2>; /* 2 MA */ + }; + }; + + hdmi_hpd_active: hdmi_hpd_active { + mux { + pins = "gpio34"; + function = "hdmi_hot"; + }; + + config { + pins = "gpio34"; + bias-pull-down; + drive-strength = <16>; + }; + }; + + hdmi_hpd_suspend: hdmi_hpd_suspend { + mux { + pins = "gpio34"; + function = "hdmi_hot"; + }; + + config { + pins = "gpio34"; + bias-pull-down; + drive-strength = <2>; + }; + }; + + hdmi_ddc_active: hdmi_ddc_active { + mux { + pins = "gpio32", "gpio33"; + function = "hdmi_ddc"; + }; + + config { + pins = "gpio32", "gpio33"; + drive-strength = <2>; + bias-pull-up; + }; + }; + + hdmi_ddc_suspend: hdmi_ddc_suspend { + mux { + pins = "gpio32", "gpio33"; + function = "hdmi_ddc"; + }; + + config { + pins = "gpio32", "gpio33"; + drive-strength = <2>; + bias-pull-down; + }; + }; +}; + +&pcie0 { + status = "okay"; + perst-gpio = <&tlmm 35 GPIO_ACTIVE_LOW>; + vddpe-3v3-supply = <&wlan_en>; + vdda-supply = <&vreg_l28a_0p925>; +}; + +&pcie1 { + status = "okay"; + perst-gpio = <&tlmm 130 GPIO_ACTIVE_LOW>; + vdda-supply = <&vreg_l28a_0p925>; +}; + +&pcie2 { + status = "okay"; + perst-gpio = <&tlmm 114 GPIO_ACTIVE_LOW>; + vdda-supply = <&vreg_l28a_0p925>; +}; + +&pcie_phy { + status = "okay"; + + vdda-phy-supply = <&vreg_l28a_0p925>; + vdda-pll-supply = <&vreg_l12a_1p8>; +}; + +&pm8994_gpios { + gpio-line-names = + "NC", + "KEY_VOLP_N", + "NC", + "BL1_PWM", + "GPIO-F", /* BL0_PWM, LSEC pin 28 */ + "BL1_EN", + "NC", + "WLAN_EN", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "DIVCLK1", + "DIVCLK2", + "DIVCLK3", + "DIVCLK4", + "BT_EN", + "PMIC_SLB", + "PMIC_BUA", + "USB_VBUS_DET"; + + pinctrl-names = "default"; + pinctrl-0 = <&ls_exp_gpio_f &bt_en_gpios>; + + ls_exp_gpio_f: pm8994_gpio5 { + pinconf { + pins = "gpio5"; + output-low; + power-source = <2>; // PM8994_GPIO_S4, 1.8V + }; + }; + + bt_en_gpios: bt_en_gpios { + pinconf { + pins = "gpio19"; + function = PMIC_GPIO_FUNC_NORMAL; + output-low; + power-source = ; // 1.8V + qcom,drive-strength = ; + bias-pull-down; + }; + }; + + wlan_en_gpios: wlan_en_gpios { + pinconf { + pins = "gpio8"; + function = PMIC_GPIO_FUNC_NORMAL; + output-low; + power-source = ; // 1.8V + qcom,drive-strength = ; + bias-pull-down; + }; + }; + + audio_mclk: clk_div1 { + pinconf { + pins = "gpio15"; + function = "func1"; + power-source = ; // 1.8V + }; + }; + + volume_up_gpio: pm8996_gpio2 { + pinconf { + pins = "gpio2"; + function = "normal"; + input-enable; + drive-push-pull; + bias-pull-up; + qcom,drive-strength = ; + power-source = ; // 1.8V + }; + }; + + divclk4_pin_a: divclk4 { + pinconf { + pins = "gpio18"; + function = PMIC_GPIO_FUNC_FUNC2; + + bias-disable; + power-source = ; + }; + }; + + usb3_vbus_det_gpio: pm8996_gpio22 { + pinconf { + pins = "gpio22"; + function = PMIC_GPIO_FUNC_NORMAL; + input-enable; + bias-pull-down; + qcom,drive-strength = ; + power-source = ; // 1.8V + }; + }; +}; + +&pm8994_mpps { + gpio-line-names = + "VDDPX_BIAS", + "WIFI_LED", + "NC", + "BT_LED", + "PM_MPP05", + "PM_MPP06", + "PM_MPP07", + "NC"; +}; + +&pm8994_spmi_regulators { + qcom,saw-reg = <&saw3>; + vdd_s11-supply = <&vph_pwr>; + + s9 { + qcom,saw-slave; + }; + s10 { + qcom,saw-slave; + }; + s11 { + qcom,saw-leader; + regulator-name = "VDD_APCC"; + regulator-always-on; + regulator-min-microvolt = <980000>; + regulator-max-microvolt = <980000>; + }; +}; + +&pmi8994_gpios { + gpio-line-names = + "NC", + "SPKR_AMP_EN1", + "SPKR_AMP_EN2", + "TP61", + "NC", + "USB2_VBUS_DET", + "NC", + "NC", + "NC", + "NC"; + + usb2_vbus_det_gpio: pmi8996_gpio6 { + pinconf { + pins = "gpio6"; + function = PMIC_GPIO_FUNC_NORMAL; + input-enable; + bias-pull-down; + qcom,drive-strength = ; + power-source = ; // 1.8V + }; + }; +}; + +&pmi8994_spmi_regulators { + vdd_s2-supply = <&vph_pwr>; + + vdd_gfx: s2@1700 { + reg = <0x1700 0x100>; + regulator-name = "VDD_GFX"; + regulator-min-microvolt = <980000>; + regulator-max-microvolt = <980000>; + }; +}; + +&rpm_requests { + pm8994-regulators { + compatible = "qcom,rpm-pm8994-regulators"; + + vdd_s1-supply = <&vph_pwr>; + vdd_s2-supply = <&vph_pwr>; + vdd_s3-supply = <&vph_pwr>; + vdd_s4-supply = <&vph_pwr>; + vdd_s5-supply = <&vph_pwr>; + vdd_s6-supply = <&vph_pwr>; + vdd_s7-supply = <&vph_pwr>; + vdd_s8-supply = <&vph_pwr>; + vdd_s9-supply = <&vph_pwr>; + vdd_s10-supply = <&vph_pwr>; + vdd_s11-supply = <&vph_pwr>; + vdd_s12-supply = <&vph_pwr>; + vdd_l1-supply = <&vreg_s1b_1p025>; + vdd_l2_l26_l28-supply = <&vreg_s3a_1p3>; + vdd_l3_l11-supply = <&vreg_s3a_1p3>; + vdd_l4_l27_l31-supply = <&vreg_s3a_1p3>; + vdd_l5_l7-supply = <&vreg_s5a_2p15>; + vdd_l6_l12_l32-supply = <&vreg_s5a_2p15>; + vdd_l8_l16_l30-supply = <&vph_pwr>; + vdd_l9_l10_l18_l22-supply = <&vph_pwr_bbyp>; + vdd_l13_l19_l23_l24-supply = <&vph_pwr_bbyp>; + vdd_l14_l15-supply = <&vreg_s5a_2p15>; + vdd_l17_l29-supply = <&vph_pwr_bbyp>; + vdd_l20_l21-supply = <&vph_pwr_bbyp>; + vdd_l25-supply = <&vreg_s3a_1p3>; + vdd_lvs1_2-supply = <&vreg_s4a_1p8>; + + vreg_s3a_1p3: s3 { + regulator-name = "vreg_s3a_1p3"; + regulator-min-microvolt = <1300000>; + regulator-max-microvolt = <1300000>; + }; + + /** + * 1.8v required on LS expansion + * for mezzanine boards + */ + vreg_s4a_1p8: s4 { + regulator-name = "vreg_s4a_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + vreg_s5a_2p15: s5 { + regulator-name = "vreg_s5a_2p15"; + regulator-min-microvolt = <2150000>; + regulator-max-microvolt = <2150000>; + }; + vreg_s7a_1p0: s7 { + regulator-name = "vreg_s7a_1p0"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + }; + + vreg_l1a_1p0: l1 { + regulator-name = "vreg_l1a_1p0"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + }; + vreg_l2a_1p25: l2 { + regulator-name = "vreg_l2a_1p25"; + regulator-min-microvolt = <1250000>; + regulator-max-microvolt = <1250000>; + }; + vreg_l3a_0p875: l3 { + regulator-name = "vreg_l3a_0p875"; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <850000>; + }; + vreg_l4a_1p225: l4 { + regulator-name = "vreg_l4a_1p225"; + regulator-min-microvolt = <1225000>; + regulator-max-microvolt = <1225000>; + }; + vreg_l6a_1p2: l6 { + regulator-name = "vreg_l6a_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + }; + vreg_l8a_1p8: l8 { + regulator-name = "vreg_l8a_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + vreg_l9a_1p8: l9 { + regulator-name = "vreg_l9a_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + vreg_l10a_1p8: l10 { + regulator-name = "vreg_l10a_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + vreg_l11a_1p15: l11 { + regulator-name = "vreg_l11a_1p15"; + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1150000>; + }; + vreg_l12a_1p8: l12 { + regulator-name = "vreg_l12a_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + vreg_l13a_2p95: l13 { + regulator-name = "vreg_l13a_2p95"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <2950000>; + }; + vreg_l14a_1p8: l14 { + regulator-name = "vreg_l14a_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + vreg_l15a_1p8: l15 { + regulator-name = "vreg_l15a_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + vreg_l16a_2p7: l16 { + regulator-name = "vreg_l16a_2p7"; + regulator-min-microvolt = <2700000>; + regulator-max-microvolt = <2700000>; + }; + vreg_l17a_2p8: l17 { + regulator-name = "vreg_l17a_2p8"; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + }; + vreg_l18a_2p85: l18 { + regulator-name = "vreg_l18a_2p85"; + regulator-min-microvolt = <2700000>; + regulator-max-microvolt = <2900000>; + }; + vreg_l19a_2p8: l19 { + regulator-name = "vreg_l19a_2p8"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + }; + vreg_l20a_2p95: l20 { + regulator-name = "vreg_l20a_2p95"; + regulator-min-microvolt = <2950000>; + regulator-max-microvolt = <2950000>; + regulator-allow-set-load; + }; + vreg_l21a_2p95: l21 { + regulator-name = "vreg_l21a_2p95"; + regulator-min-microvolt = <2950000>; + regulator-max-microvolt = <2950000>; + regulator-allow-set-load; + regulator-system-load = <200000>; + }; + vreg_l22a_3p0: l22 { + regulator-name = "vreg_l22a_3p0"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + vreg_l23a_2p8: l23 { + regulator-name = "vreg_l23a_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + }; + vreg_l24a_3p075: l24 { + regulator-name = "vreg_l24a_3p075"; + regulator-min-microvolt = <3075000>; + regulator-max-microvolt = <3075000>; + }; + vreg_l25a_1p2: l25 { + regulator-name = "vreg_l25a_1p2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-allow-set-load; + }; + vreg_l26a_0p8: l27 { + regulator-name = "vreg_l26a_0p8"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + }; + vreg_l28a_0p925: l28 { + regulator-name = "vreg_l28a_0p925"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <925000>; + regulator-allow-set-load; + }; + vreg_l29a_2p8: l29 { + regulator-name = "vreg_l29a_2p8"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + }; + vreg_l30a_1p8: l30 { + regulator-name = "vreg_l30a_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + vreg_l32a_1p8: l32 { + regulator-name = "vreg_l32a_1p8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + vreg_lvs1a_1p8: lvs1 { + regulator-name = "vreg_lvs1a_1p8"; + }; + + vreg_lvs2a_1p8: lvs2 { + regulator-name = "vreg_lvs2a_1p8"; + }; + }; + + pmi8994-regulators { + compatible = "qcom,rpm-pmi8994-regulators"; + + vdd_s1-supply = <&vph_pwr>; + vdd_s2-supply = <&vph_pwr>; + vdd_s3-supply = <&vph_pwr>; + vdd_bst_byp-supply = <&vph_pwr>; + + vph_pwr_bbyp: boost-bypass { + regulator-name = "vph_pwr_bbyp"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vreg_s1b_1p025: s1 { + regulator-name = "vreg_s1b_1p025"; + regulator-min-microvolt = <1025000>; + regulator-max-microvolt = <1025000>; + }; + }; +}; + +&sdhc2 { + /* External SD card */ + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&sdc2_state_on &sdc2_cd_on>; + pinctrl-1 = <&sdc2_state_off &sdc2_cd_off>; + cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>; + vmmc-supply = <&vreg_l21a_2p95>; + vqmmc-supply = <&vreg_l13a_2p95>; + status = "okay"; +}; + +&q6asmdai { + dai@0 { + reg = <0>; + }; + + dai@1 { + reg = <1>; + }; + + dai@2 { + reg = <2>; + }; +}; + +&sound { + compatible = "qcom,apq8096-sndcard"; + model = "DB820c"; + audio-routing = "RX_BIAS", "MCLK", + "MM_DL1", "MultiMedia1 Playback", + "MM_DL2", "MultiMedia2 Playback", + "MultiMedia3 Capture", "MM_UL3"; + + mm1-dai-link { + link-name = "MultiMedia1"; + cpu { + sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA1>; + }; + }; + + mm2-dai-link { + link-name = "MultiMedia2"; + cpu { + sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA2>; + }; + }; + + mm3-dai-link { + link-name = "MultiMedia3"; + cpu { + sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA3>; + }; + }; + + hdmi-dai-link { + link-name = "HDMI"; + cpu { + sound-dai = <&q6afedai HDMI_RX>; + }; + + platform { + sound-dai = <&q6routing>; + }; + + codec { + sound-dai = <&hdmi 0>; + }; + }; + + slim-dai-link { + link-name = "SLIM Playback"; + cpu { + sound-dai = <&q6afedai SLIMBUS_6_RX>; + }; + + platform { + sound-dai = <&q6routing>; + }; + + codec { + sound-dai = <&wcd9335 6>; + }; + }; + + slimcap-dai-link { + link-name = "SLIM Capture"; + cpu { + sound-dai = <&q6afedai SLIMBUS_0_TX>; + }; + + platform { + sound-dai = <&q6routing>; + }; + + codec { + sound-dai = <&wcd9335 1>; + }; + }; +}; + +&ufsphy { + status = "okay"; + + vdda-phy-supply = <&vreg_l28a_0p925>; + vdda-pll-supply = <&vreg_l12a_1p8>; + vddp-ref-clk-supply = <&vreg_l25a_1p2>; +}; + +&ufshc { + status = "okay"; + + vcc-supply = <&vreg_l20a_2p95>; + vccq-supply = <&vreg_l25a_1p2>; + vccq2-supply = <&vreg_s4a_1p8>; + + vcc-max-microamp = <600000>; + vccq-max-microamp = <450000>; + vccq2-max-microamp = <450000>; +}; + +&usb2 { + status = "okay"; + extcon = <&usb2_id>; + + dwc3@7600000 { + extcon = <&usb2_id>; + dr_mode = "otg"; + maximum-speed = "high-speed"; + }; +}; + +&usb3 { + status = "okay"; + extcon = <&usb3_id>; + + dwc3@6a00000 { + extcon = <&usb3_id>; + dr_mode = "otg"; + }; +}; + +&usb3phy { + status = "okay"; + + vdda-phy-supply = <&vreg_l28a_0p925>; + vdda-pll-supply = <&vreg_l12a_1p8>; + +}; + +&venus { + status = "okay"; +}; + +&wcd9335 { + clock-names = "mclk", "slimbus"; + clocks = <&div1_mclk>, + <&rpmcc RPM_SMD_BB_CLK1>; + + vdd-buck-supply = <&vreg_s4a_1p8>; + vdd-buck-sido-supply = <&vreg_s4a_1p8>; + vdd-tx-supply = <&vreg_s4a_1p8>; + vdd-rx-supply = <&vreg_s4a_1p8>; + vdd-io-supply = <&vreg_s4a_1p8>; }; diff --git a/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts b/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts index a57c60070c..567b331065 100644 --- a/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts +++ b/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts @@ -395,9 +395,6 @@ &ufsphy { vdda-phy-supply = <&vreg_l28a_0p925>; vdda-pll-supply = <&vreg_l12a_1p8>; - - vdda-phy-max-microamp = <18380>; - vdda-pll-max-microamp = <9440>; }; &venus { diff --git a/arch/arm64/boot/dts/qcom/ipq6018.dtsi b/arch/arm64/boot/dts/qcom/ipq6018.dtsi index ce4c2b4a5f..66ec561565 100644 --- a/arch/arm64/boot/dts/qcom/ipq6018.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq6018.dtsi @@ -201,7 +201,6 @@ cryptobam: dma-controller@704000 { #dma-cells = <1>; qcom,ee = <1>; qcom,controlled-remotely; - qcom,config-pipe-trust-reg = <0>; }; crypto: crypto@73a000 { @@ -347,9 +346,8 @@ qpic_bam: dma-controller@7984000 { compatible = "qcom,bam-v1.7.0"; reg = <0x0 0x07984000 0x0 0x1a000>; interrupts = ; - clocks = <&gcc GCC_QPIC_CLK>, - <&gcc GCC_QPIC_AHB_CLK>; - clock-names = "iface_clk", "bam_clk"; + clocks = <&gcc GCC_QPIC_AHB_CLK>; + clock-names = "bam_clk"; #dma-cells = <1>; qcom,ee = <0>; status = "disabled"; @@ -401,7 +399,7 @@ pcie_phy: phy@84000 { reset-names = "phy", "common"; - pcie_phy0: lane@84200 { + pcie_phy0: phy@84200 { reg = <0x0 0x84200 0x0 0x16c>, /* Serdes Tx */ <0x0 0x84400 0x0 0x200>, /* Serdes Rx */ <0x0 0x84800 0x0 0x4f4>; /* PCS: Lane0, COM, PCIE */ @@ -671,6 +669,89 @@ dwc_1: usb@7000000 { }; }; + ssphy_0: ssphy@78000 { + compatible = "qcom,ipq6018-qmp-usb3-phy"; + reg = <0x0 0x78000 0x0 0x1C4>; + #address-cells = <2>; + #size-cells = <2>; + #clock-cells = <1>; + ranges; + + clocks = <&gcc GCC_USB0_AUX_CLK>, + <&gcc GCC_USB0_PHY_CFG_AHB_CLK>, <&xo>; + clock-names = "aux", "cfg_ahb", "ref"; + + resets = <&gcc GCC_USB0_PHY_BCR>, + <&gcc GCC_USB3PHY_0_PHY_BCR>; + reset-names = "phy","common"; + status = "disabled"; + + usb0_ssphy: lane@78200 { + reg = <0x0 0x00078200 0x0 0x130>, /* Tx */ + <0x0 0x00078400 0x0 0x200>, /* Rx */ + <0x0 0x00078800 0x0 0x1F8>, /* PCS */ + <0x0 0x00078600 0x0 0x044>; /* PCS misc */ + #phy-cells = <0>; + clocks = <&gcc GCC_USB0_PIPE_CLK>; + clock-names = "pipe0"; + clock-output-names = "gcc_usb0_pipe_clk_src"; + }; + }; + + qusb_phy_0: qusb@79000 { + compatible = "qcom,ipq6018-qusb2-phy"; + reg = <0x0 0x079000 0x0 0x180>; + #phy-cells = <0>; + + clocks = <&gcc GCC_USB0_PHY_CFG_AHB_CLK>, + <&xo>; + clock-names = "cfg_ahb", "ref"; + + resets = <&gcc GCC_QUSB2_0_PHY_BCR>; + status = "disabled"; + }; + + usb3: usb3@8A00000 { + compatible = "qcom,ipq6018-dwc3", "qcom,dwc3"; + reg = <0x0 0x8AF8800 0x0 0x400>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + clocks = <&gcc GCC_SYS_NOC_USB0_AXI_CLK>, + <&gcc GCC_USB0_MASTER_CLK>, + <&gcc GCC_USB0_SLEEP_CLK>, + <&gcc GCC_USB0_MOCK_UTMI_CLK>; + clock-names = "sys_noc_axi", + "master", + "sleep", + "mock_utmi"; + + assigned-clocks = <&gcc GCC_SYS_NOC_USB0_AXI_CLK>, + <&gcc GCC_USB0_MASTER_CLK>, + <&gcc GCC_USB0_MOCK_UTMI_CLK>; + assigned-clock-rates = <133330000>, + <133330000>, + <20000000>; + + resets = <&gcc GCC_USB0_BCR>; + status = "disabled"; + + dwc_0: usb@8A00000 { + compatible = "snps,dwc3"; + reg = <0x0 0x8A00000 0x0 0xcd00>; + interrupts = ; + phys = <&qusb_phy_0>, <&usb0_ssphy>; + phy-names = "usb2-phy", "usb3-phy"; + tx-fifo-resize; + snps,is-utmi-l1-suspend; + snps,hird-threshold = /bits/ 8 <0x0>; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,ref-clock-period-ns = <0x32>; + dr_mode = "host"; + }; + }; }; wcss: wcss-smp2p { diff --git a/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts b/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts index cc08dc4eb5..b5e1eaa367 100644 --- a/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts +++ b/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts @@ -24,6 +24,8 @@ memory@40000000 { device_type = "memory"; reg = <0x0 0x40000000 0x0 0x20000000>; }; + + vreg_dummy: regulator-dummy { }; }; &blsp1_i2c2 { @@ -97,10 +99,14 @@ &qusb_phy_1 { &ssphy_0 { status = "okay"; + vdda-phy-supply = <&vreg_dummy>; + vdda-pll-supply = <&vreg_dummy>; }; &ssphy_1 { status = "okay"; + vdda-phy-supply = <&vreg_dummy>; + vdda-pll-supply = <&vreg_dummy>; }; &usb_0 { diff --git a/arch/arm64/boot/dts/qcom/ipq8074.dtsi b/arch/arm64/boot/dts/qcom/ipq8074.dtsi index 97f99663c1..e6cc261201 100644 --- a/arch/arm64/boot/dts/qcom/ipq8074.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq8074.dtsi @@ -91,7 +91,6 @@ soc: soc { ssphy_1: phy@58000 { compatible = "qcom,ipq8074-qmp-usb3-phy"; reg = <0x00058000 0x1c4>; - #clock-cells = <1>; #address-cells = <1>; #size-cells = <1>; ranges; @@ -106,12 +105,13 @@ ssphy_1: phy@58000 { reset-names = "phy","common"; status = "disabled"; - usb1_ssphy: lane@58200 { + usb1_ssphy: phy@58200 { reg = <0x00058200 0x130>, /* Tx */ <0x00058400 0x200>, /* Rx */ <0x00058800 0x1f8>, /* PCS */ <0x00058600 0x044>; /* PCS misc*/ #phy-cells = <0>; + #clock-cells = <1>; clocks = <&gcc GCC_USB1_PIPE_CLK>; clock-names = "pipe0"; clock-output-names = "gcc_usb1_pipe_clk_src"; @@ -134,7 +134,6 @@ qusb_phy_1: phy@59000 { ssphy_0: phy@78000 { compatible = "qcom,ipq8074-qmp-usb3-phy"; reg = <0x00078000 0x1c4>; - #clock-cells = <1>; #address-cells = <1>; #size-cells = <1>; ranges; @@ -149,12 +148,13 @@ ssphy_0: phy@78000 { reset-names = "phy","common"; status = "disabled"; - usb0_ssphy: lane@78200 { + usb0_ssphy: phy@78200 { reg = <0x00078200 0x130>, /* Tx */ <0x00078400 0x200>, /* Rx */ <0x00078800 0x1f8>, /* PCS */ <0x00078600 0x044>; /* PCS misc*/ #phy-cells = <0>; + #clock-cells = <1>; clocks = <&gcc GCC_USB0_PIPE_CLK>; clock-names = "pipe0"; clock-output-names = "gcc_usb0_pipe_clk_src"; @@ -174,34 +174,72 @@ qusb_phy_0: phy@79000 { status = "disabled"; }; - pcie_phy0: phy@86000 { + pcie_qmp0: phy@86000 { compatible = "qcom,ipq8074-qmp-pcie-phy"; reg = <0x00086000 0x1000>; - #phy-cells = <0>; - clocks = <&gcc GCC_PCIE0_PIPE_CLK>; - clock-names = "pipe_clk"; - clock-output-names = "pcie20_phy0_pipe_clk"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + clocks = <&gcc GCC_PCIE0_AUX_CLK>, + <&gcc GCC_PCIE0_AHB_CLK>; + clock-names = "aux", "cfg_ahb"; resets = <&gcc GCC_PCIE0_PHY_BCR>, <&gcc GCC_PCIE0PHY_PHY_BCR>; reset-names = "phy", "common"; status = "disabled"; + + pcie_phy0: phy@86200 { + reg = <0x86200 0x16c>, + <0x86400 0x200>, + <0x86800 0x4f4>; + #phy-cells = <0>; + #clock-cells = <0>; + clocks = <&gcc GCC_PCIE0_PIPE_CLK>; + clock-names = "pipe0"; + clock-output-names = "pcie_0_pipe_clk"; + }; }; - pcie_phy1: phy@8e000 { + pcie_qmp1: phy@8e000 { compatible = "qcom,ipq8074-qmp-pcie-phy"; reg = <0x0008e000 0x1000>; - #phy-cells = <0>; - clocks = <&gcc GCC_PCIE1_PIPE_CLK>; - clock-names = "pipe_clk"; - clock-output-names = "pcie20_phy1_pipe_clk"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + clocks = <&gcc GCC_PCIE1_AUX_CLK>, + <&gcc GCC_PCIE1_AHB_CLK>; + clock-names = "aux", "cfg_ahb"; resets = <&gcc GCC_PCIE1_PHY_BCR>, <&gcc GCC_PCIE1PHY_PHY_BCR>; reset-names = "phy", "common"; status = "disabled"; + + pcie_phy1: phy@8e200 { + reg = <0x8e200 0x16c>, + <0x8e400 0x200>, + <0x8e800 0x4f4>; + #phy-cells = <0>; + #clock-cells = <0>; + clocks = <&gcc GCC_PCIE1_PIPE_CLK>; + clock-names = "pipe0"; + clock-output-names = "pcie_1_pipe_clk"; + }; + }; + + mdio: mdio@90000 { + compatible = "qcom,ipq4019-mdio"; + reg = <0x00090000 0x64>; + #address-cells = <1>; + #size-cells = <0>; + + clocks = <&gcc GCC_MDIO_AHB_CLK>; + clock-names = "gcc_mdio_ahb_clk"; + + status = "disabled"; }; prng: rng@e3000 { @@ -212,7 +250,7 @@ prng: rng@e3000 { status = "disabled"; }; - cryptobam: dma@704000 { + cryptobam: dma-controller@704000 { compatible = "qcom,bam-v1.7.0"; reg = <0x00704000 0x20000>; interrupts = ; @@ -293,6 +331,25 @@ gcc: gcc@1800000 { #reset-cells = <0x1>; }; + spmi_bus: spmi@200f000 { + compatible = "qcom,spmi-pmic-arb"; + reg = <0x0200f000 0x001000>, + <0x02400000 0x800000>, + <0x02c00000 0x800000>, + <0x03800000 0x200000>, + <0x0200a000 0x000700>; + reg-names = "core", "chnls", "obsrvr", "intr", "cnfg"; + interrupts = ; + interrupt-names = "periph_irq"; + qcom,ee = <0>; + qcom,channel = <0>; + #address-cells = <2>; + #size-cells = <0>; + interrupt-controller; + #interrupt-cells = <4>; + cell-index = <0>; + }; + sdhc_1: sdhci@7824900 { compatible = "qcom,sdhci-msm-v4"; reg = <0x7824900 0x500>, <0x7824000 0x800>; @@ -411,6 +468,21 @@ blsp1_i2c3: i2c@78b7000 { status = "disabled"; }; + blsp1_i2c5: i2c@78b9000 { + compatible = "qcom,i2c-qup-v2.2.1"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x78b9000 0x600>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_AHB_CLK>, + <&gcc GCC_BLSP1_QUP5_I2C_APPS_CLK>; + clock-names = "iface", "core"; + clock-frequency = <400000>; + dmas = <&blsp_dma 21>, <&blsp_dma 20>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + blsp1_i2c6: i2c@78ba000 { compatible = "qcom,i2c-qup-v2.2.1"; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/msm8916-alcatel-idol347.dts b/arch/arm64/boot/dts/qcom/msm8916-alcatel-idol347.dts index 670bd1bebd..265e539e7e 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-alcatel-idol347.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-alcatel-idol347.dts @@ -9,6 +9,7 @@ / { model = "Alcatel OneTouch Idol 3 (4.7)"; compatible = "alcatel,idol347", "qcom,msm8916"; + chassis-type = "handset"; aliases { serial0 = &blsp1_uart2; diff --git a/arch/arm64/boot/dts/qcom/msm8916-asus-z00l.dts b/arch/arm64/boot/dts/qcom/msm8916-asus-z00l.dts index cee451e593..d4d33dd358 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-asus-z00l.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-asus-z00l.dts @@ -4,10 +4,13 @@ #include "msm8916-pm8916.dtsi" #include +#include +#include / { model = "Asus Zenfone 2 Laser"; compatible = "asus,z00l", "qcom,msm8916"; + chassis-type = "handset"; aliases { serial0 = &blsp1_uart2; @@ -40,6 +43,21 @@ volume-down { }; }; + reg_sd_vmmc: regulator-sdcard-vmmc { + compatible = "regulator-fixed"; + regulator-name = "sdcard-vmmc"; + regulator-min-microvolt = <2950000>; + regulator-max-microvolt = <2950000>; + + gpio = <&msmgpio 87 GPIO_ACTIVE_HIGH>; + enable-active-high; + + startup-delay-us = <200>; + + pinctrl-names = "default"; + pinctrl-0 = <&sd_vmmc_en_default>; + }; + usb_id: usb-id { compatible = "linux,extcon-usb-gpio"; id-gpios = <&msmgpio 110 GPIO_ACTIVE_HIGH>; @@ -48,6 +66,64 @@ usb_id: usb-id { }; }; +&blsp_i2c2 { + status = "okay"; + + magnetometer@c { + compatible = "asahi-kasei,ak09911"; + reg = <0x0c>; + + vdd-supply = <&pm8916_l8>; + vid-supply = <&pm8916_l6>; + + reset-gpios = <&msmgpio 112 GPIO_ACTIVE_LOW>; + + pinctrl-names = "default"; + pinctrl-0 = <&mag_reset_default>; + }; + + imu@68 { + compatible = "invensense,mpu6515"; + reg = <0x68>; + + interrupt-parent = <&msmgpio>; + interrupts = <36 IRQ_TYPE_EDGE_RISING>; + + vdd-supply = <&pm8916_l17>; + vddio-supply = <&pm8916_l6>; + + pinctrl-names = "default"; + pinctrl-0 = <&imu_default>; + + mount-matrix = "1", "0", "0", + "0", "-1", "0", + "0", "0", "1"; + }; +}; + +&blsp_i2c5 { + status = "okay"; + + touchscreen@38 { + compatible = "edt,edt-ft5306"; + reg = <0x38>; + + interrupt-parent = <&msmgpio>; + interrupts = <13 IRQ_TYPE_EDGE_FALLING>; + + reset-gpios = <&msmgpio 12 GPIO_ACTIVE_LOW>; + + vcc-supply = <&pm8916_l11>; + iovcc-supply = <&pm8916_l6>; + + touchscreen-size-x = <720>; + touchscreen-size-y = <1280>; + + pinctrl-names = "default"; + pinctrl-0 = <&touchscreen_default>; + }; +}; + &blsp1_uart2 { status = "okay"; }; @@ -64,6 +140,16 @@ &sdhc_1 { pinctrl-1 = <&sdc1_clk_off &sdc1_cmd_off &sdc1_data_off>; }; +&sdhc_2 { + status = "okay"; + vmmc-supply = <®_sd_vmmc>; + + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&sdc2_clk_on &sdc2_cmd_on &sdc2_data_on &sdc2_cd_on>; + pinctrl-1 = <&sdc2_clk_off &sdc2_cmd_off &sdc2_data_off &sdc2_cd_off>; + cd-gpios = <&msmgpio 38 GPIO_ACTIVE_LOW>; +}; + &usb { status = "okay"; extcon = <&usb_id>, <&usb_id>; @@ -185,6 +271,46 @@ gpio_keys_default: gpio-keys-default { bias-pull-up; }; + imu_default: imu-default { + pins = "gpio36"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; + + mag_reset_default: mag-reset-default { + pins = "gpio112"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; + + sd_vmmc_en_default: sd-vmmc-en-default { + pins = "gpio87"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; + + touchscreen_default: touchscreen-default { + pins = "gpio13"; + function = "gpio"; + + drive-strength = <2>; + bias-pull-up; + + reset { + pins = "gpio12"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; + }; + usb_id_default: usb-id-default { pins = "gpio110"; function = "gpio"; diff --git a/arch/arm64/boot/dts/qcom/msm8916-huawei-g7.dts b/arch/arm64/boot/dts/qcom/msm8916-huawei-g7.dts index e0075b5741..42d93d3fba 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-huawei-g7.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-huawei-g7.dts @@ -25,6 +25,7 @@ / { model = "Huawei Ascend G7"; compatible = "huawei,g7", "qcom,msm8916"; + chassis-type = "handset"; aliases { serial0 = &blsp1_uart2; diff --git a/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8150.dts b/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8150.dts index 1e893c0b6f..852de624f5 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8150.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8150.dts @@ -11,6 +11,7 @@ / { model = "Longcheer L8150"; compatible = "longcheer,l8150", "qcom,msm8916-v1-qrd/9-v1", "qcom,msm8916"; + chassis-type = "handset"; aliases { serial0 = &blsp1_uart2; @@ -30,14 +31,6 @@ wcnss_mem: wcnss@8b600000 { }; }; - // FIXME: Use extcon device provided by charger driver when available - usb_vbus: usb-vbus { - compatible = "linux,extcon-usb-gpio"; - vbus-gpio = <&msmgpio 62 GPIO_ACTIVE_HIGH>; - pinctrl-names = "default"; - pinctrl-0 = <&usb_vbus_default>; - }; - gpio-keys { compatible = "gpio-keys"; @@ -121,9 +114,21 @@ &blsp_i2c2 { status = "okay"; accelerometer@10 { - compatible = "bosch,bmc150_accel"; + compatible = "bosch,bmc156_accel"; reg = <0x10>; + /* + * For some reason the interrupt line is usually not connected + * to the BMC156. However, there are two pads next to the chip + * that can be shorted to make it work if needed. + * + * interrupt-parent = <&msmgpio>; + * interrupts = <116 IRQ_TYPE_EDGE_RISING>; + */ + + pinctrl-names = "default"; + pinctrl-0 = <&accel_int_default>; + vdd-supply = <&pm8916_l17>; vddio-supply = <&pm8916_l6>; @@ -133,9 +138,15 @@ accelerometer@10 { }; magnetometer@12 { - compatible = "bosch,bmc150_magn"; + compatible = "bosch,bmc156_magn"; reg = <0x12>; + interrupt-parent = <&msmgpio>; + interrupts = <113 IRQ_TYPE_EDGE_RISING>; + + pinctrl-names = "default"; + pinctrl-0 = <&magn_int_default>; + vdd-supply = <&pm8916_l17>; vddio-supply = <&pm8916_l6>; }; @@ -145,7 +156,8 @@ gyroscope@68 { reg = <0x68>; interrupt-parent = <&msmgpio>; - interrupts = <23 IRQ_TYPE_EDGE_RISING>; + interrupts = <23 IRQ_TYPE_EDGE_RISING>, + <22 IRQ_TYPE_EDGE_RISING>; pinctrl-names = "default"; pinctrl-0 = <&gyro_int_default>; @@ -196,6 +208,10 @@ &pm8916_resin { linux,code = ; }; +&pm8916_usbin { + status = "okay"; +}; + &pm8916_vib { status = "okay"; }; @@ -225,11 +241,11 @@ &sdhc_2 { &usb { status = "okay"; dr_mode = "peripheral"; - extcon = <&usb_vbus>; + extcon = <&pm8916_usbin>; }; &usb_hs_phy { - extcon = <&usb_vbus>; + extcon = <&pm8916_usbin>; }; &smd_rpm_regulators { @@ -336,6 +352,14 @@ l18 { }; &msmgpio { + accel_int_default: accel-int-default { + pins = "gpio116"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; + camera_flash_default: camera-flash-default { pins = "gpio31", "gpio32"; function = "gpio"; @@ -361,7 +385,15 @@ gpio_keys_default: gpio-keys-default { }; gyro_int_default: gyro-int-default { - pins = "gpio23"; + pins = "gpio22", "gpio23"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; + + magn_int_default: magn-int-default { + pins = "gpio113"; function = "gpio"; drive-strength = <2>; @@ -375,11 +407,4 @@ tp_int_default: tp-int-default { drive-strength = <2>; bias-disable; }; - - usb_vbus_default: usb-vbus-default { - pins = "gpio62"; - function = "gpio"; - - bias-pull-up; - }; }; diff --git a/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8910.dts b/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8910.dts index 27845189ac..f9ce123471 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8910.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8910.dts @@ -10,6 +10,7 @@ / { model = "BQ Aquaris X5 (Longcheer L8910)"; compatible = "longcheer,l8910", "qcom,msm8916"; + chassis-type = "handset"; aliases { serial0 = &blsp1_uart2; diff --git a/arch/arm64/boot/dts/qcom/msm8916-mtp.dts b/arch/arm64/boot/dts/qcom/msm8916-mtp.dts index d66c155387..7c0ceb3cff 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-mtp.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-mtp.dts @@ -5,9 +5,22 @@ /dts-v1/; -#include "msm8916-mtp.dtsi" +#include "msm8916-pm8916.dtsi" / { model = "Qualcomm Technologies, Inc. MSM 8916 MTP"; compatible = "qcom,msm8916-mtp", "qcom,msm8916-mtp/1", "qcom,msm8916"; + + aliases { + serial0 = &blsp1_uart2; + usid0 = &pm8916_0; + }; + + chosen { + stdout-path = "serial0"; + }; +}; + +&blsp1_uart2 { + status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts index 6cc2eaeb1d..4ba11b020f 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts @@ -7,6 +7,7 @@ / { model = "Samsung Galaxy A3U (EUR)"; compatible = "samsung,a3u-eur", "qcom,msm8916"; + chassis-type = "handset"; reg_panel_vdd3: regulator-panel-vdd3 { compatible = "regulator-fixed"; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts index c2eff5aebf..d978c9ac17 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts @@ -7,6 +7,7 @@ / { model = "Samsung Galaxy A5U (EUR)"; compatible = "samsung,a5u-eur", "qcom,msm8916"; + chassis-type = "handset"; reg_touch_key: regulator-touch-key { compatible = "regulator-fixed"; diff --git a/arch/arm64/boot/dts/qcom/msm8916-wingtech-wt88047.dts b/arch/arm64/boot/dts/qcom/msm8916-wingtech-wt88047.dts index 4e20cc0008..69a44c6f57 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-wingtech-wt88047.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-wingtech-wt88047.dts @@ -11,6 +11,7 @@ / { model = "Xiaomi Redmi 2 (Wingtech WT88047)"; compatible = "wingtech,wt88047", "qcom,msm8916"; + chassis-type = "handset"; aliases { serial0 = &blsp1_uart2; diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi index 8b27242724..41897eb373 100644 --- a/arch/arm64/boot/dts/qcom/msm8916.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi @@ -41,9 +41,13 @@ tz-apps@86000000 { no-map; }; - smem_mem: smem_region@86300000 { + smem@86300000 { + compatible = "qcom,smem"; reg = <0x0 0x86300000 0x0 0x100000>; no-map; + + hwlocks = <&tcsr_mutex 3>; + qcom,rpm-msg-ram = <&rpm_msg_ram>; }; hypervisor@86400000 { @@ -124,6 +128,8 @@ CPU0: cpu@0 { #cooling-cells = <2>; power-domains = <&CPU_PD0>; power-domain-names = "psci"; + qcom,acc = <&cpu0_acc>; + qcom,saw = <&cpu0_saw>; }; CPU1: cpu@1 { @@ -137,6 +143,8 @@ CPU1: cpu@1 { #cooling-cells = <2>; power-domains = <&CPU_PD1>; power-domain-names = "psci"; + qcom,acc = <&cpu1_acc>; + qcom,saw = <&cpu1_saw>; }; CPU2: cpu@2 { @@ -150,6 +158,8 @@ CPU2: cpu@2 { #cooling-cells = <2>; power-domains = <&CPU_PD2>; power-domain-names = "psci"; + qcom,acc = <&cpu2_acc>; + qcom,saw = <&cpu2_saw>; }; CPU3: cpu@3 { @@ -163,6 +173,8 @@ CPU3: cpu@3 { #cooling-cells = <2>; power-domains = <&CPU_PD3>; power-domain-names = "psci"; + qcom,acc = <&cpu3_acc>; + qcom,saw = <&cpu3_saw>; }; L2_0: l2-cache { @@ -323,15 +335,6 @@ rpmpd_opp_super_turbo: opp6 { }; }; - smem { - compatible = "qcom,smem"; - - memory-region = <&smem_mem>; - qcom,rpm-msg-ram = <&rpm_msg_ram>; - - hwlocks = <&tcsr_mutex 3>; - }; - smp2p-hexagon { compatible = "qcom,smp2p"; qcom,smem = <435>, <428>; @@ -414,7 +417,7 @@ wcnss_smsm: wcnss@6 { }; }; - soc: soc { + soc: soc@0 { #address-cells = <1>; #size-cells = <1>; ranges = <0 0 0 0xffffffff>; @@ -450,6 +453,11 @@ rpm_msg_ram: sram@60000 { reg = <0x00060000 0x8000>; }; + sram@290000 { + compatible = "qcom,msm8916-rpm-stats"; + reg = <0x00290000 0x10000>; + }; + bimc: interconnect@400000 { compatible = "qcom,msm8916-bimc"; reg = <0x00400000 0x62000>; @@ -1426,7 +1434,7 @@ lpass_codec: audio-codec@771c000 { }; sdhc_1: sdhci@7824000 { - compatible = "qcom,sdhci-msm-v4"; + compatible = "qcom,msm8916-sdhci", "qcom,sdhci-msm-v4"; reg = <0x07824900 0x11c>, <0x07824000 0x800>; reg-names = "hc_mem", "core_mem"; @@ -1444,7 +1452,7 @@ sdhc_1: sdhci@7824000 { }; sdhc_2: sdhci@7864000 { - compatible = "qcom,sdhci-msm-v4"; + compatible = "qcom,msm8916-sdhci", "qcom,sdhci-msm-v4"; reg = <0x07864900 0x11c>, <0x07864000 0x800>; reg-names = "hc_mem", "core_mem"; @@ -1771,7 +1779,7 @@ smd-edge { label = "pronto"; - wcnss { + wcnss_ctrl: wcnss { compatible = "qcom,wcnss"; qcom,smd-channels = "WCNSS_CTRL"; @@ -1877,6 +1885,54 @@ frame@b028000 { status = "disabled"; }; }; + + cpu0_acc: power-manager@b088000 { + compatible = "qcom,msm8916-acc"; + reg = <0x0b088000 0x1000>; + status = "reserved"; /* Controlled by PSCI firmware */ + }; + + cpu0_saw: power-manager@b089000 { + compatible = "qcom,msm8916-saw2-v3.0-cpu", "qcom,saw2"; + reg = <0x0b089000 0x1000>; + status = "reserved"; /* Controlled by PSCI firmware */ + }; + + cpu1_acc: power-manager@b098000 { + compatible = "qcom,msm8916-acc"; + reg = <0x0b098000 0x1000>; + status = "reserved"; /* Controlled by PSCI firmware */ + }; + + cpu1_saw: power-manager@b099000 { + compatible = "qcom,msm8916-saw2-v3.0-cpu", "qcom,saw2"; + reg = <0x0b099000 0x1000>; + status = "reserved"; /* Controlled by PSCI firmware */ + }; + + cpu2_acc: power-manager@b0a8000 { + compatible = "qcom,msm8916-acc"; + reg = <0x0b0a8000 0x1000>; + status = "reserved"; /* Controlled by PSCI firmware */ + }; + + cpu2_saw: power-manager@b0a9000 { + compatible = "qcom,msm8916-saw2-v3.0-cpu", "qcom,saw2"; + reg = <0x0b0a9000 0x1000>; + status = "reserved"; /* Controlled by PSCI firmware */ + }; + + cpu3_acc: power-manager@b0b8000 { + compatible = "qcom,msm8916-acc"; + reg = <0x0b0b8000 0x1000>; + status = "reserved"; /* Controlled by PSCI firmware */ + }; + + cpu3_saw: power-manager@b0b9000 { + compatible = "qcom,msm8916-saw2-v3.0-cpu", "qcom,saw2"; + reg = <0x0b0b9000 0x1000>; + status = "reserved"; /* Controlled by PSCI firmware */ + }; }; thermal-zones { diff --git a/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts b/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts index 1ccca83292..4da6c44bf5 100644 --- a/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts +++ b/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts @@ -16,6 +16,8 @@ / { model = "LG Nexus 5X"; compatible = "lg,bullhead", "qcom,msm8992"; + chassis-type = "handset"; + /* required for bootloader to select correct board */ qcom,msm-id = <251 0>, <252 0>; qcom,board-id = <0xb64 0>; diff --git a/arch/arm64/boot/dts/qcom/msm8992-msft-lumia-octagon-talkman.dts b/arch/arm64/boot/dts/qcom/msm8992-msft-lumia-octagon-talkman.dts index 5322b9ce58..8933b53694 100644 --- a/arch/arm64/boot/dts/qcom/msm8992-msft-lumia-octagon-talkman.dts +++ b/arch/arm64/boot/dts/qcom/msm8992-msft-lumia-octagon-talkman.dts @@ -12,4 +12,5 @@ / { model = "Microsoft Lumia 950"; compatible = "microsoft,talkman", "qcom,msm8992"; + chassis-type = "handset"; }; diff --git a/arch/arm64/boot/dts/qcom/msm8994-angler-rev-101.dts b/arch/arm64/boot/dts/qcom/msm8994-angler-rev-101.dts index c096b7758a..0e3dd48f0d 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-angler-rev-101.dts +++ b/arch/arm64/boot/dts/qcom/msm8994-angler-rev-101.dts @@ -14,6 +14,7 @@ / { model = "Huawei Nexus 6P"; compatible = "huawei,angler", "qcom,msm8994"; + chassis-type = "handset"; /* required for bootloader to select correct board */ qcom,msm-id = <207 0x20000>; qcom,pmic-id = <0x10009 0x1000A 0x0 0x0>; diff --git a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon-cityman.dts b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon-cityman.dts index d0aaf5750c..c593418202 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon-cityman.dts +++ b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon-cityman.dts @@ -12,4 +12,5 @@ / { model = "Microsoft Lumia 950 XL"; compatible = "microsoft,cityman", "qcom,msm8994"; + chassis-type = "handset"; }; diff --git a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi index 3a3790a52a..cc038f9b64 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi @@ -62,7 +62,6 @@ divclk4: divclk4 { gpio-keys { compatible = "gpio-keys"; - input-name = "gpio-keys"; autorepeat; volupkey { diff --git a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-ivy.dts b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-ivy.dts index b5e90c85aa..99388b019a 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-ivy.dts +++ b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-ivy.dts @@ -10,6 +10,7 @@ / { model = "Sony Xperia Z3+/Z4"; compatible = "sony,ivy-row", "qcom,msm8994"; + chassis-type = "handset"; }; &pm8994_l3 { diff --git a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-karin.dts b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-karin.dts index a1d1a07594..71758d2cfd 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-karin.dts +++ b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-karin.dts @@ -10,6 +10,7 @@ / { model = "Sony Xperia Z4 Tablet (LTE)"; compatible = "sony,karin-row", "qcom,msm8994"; + chassis-type = "tablet"; }; &blsp2_i2c5 { diff --git a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-satsuki.dts b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-satsuki.dts index 1385956a69..69b7df0ed6 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-satsuki.dts +++ b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-satsuki.dts @@ -10,6 +10,7 @@ / { model = "Sony Xperia Z5 Premium"; compatible = "sony,satsuki-row", "qcom,msm8994"; + chassis-type = "handset"; }; &pm8994_l14 { diff --git a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-sumire.dts b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-sumire.dts index d3ba9867a3..466508cd82 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-sumire.dts +++ b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-sumire.dts @@ -10,6 +10,7 @@ / { model = "Sony Xperia Z5"; compatible = "sony,sumire-row", "qcom,msm8994"; + chassis-type = "handset"; }; /delete-node/ &pm8994_l19; diff --git a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-suzuran.dts b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-suzuran.dts index f129479bbf..76bf50191f 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-suzuran.dts +++ b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami-suzuran.dts @@ -10,6 +10,7 @@ / { model = "Sony Xperia Z5 Compact"; compatible = "sony,suzuran-row", "qcom,msm8994"; + chassis-type = "handset"; }; &pm8994_l14 { diff --git a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi index 48de66bf19..dde7ed159c 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi @@ -29,7 +29,6 @@ / { gpio_keys { compatible = "gpio-keys"; - input-name = "gpio-keys"; #address-cells = <1>; #size-cells = <0>; autorepeat; @@ -223,7 +222,7 @@ pm8994_regulators: pm8994-regulators { vdd_l17_l29-supply = <&pmi8994_bby>; vdd_l20_l21-supply = <&pmi8994_bby>; vdd_l25-supply = <&pm8994_s3>; - vdd_lvs1_lvs2-supply = <&pm8994_s4>; + vdd_lvs1_2-supply = <&pm8994_s4>; /* S1, S2, S6 and S12 are managed by RPMPD */ diff --git a/arch/arm64/boot/dts/qcom/msm8996-mtp.dts b/arch/arm64/boot/dts/qcom/msm8996-mtp.dts index 45ed594c1b..7d9fc35bc7 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-mtp.dts +++ b/arch/arm64/boot/dts/qcom/msm8996-mtp.dts @@ -5,9 +5,31 @@ /dts-v1/; -#include "msm8996-mtp.dtsi" +#include "msm8996.dtsi" / { model = "Qualcomm Technologies, Inc. MSM 8996 MTP"; compatible = "qcom,msm8996-mtp"; + + aliases { + serial0 = &blsp2_uart2; + }; + + chosen { + stdout-path = "serial0"; + }; + + soc { + serial@75b0000 { + status = "okay"; + }; + }; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_phy { + status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-dora.dts b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-dora.dts index b4cca54dcb..4a0645db0a 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-dora.dts +++ b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-dora.dts @@ -12,6 +12,7 @@ / { model = "Sony Xperia X Performance"; compatible = "sony,dora-row", "qcom,msm8996"; + chassis-type = "handset"; }; /delete-node/ &tof_sensor; diff --git a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-kagura.dts b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-kagura.dts index be6ea855fc..d3100ddf92 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-kagura.dts +++ b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-kagura.dts @@ -12,4 +12,5 @@ / { model = "Sony Xperia XZ"; compatible = "sony,kagura-row", "qcom,msm8996"; + chassis-type = "handset"; }; diff --git a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-keyaki.dts b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-keyaki.dts index 1eee7d0fc1..3e5bdaba83 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-keyaki.dts +++ b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone-keyaki.dts @@ -12,6 +12,7 @@ / { model = "Sony Xperia XZs"; compatible = "sony,keyaki-row", "qcom,msm8996"; + chassis-type = "handset"; }; &pm8994_l19 { diff --git a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi index 507396c4d2..3bb50cecd6 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi @@ -184,6 +184,7 @@ &camera2_rst { &hsusb_phy1 { status = "okay"; + vdd-supply = <&pm8994_l28>; vdda-pll-supply = <&pm8994_l12>; vdda-phy-dpdm-supply = <&pm8994_l24>; }; @@ -589,9 +590,27 @@ pmi-gpio10-nc { }; }; -&pmi8994_spmi_regulators { +&pm8994_spmi_regulators { qcom,saw-reg = <&saw3>; + pm8994_s9: s9 { + qcom,saw-slave; + }; + + pm8994_s10: s10 { + qcom,saw-slave; + }; + + pm8994_s11: s11 { + qcom,saw-leader; + regulator-name = "vdd_apcc"; + regulator-always-on; + regulator-min-microvolt = <470000>; + regulator-max-microvolt = <1140000>; + }; +}; + +&pmi8994_spmi_regulators { vdd_gfx: pmi8994_s2: s2 { /* Pinned to a high value for now to avoid random crashes. */ @@ -600,26 +619,12 @@ pmi8994_s2: s2 { regulator-name = "vdd_gfx"; regulator-always-on; }; - - pmi8994_s9: s9 { - qcom,saw-slave; - }; - - pmi8994_s10: s10 { - qcom,saw-slave; - }; - - pmi8994_s11: s11 { - qcom,saw-leader; - regulator-always-on; - regulator-min-microvolt = <470000>; - regulator-max-microvolt = <1140000>; - }; }; &pmi8994_wled { status = "okay"; default-brightness = <512>; + qcom,num-strings = <3>; }; &rpm_requests { diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi index 6077c36019..91bc974aeb 100644 --- a/arch/arm64/boot/dts/qcom/msm8996.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi @@ -582,7 +582,6 @@ soc: soc { pcie_phy: phy@34000 { compatible = "qcom,msm8996-qmp-pcie-phy"; reg = <0x00034000 0x488>; - #clock-cells = <1>; #address-cells = <1>; #size-cells = <1>; ranges; @@ -598,12 +597,13 @@ pcie_phy: phy@34000 { reset-names = "phy", "common", "cfg"; status = "disabled"; - pciephy_0: lane@35000 { + pciephy_0: phy@35000 { reg = <0x00035000 0x130>, <0x00035200 0x200>, <0x00035400 0x1dc>; #phy-cells = <0>; + #clock-cells = <1>; clock-output-names = "pcie_0_pipe_clk_src"; clocks = <&gcc GCC_PCIE_0_PIPE_CLK>; clock-names = "pipe0"; @@ -611,7 +611,7 @@ pciephy_0: lane@35000 { reset-names = "lane0"; }; - pciephy_1: lane@36000 { + pciephy_1: phy@36000 { reg = <0x00036000 0x130>, <0x00036200 0x200>, <0x00036400 0x1dc>; @@ -624,7 +624,7 @@ pciephy_1: lane@36000 { reset-names = "lane1"; }; - pciephy_2: lane@37000 { + pciephy_2: phy@37000 { reg = <0x00037000 0x130>, <0x00037200 0x200>, <0x00037400 0x1dc>; @@ -705,6 +705,28 @@ tsens1: thermal-sensor@4ad000 { #thermal-sensor-cells = <1>; }; + cryptobam: dma@644000 { + compatible = "qcom,bam-v1.7.0"; + reg = <0x00644000 0x24000>; + interrupts = ; + clocks = <&gcc GCC_CE1_CLK>; + clock-names = "bam_clk"; + #dma-cells = <1>; + qcom,ee = <0>; + qcom,controlled-remotely = <1>; + }; + + crypto: crypto@67a000 { + compatible = "qcom,crypto-v5.4"; + reg = <0x0067a000 0x6000>; + clocks = <&gcc GCC_CE1_AHB_CLK>, + <&gcc GCC_CE1_AXI_CLK>, + <&gcc GCC_CE1_CLK>; + clock-names = "iface", "bus", "core"; + dmas = <&cryptobam 6>, <&cryptobam 7>; + dma-names = "rx", "tx"; + }; + tcsr_mutex_regs: syscon@740000 { compatible = "syscon"; reg = <0x00740000 0x40000>; @@ -940,7 +962,6 @@ hdmi_phy: hdmi-phy@9a0600 { gpu: gpu@b00000 { compatible = "qcom,adreno-530.2", "qcom,adreno"; - #stream-id-cells = <16>; reg = <0x00b00000 0x3f000>; reg-names = "kgsl_3d0_reg_memory"; @@ -1208,6 +1229,20 @@ wake { }; }; + blsp1_uart2_default: blsp1-uart2-default { + pins = "gpio41", "gpio42", "gpio43", "gpio44"; + function = "blsp_uart2"; + drive-strength = <16>; + bias-disable; + }; + + blsp1_uart2_sleep: blsp1-uart2-sleep { + pins = "gpio41", "gpio42", "gpio43", "gpio44"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + blsp1_i2c3_default: blsp1-i2c2-default { pins = "gpio47", "gpio48"; function = "blsp_i2c3"; @@ -1236,6 +1271,20 @@ blsp2_uart3_4pins_sleep: blsp2-uart2-4pins-sleep { bias-disable; }; + blsp2_i2c3_default: blsp2-i2c3 { + pins = "gpio51", "gpio52"; + function = "blsp_i2c9"; + drive-strength = <16>; + bias-disable; + }; + + blsp2_i2c3_sleep: blsp2-i2c3-sleep { + pins = "gpio51", "gpio52"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + }; + wcd_intr_default: wcd-intr-default{ pins = "gpio54"; function = "gpio"; @@ -1492,6 +1541,11 @@ data { }; }; + sram@290000 { + compatible = "qcom,rpm-stats"; + reg = <0x00290000 0x10000>; + }; + spmi_bus: qcom,spmi@400f000 { compatible = "qcom,spmi-pmic-arb"; reg = <0x0400f000 0x1000>, @@ -1743,7 +1797,7 @@ ufsphy: phy@627000 { reset-names = "ufsphy"; status = "disabled"; - ufsphy_lane: lanes@627400 { + ufsphy_lane: phy@627400 { reg = <0x627400 0x12c>, <0x627600 0x200>, <0x627c00 0x1b4>; @@ -2583,7 +2637,6 @@ usb3_dwc3: dwc3@6a00000 { usb3phy: phy@7410000 { compatible = "qcom,msm8996-qmp-usb3-phy"; reg = <0x07410000 0x1c4>; - #clock-cells = <1>; #address-cells = <1>; #size-cells = <1>; ranges; @@ -2598,12 +2651,13 @@ usb3phy: phy@7410000 { reset-names = "phy", "common"; status = "disabled"; - ssusb_phy_0: lane@7410200 { + ssusb_phy_0: phy@7410200 { reg = <0x07410200 0x200>, <0x07410400 0x130>, <0x07410600 0x1a8>; #phy-cells = <0>; + #clock-cells = <1>; clock-output-names = "usb3_phy_pipe_clk_src"; clocks = <&gcc GCC_USB3_PHY_PIPE_CLK>; clock-names = "pipe0"; @@ -2683,7 +2737,7 @@ sdhc2: sdhci@74a4900 { status = "disabled"; }; - blsp1_dma: dma@7544000 { + blsp1_dma: dma-controller@7544000 { compatible = "qcom,bam-v1.7.0"; reg = <0x07544000 0x2b000>; interrupts = ; @@ -2701,6 +2755,9 @@ blsp1_uart2: serial@7570000 { clocks = <&gcc GCC_BLSP1_UART2_APPS_CLK>, <&gcc GCC_BLSP1_AHB_CLK>; clock-names = "core", "iface"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&blsp1_uart2_default>; + pinctrl-1 = <&blsp1_uart2_sleep>; dmas = <&blsp1_dma 2>, <&blsp1_dma 3>; dma-names = "tx", "rx"; status = "disabled"; @@ -2740,7 +2797,7 @@ blsp1_i2c3: i2c@7577000 { status = "disabled"; }; - blsp2_dma: dma@7584000 { + blsp2_dma: dma-controller@7584000 { compatible = "qcom,bam-v1.7.0"; reg = <0x07584000 0x2b000>; interrupts = ; @@ -2805,6 +2862,24 @@ blsp2_i2c2: i2c@75b6000 { status = "disabled"; }; + blsp2_i2c3: i2c@75b7000 { + compatible = "qcom,i2c-qup-v2.2.1"; + reg = <0x075b7000 0x1000>; + interrupts = ; + clocks = <&gcc GCC_BLSP2_AHB_CLK>, + <&gcc GCC_BLSP2_QUP3_I2C_APPS_CLK>; + clock-names = "iface", "core"; + clock-frequency = <400000>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&blsp2_i2c3_default>; + pinctrl-1 = <&blsp2_i2c3_sleep>; + dmas = <&blsp2_dma 16>, <&blsp2_dma 17>; + dma-names = "tx", "rx"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + blsp2_i2c5: i2c@75b9000 { compatible = "qcom,i2c-qup-v2.2.1"; reg = <0x75b9000 0x1000>; diff --git a/arch/arm64/boot/dts/qcom/msm8998-asus-novago-tp370ql.dts b/arch/arm64/boot/dts/qcom/msm8998-asus-novago-tp370ql.dts index db5821be1e..e204b70504 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-asus-novago-tp370ql.dts +++ b/arch/arm64/boot/dts/qcom/msm8998-asus-novago-tp370ql.dts @@ -8,6 +8,7 @@ / { model = "Asus NovaGo TP370QL"; compatible = "asus,novago-tp370ql", "qcom,msm8998"; + chassis-type = "convertible"; }; &blsp1_i2c6 { diff --git a/arch/arm64/boot/dts/qcom/msm8998-clamshell.dtsi b/arch/arm64/boot/dts/qcom/msm8998-clamshell.dtsi index 125d7923d7..b3b352530d 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-clamshell.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998-clamshell.dtsi @@ -102,6 +102,7 @@ pm8005_s1: s1 { /* VDD_GFX supply */ &qusb2phy { status = "okay"; + vdd-supply = <&vreg_l1a_0p875>; vdda-pll-supply = <&vreg_l12a_1p8>; vdda-phy-dpdm-supply = <&vreg_l24a_3p075>; }; @@ -313,6 +314,8 @@ &ufshc { &ufsphy { status = "okay"; + vdda-phy-supply = <&vreg_l1a_0p875>; + vdda-pll-supply = <&vreg_l2a_1p2>; }; &usb3 { diff --git a/arch/arm64/boot/dts/qcom/msm8998-hp-envy-x2.dts b/arch/arm64/boot/dts/qcom/msm8998-hp-envy-x2.dts index 2407312709..1eb406b43f 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-hp-envy-x2.dts +++ b/arch/arm64/boot/dts/qcom/msm8998-hp-envy-x2.dts @@ -8,6 +8,7 @@ / { model = "HP Envy x2"; compatible = "hp,envy-x2", "qcom,msm8998"; + chassis-type = "convertible"; }; &blsp1_i2c6 { diff --git a/arch/arm64/boot/dts/qcom/msm8998-lenovo-miix-630.dts b/arch/arm64/boot/dts/qcom/msm8998-lenovo-miix-630.dts index 89492ed519..f55f6f3e3e 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-lenovo-miix-630.dts +++ b/arch/arm64/boot/dts/qcom/msm8998-lenovo-miix-630.dts @@ -8,6 +8,7 @@ / { model = "Lenovo Miix 630"; compatible = "lenovo,miix-630", "qcom,msm8998"; + chassis-type = "convertible"; }; &blsp1_i2c6 { diff --git a/arch/arm64/boot/dts/qcom/msm8998-mtp.dtsi b/arch/arm64/boot/dts/qcom/msm8998-mtp.dtsi index a1d15eab85..af67c641df 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-mtp.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998-mtp.dtsi @@ -371,10 +371,6 @@ &ufsphy { vdda-phy-supply = <&vreg_l1a_0p875>; vdda-pll-supply = <&vreg_l2a_1p2>; vddp-ref-clk-supply = <&vreg_l26a_1p2>; - vdda-phy-max-microamp = <51400>; - vdda-pll-max-microamp = <14600>; - vddp-ref-clk-max-microamp = <100>; - vddp-ref-clk-always-on; }; &usb3 { diff --git a/arch/arm64/boot/dts/qcom/msm8998-oneplus-cheeseburger.dts b/arch/arm64/boot/dts/qcom/msm8998-oneplus-cheeseburger.dts index 66b9297588..9563eb62db 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-oneplus-cheeseburger.dts +++ b/arch/arm64/boot/dts/qcom/msm8998-oneplus-cheeseburger.dts @@ -11,6 +11,7 @@ / { model = "OnePlus 5"; compatible = "oneplus,cheeseburger", "qcom,msm8998"; + chassis-type = "handset"; /* Required for bootloader to select correct board */ qcom,board-id = <8 0 16859 23>; diff --git a/arch/arm64/boot/dts/qcom/msm8998-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/msm8998-oneplus-common.dtsi index 0f5c7828a9..9823d48a91 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-oneplus-common.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998-oneplus-common.dtsi @@ -260,6 +260,7 @@ pinconf { &qusb2phy { status = "okay"; + vdd-supply = <&vreg_l1a_0p875>; vdda-pll-supply = <&vreg_l12a_1p8>; vdda-phy-dpdm-supply = <&vreg_l24a_3p075>; }; @@ -480,10 +481,6 @@ &ufsphy { vdda-phy-supply = <&vreg_l1a_0p875>; vdda-pll-supply = <&vreg_l2a_1p2>; vddp-ref-clk-supply = <&vreg_l26a_1p2>; - vdda-phy-max-microamp = <51400>; - vdda-pll-max-microamp = <14600>; - vddp-ref-clk-max-microamp = <100>; - vddp-ref-clk-always-on; }; &usb3 { diff --git a/arch/arm64/boot/dts/qcom/msm8998-oneplus-dumpling.dts b/arch/arm64/boot/dts/qcom/msm8998-oneplus-dumpling.dts index 544b9b0ae4..5d0dabbaee 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-oneplus-dumpling.dts +++ b/arch/arm64/boot/dts/qcom/msm8998-oneplus-dumpling.dts @@ -10,6 +10,7 @@ / { model = "OnePlus 5T"; compatible = "oneplus,dumpling", "qcom,msm8998"; + chassis-type = "handset"; /* Required for bootloader to select correct board */ qcom,board-id = <8 0 17801 43>; }; diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi index 228339f81c..f273bc1ff6 100644 --- a/arch/arm64/boot/dts/qcom/msm8998.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -117,7 +118,7 @@ xo: xo-board { clock-output-names = "xo_board"; }; - sleep_clk { + sleep_clk: sleep-clk { compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <32764>; @@ -859,6 +860,9 @@ gcc: clock-controller@100000 { #reset-cells = <1>; #power-domain-cells = <1>; reg = <0x00100000 0xb0000>; + + clock-names = "xo", "sleep_clk"; + clocks = <&xo>, <&sleep_clk>; }; rpm_msg_ram: sram@778000 { @@ -866,14 +870,14 @@ rpm_msg_ram: sram@778000 { reg = <0x00778000 0x7000>; }; - qfprom: qfprom@780000 { + qfprom: qfprom@784000 { compatible = "qcom,qfprom"; - reg = <0x00780000 0x621c>; + reg = <0x00784000 0x621c>; #address-cells = <1>; #size-cells = <1>; - qusb2_hstx_trim: hstx-trim@423a { - reg = <0x423a 0x1>; + qusb2_hstx_trim: hstx-trim@23a { + reg = <0x23a 0x1>; bits = <0 4>; }; }; @@ -958,10 +962,10 @@ pcie0: pci@1c00000 { interrupts = ; interrupt-names = "msi"; interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc 0 135 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 2 &intc 0 136 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 3 &intc 0 138 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 4 &intc 0 139 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map = <0 0 0 1 &intc 0 0 135 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc 0 0 136 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc 0 0 138 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc 0 0 139 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_PCIE_0_PIPE_CLK>, <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, @@ -994,7 +998,7 @@ pcie_phy: phy@1c06000 { vdda-phy-supply = <&vreg_l1a_0p875>; vdda-pll-supply = <&vreg_l2a_1p2>; - pciephy: lane@1c06800 { + pciephy: phy@1c06800 { reg = <0x01c06200 0x128>, <0x01c06400 0x1fc>, <0x01c06800 0x20c>; #phy-cells = <0>; @@ -1066,7 +1070,7 @@ ufsphy: phy@1da7000 { reset-names = "ufsphy"; resets = <&ufshc 0>; - ufsphy_lanes: lanes@1da7400 { + ufsphy_lanes: phy@1da7400 { reg = <0x01da7400 0x128>, <0x01da7600 0x1fc>, <0x01da7c00 0x1dc>, @@ -1420,6 +1424,102 @@ glink-edge { }; }; + adreno_gpu: gpu@5000000 { + compatible = "qcom,adreno-540.1", "qcom,adreno"; + reg = <0x05000000 0x40000>; + reg-names = "kgsl_3d0_reg_memory"; + + clocks = <&gcc GCC_GPU_CFG_AHB_CLK>, + <&gpucc RBBMTIMER_CLK>, + <&gcc GCC_BIMC_GFX_CLK>, + <&gcc GCC_GPU_BIMC_GFX_CLK>, + <&gpucc RBCPR_CLK>, + <&gpucc GFX3D_CLK>; + clock-names = "iface", + "rbbmtimer", + "mem", + "mem_iface", + "rbcpr", + "core"; + + interrupts = <0 300 IRQ_TYPE_LEVEL_HIGH>; + iommus = <&adreno_smmu 0>; + operating-points-v2 = <&gpu_opp_table>; + power-domains = <&rpmpd MSM8998_VDDMX>; + status = "disabled"; + + gpu_opp_table: opp-table { + compatible = "operating-points-v2"; + opp-710000097 { + opp-hz = /bits/ 64 <710000097>; + opp-level = ; + opp-supported-hw = <0xFF>; + }; + + opp-670000048 { + opp-hz = /bits/ 64 <670000048>; + opp-level = ; + opp-supported-hw = <0xFF>; + }; + + opp-596000097 { + opp-hz = /bits/ 64 <596000097>; + opp-level = ; + opp-supported-hw = <0xFF>; + }; + + opp-515000097 { + opp-hz = /bits/ 64 <515000097>; + opp-level = ; + opp-supported-hw = <0xFF>; + }; + + opp-414000000 { + opp-hz = /bits/ 64 <414000000>; + opp-level = ; + opp-supported-hw = <0xFF>; + }; + + opp-342000000 { + opp-hz = /bits/ 64 <342000000>; + opp-level = ; + opp-supported-hw = <0xFF>; + }; + + opp-257000000 { + opp-hz = /bits/ 64 <257000000>; + opp-level = ; + opp-supported-hw = <0xFF>; + }; + }; + }; + + adreno_smmu: iommu@5040000 { + compatible = "qcom,msm8998-smmu-v2", "qcom,smmu-v2"; + reg = <0x05040000 0x10000>; + clocks = <&gcc GCC_GPU_CFG_AHB_CLK>, + <&gcc GCC_BIMC_GFX_CLK>, + <&gcc GCC_GPU_BIMC_GFX_CLK>; + clock-names = "iface", "mem", "mem_iface"; + + #global-interrupts = <0>; + #iommu-cells = <1>; + interrupts = + , + , + ; + /* + * GPU-GX GDSC's parent is GPU-CX. We need to bring up the + * GPU-CX for SMMU but we need both of them up for Adreno. + * Contemporarily, we also need to manage the VDDMX rpmpd + * domain in the Adreno driver. + * Enable GPU CX/GX GDSCs here so that we can manage the + * SoC VDDMX RPM Power Domain in the Adreno driver. + */ + power-domains = <&gpucc GPU_GX_GDSC>; + status = "disabled"; + }; + gpucc: clock-controller@5065000 { compatible = "qcom,msm8998-gpucc"; #clock-cells = <1>; @@ -1921,6 +2021,11 @@ etm7_out: endpoint { }; }; + sram@290000 { + compatible = "qcom,rpm-stats"; + reg = <0x00290000 0x10000>; + }; + spmi_bus: spmi@800f000 { compatible = "qcom,spmi-pmic-arb"; reg = <0x0800f000 0x1000>, @@ -1985,7 +2090,6 @@ usb3phy: phy@c010000 { compatible = "qcom,msm8998-qmp-usb3-phy"; reg = <0x0c010000 0x18c>; status = "disabled"; - #clock-cells = <1>; #address-cells = <1>; #size-cells = <1>; ranges; @@ -1999,13 +2103,14 @@ usb3phy: phy@c010000 { <&gcc GCC_USB3PHY_PHY_BCR>; reset-names = "phy", "common"; - usb1_ssphy: lane@c010200 { + usb1_ssphy: phy@c010200 { reg = <0xc010200 0x128>, <0xc010400 0x200>, <0xc010c00 0x20c>, <0xc010600 0x128>, <0xc010800 0x200>; #phy-cells = <0>; + #clock-cells = <1>; clocks = <&gcc GCC_USB3_PHY_PIPE_CLK>; clock-names = "pipe0"; clock-output-names = "usb3_phy_pipe_clk_src"; @@ -2191,7 +2296,7 @@ blsp1_i2c6: i2c@c17a000 { #size-cells = <0>; }; - blsp2_dma: dma@c184000 { + blsp2_dma: dma-controller@c184000 { compatible = "qcom,bam-v1.7.0"; reg = <0x0c184000 0x25000>; interrupts = ; @@ -2334,6 +2439,73 @@ blsp2_i2c6: i2c@c1ba000 { #size-cells = <0>; }; + mmcc: clock-controller@c8c0000 { + compatible = "qcom,mmcc-msm8998"; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + reg = <0xc8c0000 0x40000>; + status = "disabled"; + + clock-names = "xo", + "gpll0", + "dsi0dsi", + "dsi0byte", + "dsi1dsi", + "dsi1byte", + "hdmipll", + "dplink", + "dpvco", + "core_bi_pll_test_se"; + clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>, + <&gcc GCC_MMSS_GPLL0_CLK>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>, + <0>; + }; + + mmss_smmu: iommu@cd00000 { + compatible = "qcom,msm8998-smmu-v2", "qcom,smmu-v2"; + reg = <0x0cd00000 0x40000>; + #iommu-cells = <1>; + + clocks = <&mmcc MNOC_AHB_CLK>, + <&mmcc BIMC_SMMU_AHB_CLK>, + <&rpmcc RPM_SMD_MMAXI_CLK>, + <&mmcc BIMC_SMMU_AXI_CLK>; + clock-names = "iface-mm", "iface-smmu", + "bus-mm", "bus-smmu"; + status = "disabled"; + + #global-interrupts = <0>; + interrupts = + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + }; + remoteproc_adsp: remoteproc@17300000 { compatible = "qcom,msm8998-adsp-pas"; reg = <0x17300000 0x4040>; diff --git a/arch/arm64/boot/dts/qcom/pm6150l.dtsi b/arch/arm64/boot/dts/qcom/pm6150l.dtsi index b49860cd13..3ca2860bb0 100644 --- a/arch/arm64/boot/dts/qcom/pm6150l.dtsi +++ b/arch/arm64/boot/dts/qcom/pm6150l.dtsi @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) 2019, The Linux Foundation. All rights reserved. +#include #include #include diff --git a/arch/arm64/boot/dts/qcom/pm660.dtsi b/arch/arm64/boot/dts/qcom/pm660.dtsi index e847d7209a..c482663aad 100644 --- a/arch/arm64/boot/dts/qcom/pm660.dtsi +++ b/arch/arm64/boot/dts/qcom/pm660.dtsi @@ -49,18 +49,29 @@ rtc@6000 { }; pon: pon@800 { - compatible = "qcom,pm8916-pon"; - + compatible = "qcom,pm8998-pon"; reg = <0x800>; + mode-bootloader = <0x2>; + mode-recovery = <0x1>; - pwrkey { + pon_pwrkey: pwrkey { compatible = "qcom,pm8941-pwrkey"; interrupts = <0x0 0x8 0 IRQ_TYPE_EDGE_BOTH>; debounce = <15625>; bias-pull-up; linux,code = ; + + status = "disabled"; }; + pon_resin: resin { + compatible = "qcom,pm8941-resin"; + interrupts = <0x0 0x8 1 IRQ_TYPE_EDGE_BOTH>; + debounce = <15625>; + bias-pull-up; + + status = "disabled"; + }; }; pm660_temp: temp-alarm@2400 { diff --git a/arch/arm64/boot/dts/qcom/pm660l.dtsi b/arch/arm64/boot/dts/qcom/pm660l.dtsi index 05086cbe57..cfef423536 100644 --- a/arch/arm64/boot/dts/qcom/pm660l.dtsi +++ b/arch/arm64/boot/dts/qcom/pm660l.dtsi @@ -72,13 +72,6 @@ pm660l_wled: leds@d800 { interrupt-names = "ovp"; label = "backlight"; - qcom,switching-freq = <800>; - qcom,ovp-millivolt = <29600>; - qcom,current-boost-limit = <970>; - qcom,current-limit-microamp = <20000>; - qcom,num-strings = <2>; - qcom,enabled-strings = <0 1>; - status = "disabled"; }; diff --git a/arch/arm64/boot/dts/qcom/pm8916.dtsi b/arch/arm64/boot/dts/qcom/pm8916.dtsi index 42180f1b5d..d589024328 100644 --- a/arch/arm64/boot/dts/qcom/pm8916.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8916.dtsi @@ -41,6 +41,14 @@ watchdog { }; }; + pm8916_usbin: extcon@1300 { + compatible = "qcom,pm8941-misc"; + reg = <0x1300>; + interrupts = <0x0 0x13 1 IRQ_TYPE_EDGE_BOTH>; + interrupt-names = "usb_vbus"; + status = "disabled"; + }; + pm8916_temp: temp-alarm@2400 { compatible = "qcom,spmi-temp-alarm"; reg = <0x2400>; @@ -90,14 +98,13 @@ rtc@6000 { }; pm8916_mpps: mpps@a000 { - compatible = "qcom,pm8916-mpp"; + compatible = "qcom,pm8916-mpp", "qcom,spmi-mpp"; reg = <0xa000>; gpio-controller; #gpio-cells = <2>; - interrupts = <0 0xa0 0 IRQ_TYPE_NONE>, - <0 0xa1 0 IRQ_TYPE_NONE>, - <0 0xa2 0 IRQ_TYPE_NONE>, - <0 0xa3 0 IRQ_TYPE_NONE>; + gpio-ranges = <&pm8916_mpps 0 0 4>; + interrupt-controller; + #interrupt-cells = <2>; }; pm8916_gpios: gpios@c000 { diff --git a/arch/arm64/boot/dts/qcom/pm8994.dtsi b/arch/arm64/boot/dts/qcom/pm8994.dtsi index ad19016df0..5ab46117d7 100644 --- a/arch/arm64/boot/dts/qcom/pm8994.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8994.dtsi @@ -119,18 +119,13 @@ pm8994_gpios: gpios@c000 { }; pm8994_mpps: mpps@a000 { - compatible = "qcom,pm8994-mpp"; + compatible = "qcom,pm8994-mpp", "qcom,spmi-mpp"; reg = <0xa000>; gpio-controller; #gpio-cells = <2>; - interrupts = <0 0xa0 0 IRQ_TYPE_NONE>, - <0 0xa1 0 IRQ_TYPE_NONE>, - <0 0xa2 0 IRQ_TYPE_NONE>, - <0 0xa3 0 IRQ_TYPE_NONE>, - <0 0xa4 0 IRQ_TYPE_NONE>, - <0 0xa5 0 IRQ_TYPE_NONE>, - <0 0xa6 0 IRQ_TYPE_NONE>, - <0 0xa7 0 IRQ_TYPE_NONE>; + gpio-ranges = <&pm8994_mpps 0 0 8>; + interrupt-controller; + #interrupt-cells = <2>; }; }; diff --git a/arch/arm64/boot/dts/qcom/pm8998.dtsi b/arch/arm64/boot/dts/qcom/pm8998.dtsi index 6f5bb6b37e..d09f2954b6 100644 --- a/arch/arm64/boot/dts/qcom/pm8998.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8998.dtsi @@ -84,6 +84,16 @@ adc-chan@6 { }; }; + pm8998_adc_tm: adc-tm@3400 { + compatible = "qcom,spmi-adc-tm-hc"; + reg = <0x3400>; + interrupts = <0x0 0x34 0x0 IRQ_TYPE_EDGE_RISING>; + #thermal-sensor-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + rtc@6000 { compatible = "qcom,pm8941-rtc"; reg = <0x6000>, <0x6100>; diff --git a/arch/arm64/boot/dts/qcom/pmi8994.dtsi b/arch/arm64/boot/dts/qcom/pmi8994.dtsi index a06ea9adae..6e7c252568 100644 --- a/arch/arm64/boot/dts/qcom/pmi8994.dtsi +++ b/arch/arm64/boot/dts/qcom/pmi8994.dtsi @@ -38,9 +38,6 @@ pmi8994_wled: wled@d800 { reg = <0xd800 0xd900>; interrupts = <3 0xd8 0x02 IRQ_TYPE_EDGE_RISING>; interrupt-names = "short"; - qcom,num-strings = <3>; - /* Yes, all four strings *have to* be defined or things won't work. */ - qcom,enabled-strings = <0 1 2 3>; qcom,cabc; qcom,external-pfet; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/pmi8998.dtsi b/arch/arm64/boot/dts/qcom/pmi8998.dtsi index d230c510d4..0fef5f113f 100644 --- a/arch/arm64/boot/dts/qcom/pmi8998.dtsi +++ b/arch/arm64/boot/dts/qcom/pmi8998.dtsi @@ -41,5 +41,17 @@ lab: lab { interrupt-names = "sc-err", "ocp"; }; }; + + pmi8998_wled: leds@d800 { + compatible = "qcom,pmi8998-wled"; + reg = <0xd800 0xd900>; + interrupts = <0x3 0xd8 0x1 IRQ_TYPE_EDGE_RISING>, + <0x3 0xd8 0x2 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "ovp", "short"; + label = "backlight"; + + status = "disabled"; + }; + }; }; diff --git a/arch/arm64/boot/dts/qcom/pmk8350.dtsi b/arch/arm64/boot/dts/qcom/pmk8350.dtsi index 04fc2632a0..0f94c46a14 100644 --- a/arch/arm64/boot/dts/qcom/pmk8350.dtsi +++ b/arch/arm64/boot/dts/qcom/pmk8350.dtsi @@ -19,16 +19,17 @@ pmk8350_pon: pon@1300 { compatible = "qcom,pm8998-pon"; reg = <0x1300>; - pwrkey { + pon_pwrkey: pwrkey { compatible = "qcom,pmk8350-pwrkey"; interrupts = <0x0 0x13 0x7 IRQ_TYPE_EDGE_BOTH>; linux,code = ; + status = "disabled"; }; - resin { + pon_resin: resin { compatible = "qcom,pmk8350-resin"; interrupts = <0x0 0x13 0x6 IRQ_TYPE_EDGE_BOTH>; - linux,code = ; + status = "disabled"; }; }; @@ -59,6 +60,7 @@ pmk8350_rtc: rtc@6100 { reg = <0x6100>, <0x6200>; reg-names = "rtc", "alarm"; interrupts = <0x0 0x62 0x1 IRQ_TYPE_EDGE_RISING>; + status = "disabled"; }; pmk8350_gpios: gpio@b000 { diff --git a/arch/arm64/boot/dts/qcom/qcs404.dtsi b/arch/arm64/boot/dts/qcom/qcs404.dtsi index ca5be16479..6db753b493 100644 --- a/arch/arm64/boot/dts/qcom/qcs404.dtsi +++ b/arch/arm64/boot/dts/qcom/qcs404.dtsi @@ -719,6 +719,11 @@ tcsr: syscon@1937000 { reg = <0x01937000 0x25000>; }; + sram@290000 { + compatible = "qcom,rpm-stats"; + reg = <0x00290000 0x10000>; + }; + spmi_bus: spmi@200f000 { compatible = "qcom,spmi-pmic-arb"; reg = <0x0200f000 0x001000>, diff --git a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts index 28d5b55285..845eb7a6bf 100644 --- a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts +++ b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts @@ -16,6 +16,8 @@ / { model = "Qualcomm Technologies, Inc. Robotics RB5"; compatible = "qcom,qrb5165-rb5", "qcom,sm8250"; + qcom,msm-id = <455 0x20001>; + qcom,board-id = <11 3>; aliases { serial0 = &uart12; @@ -1222,9 +1224,7 @@ &ufs_mem_phy { status = "okay"; vdda-phy-supply = <&vreg_l5a_0p88>; - vdda-max-microamp = <89900>; vdda-pll-supply = <&vreg_l9a_1p2>; - vdda-pll-max-microamp = <18800>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts index 5ae2ddc65f..8756c2b25c 100644 --- a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts +++ b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts @@ -307,6 +307,16 @@ &qupv3_id_1 { status = "okay"; }; +&remoteproc_adsp { + status = "okay"; + firmware-name = "qcom/sa8155p/adsp.mdt"; +}; + +&remoteproc_cdsp { + status = "okay"; + firmware-name = "qcom/sa8155p/cdsp.mdt"; +}; + &uart2 { status = "okay"; }; @@ -328,9 +338,7 @@ &ufs_mem_phy { status = "okay"; vdda-phy-supply = <&vreg_l8c_1p2>; - vdda-max-microamp = <87100>; vdda-pll-supply = <&vreg_l5a_0p88>; - vdda-pll-max-microamp = <18300>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts index 21b516e069..8290d03604 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts @@ -23,6 +23,20 @@ &charger_thermal { status = "disabled"; }; +&pm6150_adc { + status = "disabled"; + + /delete-node/ skin-temp-thermistor@4e; + /delete-node/ charger-thermistor@4f; +}; + +&pm6150_adc_tm { + status = "disabled"; + + /delete-node/ charger-thermistor@0; + /delete-node/ skin-temp-thermistor@1; +}; + /* * CoachZ rev1 is stuffed with a 47k NTC as thermistor for skin temperature, * which currently is not supported by the PM6150 ADC driver. Disable the diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz.dtsi index 81098aa968..14ed09f30a 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz.dtsi @@ -11,6 +11,7 @@ ap_h1_spi: &spi0 {}; #include "sc7180-trogdor.dtsi" +#include "sc7180-trogdor-ti-sn65dsi86.dtsi" /* Deleted nodes from trogdor.dtsi */ diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-limozeen-nots-r4.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-limozeen-nots-r4.dts index 6ebde08285..850776c532 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-limozeen-nots-r4.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-limozeen-nots-r4.dts @@ -5,7 +5,7 @@ * Copyright 2020 Google LLC. */ -#include "sc7180-trogdor-lazor-limozeen-nots.dts" +#include "sc7180-trogdor-lazor-limozeen-nots-r5.dts" / { model = "Google Lazor Limozeen without Touchscreen (rev4)"; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts index 30e3e769d2..b142006478 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts @@ -7,6 +7,8 @@ /dts-v1/; +#include "sc7180.dtsi" +#include "sc7180-trogdor-ti-sn65dsi86.dtsi" #include "sc7180-trogdor-lazor.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts index c2ef06367b..59740799fa 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts @@ -7,6 +7,8 @@ /dts-v1/; +#include "sc7180.dtsi" +#include "sc7180-trogdor-ti-sn65dsi86.dtsi" #include "sc7180-trogdor-lazor.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3-kb.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3-kb.dts index dcb41afdc8..18ef9da719 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3-kb.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3-kb.dts @@ -7,12 +7,17 @@ /dts-v1/; +#include "sc7180.dtsi" +#include "sc7180-trogdor-ti-sn65dsi86.dtsi" #include "sc7180-trogdor-lazor.dtsi" #include "sc7180-lite.dtsi" / { - model = "Google Lazor (rev3+) with KB Backlight"; - compatible = "google,lazor-sku2", "qcom,sc7180"; + model = "Google Lazor (rev3 - 8) with KB Backlight"; + compatible = "google,lazor-rev3-sku2", "google,lazor-rev4-sku2", + "google,lazor-rev5-sku2", "google,lazor-rev6-sku2", + "google,lazor-rev7-sku2", "google,lazor-rev8-sku2", + "qcom,sc7180"; }; &keyboard_backlight { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3-lte.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3-lte.dts index be44900602..c5c9feff41 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3-lte.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3-lte.dts @@ -7,12 +7,17 @@ /dts-v1/; +#include "sc7180.dtsi" +#include "sc7180-trogdor-ti-sn65dsi86.dtsi" #include "sc7180-trogdor-lazor.dtsi" #include "sc7180-trogdor-lte-sku.dtsi" / { - model = "Google Lazor (rev3+) with LTE"; - compatible = "google,lazor-sku0", "qcom,sc7180"; + model = "Google Lazor (rev3 - 8) with LTE"; + compatible = "google,lazor-rev3-sku0", "google,lazor-rev4-sku0", + "google,lazor-rev5-sku0", "google,lazor-rev6-sku0", + "google,lazor-rev7-sku0", "google,lazor-rev8-sku0", + "qcom,sc7180"; }; &ap_sar_sensor { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3.dts index b474df47cd..7adcedbf08 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r3.dts @@ -7,10 +7,14 @@ /dts-v1/; +#include "sc7180.dtsi" +#include "sc7180-trogdor-ti-sn65dsi86.dtsi" #include "sc7180-trogdor-lazor.dtsi" #include "sc7180-lite.dtsi" / { - model = "Google Lazor (rev3+)"; - compatible = "google,lazor", "qcom,sc7180"; + model = "Google Lazor (rev3 - 8)"; + compatible = "google,lazor-rev3", "google,lazor-rev4", + "google,lazor-rev5", "google,lazor-rev6", "google,lazor-rev7", + "google,lazor-rev8", "qcom,sc7180"; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi index 00535aaa43..69666f9217 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi @@ -5,8 +5,6 @@ * Copyright 2020 Google LLC. */ -#include "sc7180.dtsi" - ap_ec_spi: &spi6 {}; ap_h1_spi: &spi0 {}; @@ -54,6 +52,18 @@ &panel { compatible = "boe,nv133fhm-n62"; }; +&pm6150_adc { + status = "disabled"; + + /delete-node/ charger-thermistor@4f; +}; + +&pm6150_adc_tm { + status = "disabled"; + + /delete-node/ charger-thermistor@0; +}; + &trackpad { interrupts = <58 IRQ_TYPE_EDGE_FALLING>; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lte-sku.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lte-sku.dtsi index 469aad4e59..fd4b712037 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lte-sku.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lte-sku.dtsi @@ -17,3 +17,14 @@ &remoteproc_mpss { firmware-name = "qcom/sc7180-trogdor/modem/mba.mbn", "qcom/sc7180-trogdor/modem/qdsp6sw.mbn"; }; + +&ipa { + status = "okay"; + + /* + * Trogdor doesn't have QHEE (Qualcomm's EL2 blob), so the + * modem needs to cover certain init steps (GSI init), and + * the AP needs to wait for it. + */ + modem-init; +}; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts index e122a6b481..76a130bad6 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts @@ -26,6 +26,14 @@ &charger_thermal { status = "disabled"; }; +&pm6150_adc { + /delete-node/ charger-thermistor@4f; +}; + +&pm6150_adc_tm { + /delete-node/ charger-thermistor@0; +}; + &pp3300_hub { /* pp3300_l7c is used to power the USB hub */ /delete-property/regulator-always-on; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r2.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r2.dts index 4f32e6733f..88cf2246c1 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r2.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r2.dts @@ -22,3 +22,11 @@ / { &charger_thermal { status = "disabled"; }; + +&pm6150_adc { + /delete-node/ charger-thermistor@4f; +}; + +&pm6150_adc_tm { + /delete-node/ charger-thermistor@0; +}; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom.dtsi index b7b5264888..e90f99ef53 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom.dtsi @@ -11,6 +11,7 @@ ap_h1_spi: &spi0 {}; #include "sc7180-trogdor.dtsi" +#include "sc7180-trogdor-ti-sn65dsi86.dtsi" / { thermal-zones { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts index 2b522f9e0d..457c254998 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts @@ -13,6 +13,7 @@ ap_h1_spi: &spi0 {}; #include "sc7180-trogdor.dtsi" +#include "sc7180-trogdor-ti-sn65dsi86.dtsi" / { model = "Google Trogdor (rev1+)"; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi index 70c88c37de..bd5909ffb3 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi @@ -601,15 +601,10 @@ &camcc { &dsi0 { status = "okay"; vdda-supply = <&vdda_mipi_dsi0_1p2>; +}; - ports { - port@1 { - endpoint { - remote-endpoint = <&sn65dsi86_in>; - data-lanes = <0 1 2 3>; - }; - }; - }; +&dsi0_out { + data-lanes = <0 1 2 3>; }; &dsi_phy { @@ -617,70 +612,6 @@ &dsi_phy { vdds-supply = <&vdda_mipi_dsi0_pll>; }; -edp_brij_i2c: &i2c2 { - status = "okay"; - clock-frequency = <400000>; - - sn65dsi86_bridge: bridge@2d { - compatible = "ti,sn65dsi86"; - reg = <0x2d>; - pinctrl-names = "default"; - pinctrl-0 = <&edp_brij_en>, <&edp_brij_irq>; - gpio-controller; - #gpio-cells = <2>; - - interrupt-parent = <&tlmm>; - interrupts = <11 IRQ_TYPE_LEVEL_HIGH>; - - enable-gpios = <&tlmm 104 GPIO_ACTIVE_HIGH>; - - vpll-supply = <&pp1800_edp_vpll>; - vccio-supply = <&pp1800_brij_vccio>; - vcca-supply = <&pp1200_brij>; - vcc-supply = <&pp1200_brij>; - - clocks = <&rpmhcc RPMH_LN_BB_CLK3>; - clock-names = "refclk"; - - no-hpd; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - sn65dsi86_in: endpoint { - remote-endpoint = <&dsi0_out>; - }; - }; - - port@1 { - reg = <1>; - sn65dsi86_out: endpoint { - data-lanes = <0 1>; - remote-endpoint = <&panel_in_edp>; - }; - }; - }; - - aux-bus { - panel: panel { - /* Compatible will be filled in per-board */ - power-supply = <&pp3300_dx_edp>; - backlight = <&backlight>; - hpd-gpios = <&sn65dsi86_bridge 2 GPIO_ACTIVE_HIGH>; - - port { - panel_in_edp: endpoint { - remote-endpoint = <&sn65dsi86_out>; - }; - }; - }; - }; - }; -}; - ap_sar_sensor_i2c: &i2c5 { clock-frequency = <400000>; @@ -750,17 +681,6 @@ alc5682: codec@1a { }; }; -&ipa { - status = "okay"; - - /* - * Trogdor doesn't have QHEE (Qualcomm's EL2 blob), so the - * modem needs to cover certain init steps (GSI init), and - * the AP needs to wait for it. - */ - modem-init; -}; - &lpass_cpu { status = "okay"; @@ -1244,19 +1164,6 @@ pinconf { }; }; - edp_brij_irq: edp-brij-irq { - pinmux { - pins = "gpio11"; - function = "gpio"; - }; - - pinconf { - pins = "gpio11"; - drive-strength = <2>; - bias-pull-down; - }; - }; - en_pp3300_codec: en-pp3300-codec { pinmux { pins = "gpio83"; @@ -1523,13 +1430,13 @@ pinconf-clk { pinconf-cmd { pins = "sdc1_cmd"; bias-pull-up; - drive-strength = <10>; + drive-strength = <16>; }; pinconf-data { pins = "sdc1_data"; bias-pull-up; - drive-strength = <10>; + drive-strength = <16>; }; pinconf-rclk { diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi index 495c15deac..2151cd8c8c 100644 --- a/arch/arm64/boot/dts/qcom/sc7180.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -1922,14 +1921,15 @@ remoteproc_mpss: remoteproc@4080000 { clock-names = "iface", "bus", "nav", "snoc_axi", "mnoc_axi", "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_MODEM>, - <&rpmhpd SC7180_CX>, + power-domains = <&rpmhpd SC7180_CX>, <&rpmhpd SC7180_MX>, <&rpmhpd SC7180_MSS>; - power-domain-names = "load_state", "cx", "mx", "mss"; + power-domain-names = "cx", "mx", "mss"; memory-region = <&mpss_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&modem_smp2p_out 0>; qcom,smem-state-names = "stop"; @@ -1952,7 +1952,6 @@ glink-edge { gpu: gpu@5000000 { compatible = "qcom,adreno-618.0", "qcom,adreno"; - #stream-id-cells = <16>; reg = <0 0x05000000 0 0x40000>, <0 0x0509e000 0 0x1000>, <0 0x05061000 0 0x800>; reg-names = "kgsl_3d0_reg_memory", "cx_mem", "cx_dbgc"; @@ -2647,7 +2646,7 @@ opp-300000000 { }; qspi: spi@88dc000 { - compatible = "qcom,qspi-v1"; + compatible = "qcom,sc7180-qspi", "qcom,qspi-v1"; reg = <0 0x088dc000 0 0x600>; #address-cells = <1>; #size-cells = <0>; @@ -3219,12 +3218,16 @@ aoss_reset: reset-controller@c2a0000 { aoss_qmp: power-controller@c300000 { compatible = "qcom,sc7180-aoss-qmp"; - reg = <0 0x0c300000 0 0x100000>; + reg = <0 0x0c300000 0 0x400>; interrupts = ; mboxes = <&apss_shared 0>; #clock-cells = <0>; - #power-domain-cells = <1>; + }; + + sram@c3f0000 { + compatible = "qcom,rpmh-stats"; + reg = <0 0x0c3f0000 0 0x400>; }; spmi_bus: spmi@c440000 { @@ -3246,6 +3249,21 @@ spmi_bus: spmi@c440000 { cell-index = <0>; }; + imem@146aa000 { + compatible = "simple-mfd"; + reg = <0 0x146aa000 0 0x2000>; + + #address-cells = <1>; + #size-cells = <1>; + + ranges = <0 0 0x146aa000 0x2000>; + + pil-reloc@94c { + compatible = "qcom,pil-reloc-info"; + reg = <0x94c 0xc8>; + }; + }; + apps_smmu: iommu@15000000 { compatible = "qcom,sc7180-smmu-500", "arm,mmu-500"; reg = <0 0x15000000 0 0x100000>; diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp.dts b/arch/arm64/boot/dts/qcom/sc7280-idp.dts index 64fc22aff3..9b991ba5da 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-idp.dts +++ b/arch/arm64/boot/dts/qcom/sc7280-idp.dts @@ -61,6 +61,18 @@ &ipa { modem-init; }; +&pmk8350_rtc { + status = "okay"; +}; + +&nvme_pwren { + pins = "gpio19"; +}; + +&nvme_3v3_regulator { + gpio = <&tlmm 19 GPIO_ACTIVE_HIGH>; +}; + &pmk8350_vadc { pmr735a_die_temp { reg = ; diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi index 371a2a9dcf..d623d71d8b 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi @@ -7,11 +7,96 @@ #include #include +#include #include "sc7280.dtsi" #include "pm7325.dtsi" #include "pm8350c.dtsi" #include "pmk8350.dtsi" +/ { + gpio-keys { + compatible = "gpio-keys"; + label = "gpio-keys"; + + pinctrl-names = "default"; + pinctrl-0 = <&key_vol_up_default>; + + volume-up { + label = "volume_up"; + gpios = <&pm7325_gpios 6 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + linux,code = ; + gpio-key,wakeup; + debounce-interval = <15>; + linux,can-disable; + }; + }; + + nvme_3v3_regulator: nvme-3v3-regulator { + compatible = "regulator-fixed"; + regulator-name = "VLDO_3V3"; + + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + enable-active-high; + pinctrl-names = "default"; + pinctrl-0 = <&nvme_pwren>; + }; +}; + +/* + * Reserved memory changes + * + * Delete all unused memory nodes and define the peripheral memory regions + * required by the board dts. + * + */ + +/delete-node/ &hyp_mem; +/delete-node/ &xbl_mem; +/delete-node/ &reserved_xbl_uefi_log; +/delete-node/ &sec_apps_mem; + +/* Increase the size from 2.5MB to 8MB */ +&rmtfs_mem { + reg = <0x0 0x9c900000 0x0 0x800000>; +}; + +/ { + reserved-memory { + adsp_mem: memory@86700000 { + reg = <0x0 0x86700000 0x0 0x2800000>; + no-map; + }; + + camera_mem: memory@8ad00000 { + reg = <0x0 0x8ad00000 0x0 0x500000>; + no-map; + }; + + venus_mem: memory@8b200000 { + reg = <0x0 0x8b200000 0x0 0x500000>; + no-map; + }; + + mpss_mem: memory@8b800000 { + reg = <0x0 0x8b800000 0x0 0xf600000>; + no-map; + }; + + wpss_mem: memory@9ae00000 { + reg = <0x0 0x9ae00000 0x0 0x1900000>; + no-map; + }; + + mba_mem: memory@9c700000 { + reg = <0x0 0x9c700000 0x0 0x200000>; + no-map; + }; + }; +}; + &apps_rsc { pm7325-regulators { compatible = "qcom,pm7325-rpmh-regulators"; @@ -199,6 +284,23 @@ &ipa { modem-init; }; +&pcie1 { + status = "okay"; + perst-gpio = <&tlmm 2 GPIO_ACTIVE_LOW>; + + vddpe-3v3-supply = <&nvme_3v3_regulator>; + + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_reset_n>, <&pcie1_wake_n>; +}; + +&pcie1_phy { + status = "okay"; + + vdda-phy-supply = <&vreg_l10c_0p8>; + vdda-pll-supply = <&vreg_l6b_1p2>; +}; + &pmk8350_vadc { pmk8350_die_temp { reg = ; @@ -207,10 +309,39 @@ pmk8350_die_temp { }; }; +&qfprom { + vcc-supply = <&vreg_l1c_1p8>; +}; + +&qspi { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&qspi_clk>, <&qspi_cs0>, <&qspi_data01>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <37500000>; + spi-tx-bus-width = <2>; + spi-rx-bus-width = <2>; + }; +}; + &qupv3_id_0 { status = "okay"; }; +&qupv3_id_1 { + status = "okay"; +}; + +&remoteproc_mpss { + status = "okay"; + compatible = "qcom,sc7280-mss-pil"; + iommus = <&apps_smmu 0x124 0x0>, <&apps_smmu 0x488 0x7>; + memory-region = <&mba_mem &mpss_mem>; +}; + &sdhc_1 { status = "okay"; @@ -240,6 +371,7 @@ &sdhc_2 { }; &uart5 { + compatible = "qcom,geni-debug-uart"; status = "okay"; }; @@ -282,20 +414,146 @@ &usb_2_hsphy { vdda18-supply = <&vreg_l1c_1p8>; }; +&uart7 { + status = "okay"; + + /delete-property/interrupts; + interrupts-extended = <&intc GIC_SPI 608 IRQ_TYPE_LEVEL_HIGH>, + <&tlmm 31 IRQ_TYPE_EDGE_FALLING>; + pinctrl-names = "default", "sleep"; + pinctrl-1 = <&qup_uart7_sleep_cts>, <&qup_uart7_sleep_rts>, <&qup_uart7_sleep_tx>, <&qup_uart7_sleep_rx>; +}; + /* PINCTRL - additions to nodes defined in sc7280.dtsi */ -&qup_uart5_default { - tx { - pins = "gpio46"; - drive-strength = <2>; +&pm7325_gpios { + key_vol_up_default: key-vol-up-default { + pins = "gpio6"; + function = "normal"; + input-enable; + bias-pull-up; + power-source = <0>; + qcom,drive-strength = <3>; + }; +}; + +&qspi_cs0 { + bias-disable; +}; + +&qspi_clk { + bias-disable; +}; + +&qspi_data01 { + /* High-Z when no transfers; nice to park the lines */ + bias-pull-up; +}; + +&qup_uart5_tx { + drive-strength = <2>; + bias-disable; +}; + +&qup_uart5_rx { + drive-strength = <2>; + bias-pull-up; +}; + +&qup_uart7_cts { + /* + * Configure a pull-down on CTS to match the pull of + * the Bluetooth module. + */ + bias-pull-down; +}; + +&qup_uart7_rts { + /* We'll drive RTS, so no pull */ + drive-strength = <2>; + bias-disable; +}; + +&qup_uart7_tx { + /* We'll drive TX, so no pull */ + drive-strength = <2>; + bias-disable; +}; + +&qup_uart7_rx { + /* + * Configure a pull-up on RX. This is needed to avoid + * garbage data when the TX pin of the Bluetooth module is + * in tri-state (module powered off or not driving the + * signal yet). + */ + bias-pull-up; +}; + +&tlmm { + nvme_pwren: nvme-pwren { + function = "gpio"; + }; + + pcie1_reset_n: pcie1-reset-n { + pins = "gpio2"; + function = "gpio"; + + drive-strength = <16>; + output-low; bias-disable; }; - rx { - pins = "gpio47"; + pcie1_wake_n: pcie1-wake-n { + pins = "gpio3"; + function = "gpio"; + drive-strength = <2>; bias-pull-up; }; + + qup_uart7_sleep_cts: qup-uart7-sleep-cts { + pins = "gpio28"; + function = "gpio"; + /* + * Configure a pull-down on CTS to match the pull of + * the Bluetooth module. + */ + bias-pull-down; + }; + + qup_uart7_sleep_rts: qup-uart7-sleep-rts { + pins = "gpio29"; + function = "gpio"; + /* + * Configure pull-down on RTS. As RTS is active low + * signal, pull it low to indicate the BT SoC that it + * can wakeup the system anytime from suspend state by + * pulling RX low (by sending wakeup bytes). + */ + bias-pull-down; + }; + + qup_uart7_sleep_tx: qup-uart7-sleep-tx { + pins = "gpio30"; + function = "gpio"; + /* + * Configure pull-up on TX when it isn't actively driven + * to prevent BT SoC from receiving garbage during sleep. + */ + bias-pull-up; + }; + + qup_uart7_sleep_rx: qup-uart7-sleep-rx { + pins = "gpio31"; + function = "gpio"; + /* + * Configure a pull-up on RX. This is needed to avoid + * garbage data when the TX pin of the Bluetooth module + * is floating which may cause spurious wakeups. + */ + bias-pull-up; + }; }; &sdc1_on { @@ -336,6 +594,7 @@ data { }; sd-cd { + pins = "gpio91"; bias-pull-up; }; }; diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp2.dts b/arch/arm64/boot/dts/qcom/sc7280-idp2.dts index 1fc2addc8a..0382c77065 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-idp2.dts +++ b/arch/arm64/boot/dts/qcom/sc7280-idp2.dts @@ -8,6 +8,7 @@ /dts-v1/; #include "sc7280-idp.dtsi" +#include "sc7280-idp-ec-h1.dtsi" / { model = "Qualcomm Technologies, Inc. sc7280 IDP SKU2 platform"; @@ -21,3 +22,11 @@ chosen { stdout-path = "serial0:115200n8"; }; }; + +&nvme_pwren { + pins = "gpio51"; +}; + +&nvme_3v3_regulator { + gpio = <&tlmm 51 GPIO_ACTIVE_HIGH>; +}; diff --git a/arch/arm64/boot/dts/qcom/sc7280.dtsi b/arch/arm64/boot/dts/qcom/sc7280.dtsi index 692973c4f4..937c2e0e93 100644 --- a/arch/arm64/boot/dts/qcom/sc7280.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280.dtsi @@ -5,12 +5,14 @@ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. */ +#include #include +#include #include +#include #include #include #include -#include #include #include #include @@ -26,8 +28,40 @@ / { chosen { }; aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + i2c6 = &i2c6; + i2c7 = &i2c7; + i2c8 = &i2c8; + i2c9 = &i2c9; + i2c10 = &i2c10; + i2c11 = &i2c11; + i2c12 = &i2c12; + i2c13 = &i2c13; + i2c14 = &i2c14; + i2c15 = &i2c15; mmc1 = &sdhc_1; mmc2 = &sdhc_2; + spi0 = &spi0; + spi1 = &spi1; + spi2 = &spi2; + spi3 = &spi3; + spi4 = &spi4; + spi5 = &spi5; + spi6 = &spi6; + spi7 = &spi7; + spi8 = &spi8; + spi9 = &spi9; + spi10 = &spi10; + spi11 = &spi11; + spi12 = &spi12; + spi13 = &spi13; + spi14 = &spi14; + spi15 = &spi15; }; clocks { @@ -49,6 +83,16 @@ reserved-memory { #size-cells = <2>; ranges; + hyp_mem: memory@80000000 { + reg = <0x0 0x80000000 0x0 0x600000>; + no-map; + }; + + xbl_mem: memory@80600000 { + reg = <0x0 0x80600000 0x0 0x200000>; + no-map; + }; + aop_mem: memory@80800000 { reg = <0x0 0x80800000 0x0 0x60000>; no-map; @@ -60,6 +104,16 @@ aop_cmd_db_mem: memory@80860000 { no-map; }; + reserved_xbl_uefi_log: memory@80880000 { + reg = <0x0 0x80884000 0x0 0x10000>; + no-map; + }; + + sec_apps_mem: memory@808ff000 { + reg = <0x0 0x808ff000 0x0 0x1000>; + no-map; + }; + smem_mem: memory@80900000 { reg = <0x0 0x80900000 0x0 0x200000>; no-map; @@ -70,10 +124,29 @@ cpucp_mem: memory@80b00000 { reg = <0x0 0x80b00000 0x0 0x100000>; }; + wlan_fw_mem: memory@80c00000 { + reg = <0x0 0x80c00000 0x0 0xc00000>; + no-map; + }; + + video_mem: memory@8b200000 { + reg = <0x0 0x8b200000 0x0 0x500000>; + no-map; + }; + ipa_fw_mem: memory@8b700000 { reg = <0 0x8b700000 0 0x10000>; no-map; }; + + rmtfs_mem: memory@9c900000 { + compatible = "qcom,rmtfs-mem"; + reg = <0x0 0x9c900000 0x0 0x280000>; + no-map; + + qcom,client-id = <1>; + qcom,vmid = <15>; + }; }; cpus { @@ -219,6 +292,42 @@ L2_700: l2-cache { }; }; + cpu-map { + cluster0 { + core0 { + cpu = <&CPU0>; + }; + + core1 { + cpu = <&CPU1>; + }; + + core2 { + cpu = <&CPU2>; + }; + + core3 { + cpu = <&CPU3>; + }; + + core4 { + cpu = <&CPU4>; + }; + + core5 { + cpu = <&CPU5>; + }; + + core6 { + cpu = <&CPU6>; + }; + + core7 { + cpu = <&CPU7>; + }; + }; + }; + idle-states { entry-method = "psci"; @@ -415,6 +524,49 @@ psci { method = "smc"; }; + qspi_opp_table: qspi-opp-table { + compatible = "operating-points-v2"; + + opp-75000000 { + opp-hz = /bits/ 64 <75000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-150000000 { + opp-hz = /bits/ 64 <150000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + + qup_opp_table: qup-opp-table { + compatible = "operating-points-v2"; + + opp-75000000 { + opp-hz = /bits/ 64 <75000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-128000000 { + opp-hz = /bits/ 64 <128000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + }; + soc: soc@0 { #address-cells = <2>; #size-cells = <2>; @@ -427,7 +579,8 @@ gcc: clock-controller@100000 { reg = <0 0x00100000 0 0x1f0000>; clocks = <&rpmhcc RPMH_CXO_CLK>, <&rpmhcc RPMH_CXO_CLK_A>, <&sleep_clk>, - <0>, <0>, <0>, <0>, <0>, <0>; + <0>, <&pcie1_lane 0>, + <0>, <0>, <0>, <0>; clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk", "pcie_0_pipe_clk", "pcie_1_pipe_clk", "ufs_phy_rx_symbol_0_clk", "ufs_phy_rx_symbol_1_clk", @@ -517,22 +670,857 @@ opp-384000000 { qupv3_id_0: geniqup@9c0000 { compatible = "qcom,geni-se-qup"; reg = <0 0x009c0000 0 0x2000>; - clock-names = "m-ahb", "s-ahb"; clocks = <&gcc GCC_QUPV3_WRAP_0_M_AHB_CLK>, <&gcc GCC_QUPV3_WRAP_0_S_AHB_CLK>; + clock-names = "m-ahb", "s-ahb"; #address-cells = <2>; #size-cells = <2>; ranges; + iommus = <&apps_smmu 0x123 0x0>; status = "disabled"; - uart5: serial@994000 { - compatible = "qcom,geni-debug-uart"; - reg = <0 0x00994000 0 0x4000>; + i2c0: i2c@980000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00980000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; clock-names = "se"; - clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>; pinctrl-names = "default"; - pinctrl-0 = <&qup_uart5_default>; + pinctrl-0 = <&qup_i2c0_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi0: spi@980000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00980000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi0_data_clk>, <&qup_spi0_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart0: serial@980000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00980000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart0_cts>, <&qup_uart0_rts>, <&qup_uart0_tx>, <&qup_uart0_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c1: i2c@984000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00984000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c1_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi1: spi@984000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00984000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi1_data_clk>, <&qup_spi1_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart1: serial@984000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00984000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart1_cts>, <&qup_uart1_rts>, <&qup_uart1_tx>, <&qup_uart1_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c2: i2c@988000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00988000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c2_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi2: spi@988000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00988000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi2_data_clk>, <&qup_spi2_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart2: serial@988000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00988000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart2_cts>, <&qup_uart2_rts>, <&qup_uart2_tx>, <&qup_uart2_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c3: i2c@98c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x0098c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c3_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi3: spi@98c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x0098c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi3_data_clk>, <&qup_spi3_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart3: serial@98c000 { + compatible = "qcom,geni-uart"; + reg = <0 0x0098c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart3_cts>, <&qup_uart3_rts>, <&qup_uart3_tx>, <&qup_uart3_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c4: i2c@990000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00990000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c4_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi4: spi@990000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00990000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi4_data_clk>, <&qup_spi4_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart4: serial@990000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00990000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart4_cts>, <&qup_uart4_rts>, <&qup_uart4_tx>, <&qup_uart4_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c5: i2c@994000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00994000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c5_data_clk>; interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi5: spi@994000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00994000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi5_data_clk>, <&qup_spi5_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart5: serial@994000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00994000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart5_cts>, <&qup_uart5_rts>, <&qup_uart5_tx>, <&qup_uart5_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c6: i2c@998000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00998000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c6_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi6: spi@998000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00998000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi6_data_clk>, <&qup_spi6_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart6: serial@998000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00998000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart6_cts>, <&qup_uart6_rts>, <&qup_uart6_tx>, <&qup_uart6_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c7: i2c@99c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x0099c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c7_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi7: spi@99c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x0099c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi7_data_clk>, <&qup_spi7_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart7: serial@99c000 { + compatible = "qcom,geni-uart"; + reg = <0 0x0099c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart7_cts>, <&qup_uart7_rts>, <&qup_uart7_tx>, <&qup_uart7_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_0 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + }; + + qupv3_id_1: geniqup@ac0000 { + compatible = "qcom,geni-se-qup"; + reg = <0 0x00ac0000 0 0x2000>; + clocks = <&gcc GCC_QUPV3_WRAP_1_M_AHB_CLK>, + <&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>; + clock-names = "m-ahb", "s-ahb"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + iommus = <&apps_smmu 0x43 0x0>; + status = "disabled"; + + i2c8: i2c@a80000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a80000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c8_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi8: spi@a80000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a80000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi8_data_clk>, <&qup_spi8_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart8: serial@a80000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00a80000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart8_cts>, <&qup_uart8_rts>, <&qup_uart8_tx>, <&qup_uart8_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c9: i2c@a84000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a84000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c9_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi9: spi@a84000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a84000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi9_data_clk>, <&qup_spi9_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart9: serial@a84000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00a84000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart9_cts>, <&qup_uart9_rts>, <&qup_uart9_tx>, <&qup_uart9_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c10: i2c@a88000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a88000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c10_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi10: spi@a88000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a88000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi10_data_clk>, <&qup_spi10_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart10: serial@a88000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00a88000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart10_cts>, <&qup_uart10_rts>, <&qup_uart10_tx>, <&qup_uart10_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c11: i2c@a8c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a8c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c11_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi11: spi@a8c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a8c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi11_data_clk>, <&qup_spi11_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart11: serial@a8c000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00a8c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart11_cts>, <&qup_uart11_rts>, <&qup_uart11_tx>, <&qup_uart11_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c12: i2c@a90000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a90000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c12_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi12: spi@a90000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a90000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi12_data_clk>, <&qup_spi12_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart12: serial@a90000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00a90000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart12_cts>, <&qup_uart12_rts>, <&qup_uart12_tx>, <&qup_uart12_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c13: i2c@a94000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a94000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c13_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi13: spi@a94000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a94000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi13_data_clk>, <&qup_spi13_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart13: serial@a94000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00a94000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart13_cts>, <&qup_uart13_rts>, <&qup_uart13_tx>, <&qup_uart13_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c14: i2c@a98000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a98000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c14_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi14: spi@a98000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a98000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi14_data_clk>, <&qup_spi14_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart14: serial@a98000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00a98000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart14_cts>, <&qup_uart14_rts>, <&qup_uart14_tx>, <&qup_uart14_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + i2c15: i2c@a9c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a9c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c15_data_clk>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "qup-core", "qup-config", + "qup-memory"; + status = "disabled"; + }; + + spi15: spi@a9c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a9c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_spi15_data_clk>, <&qup_spi15_cs>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; + status = "disabled"; + }; + + uart15: serial@a9c000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00a9c000 0 0x4000>; + clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>; + clock-names = "se"; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart15_cts>, <&qup_uart15_rts>, <&qup_uart15_tx>, <&qup_uart15_rx>; + interrupts = ; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qup_opp_table>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; status = "disabled"; }; }; @@ -586,6 +1574,117 @@ mmss_noc: interconnect@1740000 { qcom,bcm-voters = <&apps_bcm_voter>; }; + pcie1: pci@1c08000 { + compatible = "qcom,pcie-sc7280"; + reg = <0 0x01c08000 0 0x3000>, + <0 0x40000000 0 0xf1d>, + <0 0x40000f20 0 0xa8>, + <0 0x40001000 0 0x1000>, + <0 0x40100000 0 0x100000>; + + reg-names = "parf", "dbi", "elbi", "atu", "config"; + device_type = "pci"; + linux,pci-domain = <1>; + bus-range = <0x00 0xff>; + num-lanes = <2>; + + #address-cells = <3>; + #size-cells = <2>; + + ranges = <0x01000000 0x0 0x40200000 0x0 0x40200000 0x0 0x100000>, + <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x1fd00000>; + + interrupts = ; + interrupt-names = "msi"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc 0 0 0 434 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc 0 0 0 435 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc 0 0 0 438 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc 0 0 0 439 IRQ_TYPE_LEVEL_HIGH>; + + clocks = <&gcc GCC_PCIE_1_PIPE_CLK>, + <&gcc GCC_PCIE_1_PIPE_CLK_SRC>, + <&pcie1_lane 0>, + <&rpmhcc RPMH_CXO_CLK>, + <&gcc GCC_PCIE_1_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>, + <&gcc GCC_AGGRE_NOC_PCIE_TBU_CLK>, + <&gcc GCC_DDRSS_PCIE_SF_CLK>; + + clock-names = "pipe", + "pipe_mux", + "phy_pipe", + "ref", + "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a", + "tbu", + "ddrss_sf_tbu"; + + assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; + assigned-clock-rates = <19200000>; + + resets = <&gcc GCC_PCIE_1_BCR>; + reset-names = "pci"; + + power-domains = <&gcc GCC_PCIE_1_GDSC>; + + phys = <&pcie1_lane>; + phy-names = "pciephy"; + + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_clkreq_n>; + + iommus = <&apps_smmu 0x1c80 0x1>; + + iommu-map = <0x0 &apps_smmu 0x1c80 0x1>, + <0x100 &apps_smmu 0x1c81 0x1>; + + status = "disabled"; + }; + + pcie1_phy: phy@1c0e000 { + compatible = "qcom,sm8250-qmp-gen3x2-pcie-phy"; + reg = <0 0x01c0e000 0 0x1c0>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + clocks = <&gcc GCC_PCIE_1_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_CLKREF_EN>, + <&gcc GCC_PCIE1_PHY_RCHNG_CLK>; + clock-names = "aux", "cfg_ahb", "ref", "refgen"; + + resets = <&gcc GCC_PCIE_1_PHY_BCR>; + reset-names = "phy"; + + assigned-clocks = <&gcc GCC_PCIE1_PHY_RCHNG_CLK>; + assigned-clock-rates = <100000000>; + + status = "disabled"; + + pcie1_lane: lanes@1c0e200 { + reg = <0 0x01c0e200 0 0x170>, + <0 0x01c0e400 0 0x200>, + <0 0x01c0ea00 0 0x1f0>, + <0 0x01c0e600 0 0x170>, + <0 0x01c0e800 0 0x200>, + <0 0x01c0ee00 0 0xf4>; + clocks = <&gcc GCC_PCIE_1_PIPE_CLK>; + clock-names = "pipe0"; + + #phy-cells = <0>; + #clock-cells = <1>; + clock-output-names = "pcie_1_pipe_clk"; + }; + }; + ipa: ipa@1e40000 { compatible = "qcom,sc7280-ipa"; @@ -598,8 +1697,8 @@ ipa: ipa@1e40000 { "ipa-shared", "gsi"; - interrupts-extended = <&intc 0 654 IRQ_TYPE_EDGE_RISING>, - <&intc 0 432 IRQ_TYPE_LEVEL_HIGH>, + interrupts-extended = <&intc GIC_SPI 654 IRQ_TYPE_EDGE_RISING>, + <&intc GIC_SPI 432 IRQ_TYPE_LEVEL_HIGH>, <&ipa_smp2p_in 0 IRQ_TYPE_EDGE_RISING>, <&ipa_smp2p_in 1 IRQ_TYPE_EDGE_RISING>; interrupt-names = "ipa", @@ -629,6 +1728,11 @@ tcsr_mutex: hwlock@1f40000 { #hwlock-cells = <1>; }; + tcsr: syscon@1fc0000 { + compatible = "qcom,sc7280-tcsr", "syscon"; + reg = <0 0x01fc0000 0 0x30000>; + }; + lpasscc: lpasscc@3000000 { compatible = "qcom,sc7280-lpasscc"; reg = <0 0x03000000 0 0x40>, @@ -647,6 +1751,85 @@ lpass_ag_noc: interconnect@3c40000 { qcom,bcm-voters = <&apps_bcm_voter>; }; + gpu: gpu@3d00000 { + compatible = "qcom,adreno-635.0", "qcom,adreno"; + reg = <0 0x03d00000 0 0x40000>, + <0 0x03d9e000 0 0x1000>, + <0 0x03d61000 0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; + interrupts = ; + iommus = <&adreno_smmu 0 0x401>; + operating-points-v2 = <&gpu_opp_table>; + qcom,gmu = <&gmu>; + interconnects = <&gem_noc MASTER_GFX3D 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "gfx-mem"; + #cooling-cells = <2>; + + gpu_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-315000000 { + opp-hz = /bits/ 64 <315000000>; + opp-level = ; + opp-peak-kBps = <1804000>; + }; + + opp-450000000 { + opp-hz = /bits/ 64 <450000000>; + opp-level = ; + opp-peak-kBps = <4068000>; + }; + + opp-550000000 { + opp-hz = /bits/ 64 <550000000>; + opp-level = ; + opp-peak-kBps = <6832000>; + }; + }; + }; + + gmu: gmu@3d69000 { + compatible="qcom,adreno-gmu-635.0", "qcom,adreno-gmu"; + reg = <0 0x03d6a000 0 0x34000>, + <0 0x3de0000 0 0x10000>, + <0 0x0b290000 0 0x10000>; + reg-names = "gmu", "rscc", "gmu_pdc"; + interrupts = , + ; + interrupt-names = "hfi", "gmu"; + clocks = <&gpucc 5>, + <&gpucc 8>, + <&gcc GCC_DDRSS_GPU_AXI_CLK>, + <&gcc GCC_GPU_MEMNOC_GFX_CLK>, + <&gpucc 2>, + <&gpucc 15>, + <&gpucc 11>; + clock-names = "gmu", + "cxo", + "axi", + "memnoc", + "ahb", + "hub", + "smmu_vote"; + power-domains = <&gpucc 0>, + <&gpucc 1>; + power-domain-names = "cx", + "gx"; + iommus = <&adreno_smmu 5 0x400>; + operating-points-v2 = <&gmu_opp_table>; + + gmu_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-level = ; + }; + }; + }; + gpucc: clock-controller@3d90000 { compatible = "qcom,sc7280-gpucc"; reg = <0 0x03d90000 0 0x9000>; @@ -661,6 +1844,95 @@ gpucc: clock-controller@3d90000 { #power-domain-cells = <1>; }; + adreno_smmu: iommu@3da0000 { + compatible = "qcom,sc7280-smmu-500", "qcom,adreno-smmu", "arm,mmu-500"; + reg = <0 0x03da0000 0 0x20000>; + #iommu-cells = <2>; + #global-interrupts = <2>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + ; + + clocks = <&gcc GCC_GPU_MEMNOC_GFX_CLK>, + <&gcc GCC_GPU_SNOC_DVM_GFX_CLK>, + <&gpucc 2>, + <&gpucc 11>, + <&gpucc 5>, + <&gpucc 15>, + <&gpucc 13>; + clock-names = "gcc_gpu_memnoc_gfx_clk", + "gcc_gpu_snoc_dvm_gfx_clk", + "gpu_cc_ahb_clk", + "gpu_cc_hlos1_vote_gpu_smmu_clk", + "gpu_cc_cx_gmu_clk", + "gpu_cc_hub_cx_int_clk", + "gpu_cc_hub_aon_clk"; + + power-domains = <&gpucc 0>; + }; + + remoteproc_mpss: remoteproc@4080000 { + compatible = "qcom,sc7280-mpss-pas"; + reg = <0 0x04080000 0 0x10000>, <0 0x04180000 0 0x48>; + reg-names = "qdsp6", "rmb"; + + interrupts-extended = <&intc GIC_SPI 264 IRQ_TYPE_EDGE_RISING>, + <&modem_smp2p_in 0 IRQ_TYPE_EDGE_RISING>, + <&modem_smp2p_in 1 IRQ_TYPE_EDGE_RISING>, + <&modem_smp2p_in 2 IRQ_TYPE_EDGE_RISING>, + <&modem_smp2p_in 3 IRQ_TYPE_EDGE_RISING>, + <&modem_smp2p_in 7 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "wdog", "fatal", "ready", "handover", + "stop-ack", "shutdown-ack"; + + clocks = <&gcc GCC_MSS_CFG_AHB_CLK>, + <&gcc GCC_MSS_OFFLINE_AXI_CLK>, + <&gcc GCC_MSS_SNOC_AXI_CLK>, + <&rpmhcc RPMH_PKA_CLK>, + <&rpmhcc RPMH_CXO_CLK>; + clock-names = "iface", "offline", "snoc_axi", "pka", "xo"; + + power-domains = <&rpmhpd SC7280_CX>, + <&rpmhpd SC7280_MSS>; + power-domain-names = "cx", "mss"; + + memory-region = <&mpss_mem>; + + qcom,qmp = <&aoss_qmp>; + + qcom,smem-states = <&modem_smp2p_out 0>; + qcom,smem-state-names = "stop"; + + resets = <&aoss_reset AOSS_CC_MSS_RESTART>, + <&pdc_reset PDC_MODEM_SYNC_RESET>; + reset-names = "mss_restart", "pdc_reset"; + + qcom,halt-regs = <&tcsr_mutex 0x23000 0x25000 0x28000 0x33000>; + qcom,ext-regs = <&tcsr 0x10000 0x10004 &tcsr_mutex 0x26004 0x26008>; + qcom,qaccept-regs = <&tcsr_mutex 0x23030 0x23040 0x23020>; + + status = "disabled"; + + glink-edge { + interrupts-extended = <&ipcc IPCC_CLIENT_MPSS + IPCC_MPROC_SIGNAL_GLINK_QMP + IRQ_TYPE_EDGE_RISING>; + mboxes = <&ipcc IPCC_CLIENT_MPSS + IPCC_MPROC_SIGNAL_GLINK_QMP>; + label = "modem"; + qcom,remote-pid = <1>; + }; + }; + stm@6002000 { compatible = "arm,coresight-stm", "arm,primecell"; reg = <0 0x06002000 0 0x1000>, @@ -1314,6 +2586,23 @@ usb_2_dwc3: usb@8c00000 { }; }; + qspi: spi@88dc000 { + compatible = "qcom,sc7280-qspi", "qcom,qspi-v1"; + reg = <0 0x088dc000 0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = ; + clocks = <&gcc GCC_QSPI_CNOC_PERIPH_AHB_CLK>, + <&gcc GCC_QSPI_CORE_CLK>; + clock-names = "iface", "core"; + interconnects = <&gem_noc MASTER_APPSS_PROC 0 + &cnoc2 SLAVE_QSPI_0 0>; + interconnect-names = "qspi-config"; + power-domains = <&rpmhpd SC7280_CX>; + operating-points-v2 = <&qspi_opp_table>; + status = "disabled"; + }; + dc_noc: interconnect@90e0000 { reg = <0 0x090e0000 0 0x5080>; compatible = "qcom,sc7280-dc-noc"; @@ -1391,6 +2680,76 @@ usb_1_dwc3: usb@a600000 { }; }; + venus: video-codec@aa00000 { + compatible = "qcom,sc7280-venus"; + reg = <0 0x0aa00000 0 0xd0600>; + interrupts = ; + + clocks = <&videocc VIDEO_CC_MVSC_CORE_CLK>, + <&videocc VIDEO_CC_MVSC_CTL_AXI_CLK>, + <&videocc VIDEO_CC_VENUS_AHB_CLK>, + <&videocc VIDEO_CC_MVS0_CORE_CLK>, + <&videocc VIDEO_CC_MVS0_AXI_CLK>; + clock-names = "core", "bus", "iface", + "vcodec_core", "vcodec_bus"; + + power-domains = <&videocc MVSC_GDSC>, + <&videocc MVS0_GDSC>, + <&rpmhpd SC7280_CX>; + power-domain-names = "venus", "vcodec0", "cx"; + operating-points-v2 = <&venus_opp_table>; + + interconnects = <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_VENUS_CFG 0>, + <&mmss_noc MASTER_VIDEO_P0 0 &mc_virt SLAVE_EBI1 0>; + interconnect-names = "cpu-cfg", "video-mem"; + + iommus = <&apps_smmu 0x2180 0x20>, + <&apps_smmu 0x2184 0x20>; + memory-region = <&video_mem>; + + video-decoder { + compatible = "venus-decoder"; + }; + + video-encoder { + compatible = "venus-encoder"; + }; + + video-firmware { + iommus = <&apps_smmu 0x21a2 0x0>; + }; + + venus_opp_table: venus-opp-table { + compatible = "operating-points-v2"; + + opp-133330000 { + opp-hz = /bits/ 64 <133330000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-335000000 { + opp-hz = /bits/ 64 <335000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + + opp-424000000 { + opp-hz = /bits/ 64 <424000000>; + required-opps = <&rpmhpd_opp_nom>; + }; + + opp-460000048 { + opp-hz = /bits/ 64 <460000048>; + required-opps = <&rpmhpd_opp_turbo>; + }; + }; + + }; + videocc: clock-controller@aaf0000 { compatible = "qcom,sc7280-videocc"; reg = <0 0xaaf0000 0 0x10000>; @@ -1469,7 +2828,7 @@ aoss_reset: reset-controller@c2a0000 { aoss_qmp: power-controller@c300000 { compatible = "qcom,sc7280-aoss-qmp"; - reg = <0 0x0c300000 0 0x100000>; + reg = <0 0x0c300000 0 0x400>; interrupts-extended = <&ipcc IPCC_CLIENT_AOP IPCC_MPROC_SIGNAL_GLINK_QMP IRQ_TYPE_EDGE_RISING>; @@ -1477,7 +2836,11 @@ IPCC_MPROC_SIGNAL_GLINK_QMP IPCC_MPROC_SIGNAL_GLINK_QMP>; #clock-cells = <0>; - #power-domain-cells = <1>; + }; + + sram@c3f0000 { + compatible = "qcom,rpmh-stats"; + reg = <0 0x0c3f0000 0 0x400>; }; spmi_bus: spmi@c440000 { @@ -1509,11 +2872,518 @@ tlmm: pinctrl@f100000 { gpio-ranges = <&tlmm 0 0 175>; wakeup-parent = <&pdc>; - qup_uart5_default: qup-uart5-default { - pins = "gpio46", "gpio47"; + pcie1_clkreq_n: pcie1-clkreq-n { + pins = "gpio79"; + function = "pcie1_clkreqn"; + drive-strength = <2>; + bias-pull-up; + }; + + qspi_clk: qspi-clk { + pins = "gpio14"; + function = "qspi_clk"; + }; + + qspi_cs0: qspi-cs0 { + pins = "gpio15"; + function = "qspi_cs"; + }; + + qspi_cs1: qspi-cs1 { + pins = "gpio19"; + function = "qspi_cs"; + }; + + qspi_data01: qspi-data01 { + pins = "gpio12", "gpio13"; + function = "qspi_data"; + }; + + qspi_data12: qspi-data12 { + pins = "gpio16", "gpio17"; + function = "qspi_data"; + }; + + qup_i2c0_data_clk: qup-i2c0-data-clk { + pins = "gpio0", "gpio1"; + function = "qup00"; + }; + + qup_i2c1_data_clk: qup-i2c1-data-clk { + pins = "gpio4", "gpio5"; + function = "qup01"; + }; + + qup_i2c2_data_clk: qup-i2c2-data-clk { + pins = "gpio8", "gpio9"; + function = "qup02"; + }; + + qup_i2c3_data_clk: qup-i2c3-data-clk { + pins = "gpio12", "gpio13"; + function = "qup03"; + }; + + qup_i2c4_data_clk: qup-i2c4-data-clk { + pins = "gpio16", "gpio17"; + function = "qup04"; + }; + + qup_i2c5_data_clk: qup-i2c5-data-clk { + pins = "gpio20", "gpio21"; + function = "qup05"; + }; + + qup_i2c6_data_clk: qup-i2c6-data-clk { + pins = "gpio24", "gpio25"; + function = "qup06"; + }; + + qup_i2c7_data_clk: qup-i2c7-data-clk { + pins = "gpio28", "gpio29"; + function = "qup07"; + }; + + qup_i2c8_data_clk: qup-i2c8-data-clk { + pins = "gpio32", "gpio33"; + function = "qup10"; + }; + + qup_i2c9_data_clk: qup-i2c9-data-clk { + pins = "gpio36", "gpio37"; + function = "qup11"; + }; + + qup_i2c10_data_clk: qup-i2c10-data-clk { + pins = "gpio40", "gpio41"; + function = "qup12"; + }; + + qup_i2c11_data_clk: qup-i2c11-data-clk { + pins = "gpio44", "gpio45"; function = "qup13"; }; + qup_i2c12_data_clk: qup-i2c12-data-clk { + pins = "gpio48", "gpio49"; + function = "qup14"; + }; + + qup_i2c13_data_clk: qup-i2c13-data-clk { + pins = "gpio52", "gpio53"; + function = "qup15"; + }; + + qup_i2c14_data_clk: qup-i2c14-data-clk { + pins = "gpio56", "gpio57"; + function = "qup16"; + }; + + qup_i2c15_data_clk: qup-i2c15-data-clk { + pins = "gpio60", "gpio61"; + function = "qup17"; + }; + + qup_spi0_data_clk: qup-spi0-data-clk { + pins = "gpio0", "gpio1", "gpio2"; + function = "qup00"; + }; + + qup_spi0_cs: qup-spi0-cs { + pins = "gpio3"; + function = "qup00"; + }; + + qup_spi0_cs_gpio: qup-spi0-cs-gpio { + pins = "gpio3"; + function = "gpio"; + }; + + qup_spi1_data_clk: qup-spi1-data-clk { + pins = "gpio4", "gpio5", "gpio6"; + function = "qup01"; + }; + + qup_spi1_cs: qup-spi1-cs { + pins = "gpio7"; + function = "qup01"; + }; + + qup_spi1_cs_gpio: qup-spi1-cs-gpio { + pins = "gpio7"; + function = "gpio"; + }; + + qup_spi2_data_clk: qup-spi2-data-clk { + pins = "gpio8", "gpio9", "gpio10"; + function = "qup02"; + }; + + qup_spi2_cs: qup-spi2-cs { + pins = "gpio11"; + function = "qup02"; + }; + + qup_spi2_cs_gpio: qup-spi2-cs-gpio { + pins = "gpio11"; + function = "gpio"; + }; + + qup_spi3_data_clk: qup-spi3-data-clk { + pins = "gpio12", "gpio13", "gpio14"; + function = "qup03"; + }; + + qup_spi3_cs: qup-spi3-cs { + pins = "gpio15"; + function = "qup03"; + }; + + qup_spi3_cs_gpio: qup-spi3-cs-gpio { + pins = "gpio15"; + function = "gpio"; + }; + + qup_spi4_data_clk: qup-spi4-data-clk { + pins = "gpio16", "gpio17", "gpio18"; + function = "qup04"; + }; + + qup_spi4_cs: qup-spi4-cs { + pins = "gpio19"; + function = "qup04"; + }; + + qup_spi4_cs_gpio: qup-spi4-cs-gpio { + pins = "gpio19"; + function = "gpio"; + }; + + qup_spi5_data_clk: qup-spi5-data-clk { + pins = "gpio20", "gpio21", "gpio22"; + function = "qup05"; + }; + + qup_spi5_cs: qup-spi5-cs { + pins = "gpio23"; + function = "qup05"; + }; + + qup_spi5_cs_gpio: qup-spi5-cs-gpio { + pins = "gpio23"; + function = "gpio"; + }; + + qup_spi6_data_clk: qup-spi6-data-clk { + pins = "gpio24", "gpio25", "gpio26"; + function = "qup06"; + }; + + qup_spi6_cs: qup-spi6-cs { + pins = "gpio27"; + function = "qup06"; + }; + + qup_spi6_cs_gpio: qup-spi6-cs-gpio { + pins = "gpio27"; + function = "gpio"; + }; + + qup_spi7_data_clk: qup-spi7-data-clk { + pins = "gpio28", "gpio29", "gpio30"; + function = "qup07"; + }; + + qup_spi7_cs: qup-spi7-cs { + pins = "gpio31"; + function = "qup07"; + }; + + qup_spi7_cs_gpio: qup-spi7-cs-gpio { + pins = "gpio31"; + function = "gpio"; + }; + + qup_spi8_data_clk: qup-spi8-data-clk { + pins = "gpio32", "gpio33", "gpio34"; + function = "qup10"; + }; + + qup_spi8_cs: qup-spi8-cs { + pins = "gpio35"; + function = "qup10"; + }; + + qup_spi8_cs_gpio: qup-spi8-cs-gpio { + pins = "gpio35"; + function = "gpio"; + }; + + qup_spi9_data_clk: qup-spi9-data-clk { + pins = "gpio36", "gpio37", "gpio38"; + function = "qup11"; + }; + + qup_spi9_cs: qup-spi9-cs { + pins = "gpio39"; + function = "qup11"; + }; + + qup_spi9_cs_gpio: qup-spi9-cs-gpio { + pins = "gpio39"; + function = "gpio"; + }; + + qup_spi10_data_clk: qup-spi10-data-clk { + pins = "gpio40", "gpio41", "gpio42"; + function = "qup12"; + }; + + qup_spi10_cs: qup-spi10-cs { + pins = "gpio43"; + function = "qup12"; + }; + + qup_spi10_cs_gpio: qup-spi10-cs-gpio { + pins = "gpio43"; + function = "gpio"; + }; + + qup_spi11_data_clk: qup-spi11-data-clk { + pins = "gpio44", "gpio45", "gpio46"; + function = "qup13"; + }; + + qup_spi11_cs: qup-spi11-cs { + pins = "gpio47"; + function = "qup13"; + }; + + qup_spi11_cs_gpio: qup-spi11-cs-gpio { + pins = "gpio47"; + function = "gpio"; + }; + + qup_spi12_data_clk: qup-spi12-data-clk { + pins = "gpio48", "gpio49", "gpio50"; + function = "qup14"; + }; + + qup_spi12_cs: qup-spi12-cs { + pins = "gpio51"; + function = "qup14"; + }; + + qup_spi12_cs_gpio: qup-spi12-cs-gpio { + pins = "gpio51"; + function = "gpio"; + }; + + qup_spi13_data_clk: qup-spi13-data-clk { + pins = "gpio52", "gpio53", "gpio54"; + function = "qup15"; + }; + + qup_spi13_cs: qup-spi13-cs { + pins = "gpio55"; + function = "qup15"; + }; + + qup_spi13_cs_gpio: qup-spi13-cs-gpio { + pins = "gpio55"; + function = "gpio"; + }; + + qup_spi14_data_clk: qup-spi14-data-clk { + pins = "gpio56", "gpio57", "gpio58"; + function = "qup16"; + }; + + qup_spi14_cs: qup-spi14-cs { + pins = "gpio59"; + function = "qup16"; + }; + + qup_spi14_cs_gpio: qup-spi14-cs-gpio { + pins = "gpio59"; + function = "gpio"; + }; + + qup_spi15_data_clk: qup-spi15-data-clk { + pins = "gpio60", "gpio61", "gpio62"; + function = "qup17"; + }; + + qup_spi15_cs: qup-spi15-cs { + pins = "gpio63"; + function = "qup17"; + }; + + qup_spi15_cs_gpio: qup-spi15-cs-gpio { + pins = "gpio63"; + function = "gpio"; + }; + + qup_uart0_cts: qup-uart0-cts { + pins = "gpio0"; + function = "qup00"; + }; + + qup_uart0_rts: qup-uart0-rts { + pins = "gpio1"; + function = "qup00"; + }; + + qup_uart0_tx: qup-uart0-tx { + pins = "gpio2"; + function = "qup00"; + }; + + qup_uart0_rx: qup-uart0-rx { + pins = "gpio3"; + function = "qup00"; + }; + + qup_uart1_cts: qup-uart1-cts { + pins = "gpio4"; + function = "qup01"; + }; + + qup_uart1_rts: qup-uart1-rts { + pins = "gpio5"; + function = "qup01"; + }; + + qup_uart1_tx: qup-uart1-tx { + pins = "gpio6"; + function = "qup01"; + }; + + qup_uart1_rx: qup-uart1-rx { + pins = "gpio7"; + function = "qup01"; + }; + + qup_uart2_cts: qup-uart2-cts { + pins = "gpio8"; + function = "qup02"; + }; + + qup_uart2_rts: qup-uart2-rts { + pins = "gpio9"; + function = "qup02"; + }; + + qup_uart2_tx: qup-uart2-tx { + pins = "gpio10"; + function = "qup02"; + }; + + qup_uart2_rx: qup-uart2-rx { + pins = "gpio11"; + function = "qup02"; + }; + + qup_uart3_cts: qup-uart3-cts { + pins = "gpio12"; + function = "qup03"; + }; + + qup_uart3_rts: qup-uart3-rts { + pins = "gpio13"; + function = "qup03"; + }; + + qup_uart3_tx: qup-uart3-tx { + pins = "gpio14"; + function = "qup03"; + }; + + qup_uart3_rx: qup-uart3-rx { + pins = "gpio15"; + function = "qup03"; + }; + + qup_uart4_cts: qup-uart4-cts { + pins = "gpio16"; + function = "qup04"; + }; + + qup_uart4_rts: qup-uart4-rts { + pins = "gpio17"; + function = "qup04"; + }; + + qup_uart4_tx: qup-uart4-tx { + pins = "gpio18"; + function = "qup04"; + }; + + qup_uart4_rx: qup-uart4-rx { + pins = "gpio19"; + function = "qup04"; + }; + + qup_uart5_cts: qup-uart5-cts { + pins = "gpio20"; + function = "qup05"; + }; + + qup_uart5_rts: qup-uart5-rts { + pins = "gpio21"; + function = "qup05"; + }; + + qup_uart5_tx: qup-uart5-tx { + pins = "gpio22"; + function = "qup05"; + }; + + qup_uart5_rx: qup-uart5-rx { + pins = "gpio23"; + function = "qup05"; + }; + + qup_uart6_cts: qup-uart6-cts { + pins = "gpio24"; + function = "qup06"; + }; + + qup_uart6_rts: qup-uart6-rts { + pins = "gpio25"; + function = "qup06"; + }; + + qup_uart6_tx: qup-uart6-tx { + pins = "gpio26"; + function = "qup06"; + }; + + qup_uart6_rx: qup-uart6-rx { + pins = "gpio27"; + function = "qup06"; + }; + + qup_uart7_cts: qup-uart7-cts { + pins = "gpio28"; + function = "qup07"; + }; + + qup_uart7_rts: qup-uart7-rts { + pins = "gpio29"; + function = "qup07"; + }; + + qup_uart7_tx: qup-uart7-tx { + pins = "gpio30"; + function = "qup07"; + }; + + qup_uart7_rx: qup-uart7-rx { + pins = "gpio31"; + function = "qup07"; + }; + sdc1_on: sdc1-on { clk { pins = "sdc1_clk"; @@ -1569,10 +3439,6 @@ cmd { data { pins = "sdc2_data"; }; - - sd-cd { - pins = "gpio91"; - }; }; sdc2_off: sdc2-off { @@ -1594,6 +3460,181 @@ data { bias-bus-hold; }; }; + + qup_uart8_cts: qup-uart8-cts { + pins = "gpio32"; + function = "qup10"; + }; + + qup_uart8_rts: qup-uart8-rts { + pins = "gpio33"; + function = "qup10"; + }; + + qup_uart8_tx: qup-uart8-tx { + pins = "gpio34"; + function = "qup10"; + }; + + qup_uart8_rx: qup-uart8-rx { + pins = "gpio35"; + function = "qup10"; + }; + + qup_uart9_cts: qup-uart9-cts { + pins = "gpio36"; + function = "qup11"; + }; + + qup_uart9_rts: qup-uart9-rts { + pins = "gpio37"; + function = "qup11"; + }; + + qup_uart9_tx: qup-uart9-tx { + pins = "gpio38"; + function = "qup11"; + }; + + qup_uart9_rx: qup-uart9-rx { + pins = "gpio39"; + function = "qup11"; + }; + + qup_uart10_cts: qup-uart10-cts { + pins = "gpio40"; + function = "qup12"; + }; + + qup_uart10_rts: qup-uart10-rts { + pins = "gpio41"; + function = "qup12"; + }; + + qup_uart10_tx: qup-uart10-tx { + pins = "gpio42"; + function = "qup12"; + }; + + qup_uart10_rx: qup-uart10-rx { + pins = "gpio43"; + function = "qup12"; + }; + + qup_uart11_cts: qup-uart11-cts { + pins = "gpio44"; + function = "qup13"; + }; + + qup_uart11_rts: qup-uart11-rts { + pins = "gpio45"; + function = "qup13"; + }; + + qup_uart11_tx: qup-uart11-tx { + pins = "gpio46"; + function = "qup13"; + }; + + qup_uart11_rx: qup-uart11-rx { + pins = "gpio47"; + function = "qup13"; + }; + + qup_uart12_cts: qup-uart12-cts { + pins = "gpio48"; + function = "qup14"; + }; + + qup_uart12_rts: qup-uart12-rts { + pins = "gpio49"; + function = "qup14"; + }; + + qup_uart12_tx: qup-uart12-tx { + pins = "gpio50"; + function = "qup14"; + }; + + qup_uart12_rx: qup-uart12-rx { + pins = "gpio51"; + function = "qup14"; + }; + + qup_uart13_cts: qup-uart13-cts { + pins = "gpio52"; + function = "qup15"; + }; + + qup_uart13_rts: qup-uart13-rts { + pins = "gpio53"; + function = "qup15"; + }; + + qup_uart13_tx: qup-uart13-tx { + pins = "gpio54"; + function = "qup15"; + }; + + qup_uart13_rx: qup-uart13-rx { + pins = "gpio55"; + function = "qup15"; + }; + + qup_uart14_cts: qup-uart14-cts { + pins = "gpio56"; + function = "qup16"; + }; + + qup_uart14_rts: qup-uart14-rts { + pins = "gpio57"; + function = "qup16"; + }; + + qup_uart14_tx: qup-uart14-tx { + pins = "gpio58"; + function = "qup16"; + }; + + qup_uart14_rx: qup-uart14-rx { + pins = "gpio59"; + function = "qup16"; + }; + + qup_uart15_cts: qup-uart15-cts { + pins = "gpio60"; + function = "qup17"; + }; + + qup_uart15_rts: qup-uart15-rts { + pins = "gpio61"; + function = "qup17"; + }; + + qup_uart15_tx: qup-uart15-tx { + pins = "gpio62"; + function = "qup17"; + }; + + qup_uart15_rx: qup-uart15-rx { + pins = "gpio63"; + function = "qup17"; + }; + }; + + imem@146a5000 { + compatible = "qcom,sc7280-imem", "syscon"; + reg = <0 0x146a5000 0 0x6000>; + + #address-cells = <1>; + #size-cells = <1>; + + ranges = <0 0 0x146a5000 0x6000>; + + pil-reloc@594c { + compatible = "qcom,pil-reloc-info"; + reg = <0x594c 0xc8>; + }; }; apps_smmu: iommu@15000000 { @@ -2467,16 +4508,16 @@ cpuss1_crit: cluster0-crit { }; gpuss0-thermal { - polling-delay-passive = <0>; + polling-delay-passive = <100>; polling-delay = <0>; thermal-sensors = <&tsens1 1>; trips { gpuss0_alert0: trip-point0 { - temperature = <90000>; + temperature = <95000>; hysteresis = <2000>; - type = "hot"; + type = "passive"; }; gpuss0_crit: gpuss0-crit { @@ -2485,19 +4526,26 @@ gpuss0_crit: gpuss0-crit { type = "critical"; }; }; + + cooling-maps { + map0 { + trip = <&gpuss0_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; gpuss1-thermal { - polling-delay-passive = <0>; + polling-delay-passive = <100>; polling-delay = <0>; thermal-sensors = <&tsens1 2>; trips { gpuss1_alert0: trip-point0 { - temperature = <90000>; + temperature = <95000>; hysteresis = <2000>; - type = "hot"; + type = "passive"; }; gpuss1_crit: gpuss1-crit { @@ -2506,6 +4554,13 @@ gpuss1_crit: gpuss1-crit { type = "critical"; }; }; + + cooling-maps { + map0 { + trip = <&gpuss1_alert0>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; }; nspss0-thermal { diff --git a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-ganges-kirin.dts b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-ganges-kirin.dts index a4e1fb8ca5..71b448978e 100644 --- a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-ganges-kirin.dts +++ b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-ganges-kirin.dts @@ -11,6 +11,7 @@ / { model = "Sony Xperia 10"; compatible = "sony,kirin-row", "qcom,sdm630"; + chassis-type = "handset"; chosen { framebuffer@9d400000 { diff --git a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-discovery.dts b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-discovery.dts index c574e430ba..dd484a9bc7 100644 --- a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-discovery.dts +++ b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-discovery.dts @@ -11,4 +11,5 @@ / { model = "Sony Xperia XA2 Ultra"; compatible = "sony,discovery-row", "qcom,sdm630"; + chassis-type = "handset"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-pioneer.dts b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-pioneer.dts index a93ff3ab1b..2da83cd411 100644 --- a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-pioneer.dts +++ b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-pioneer.dts @@ -11,4 +11,5 @@ / { model = "Sony Xperia XA2"; compatible = "sony,pioneer-row", "qcom,sdm630"; + chassis-type = "handset"; }; diff --git a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-voyager.dts b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-voyager.dts index 59a679c205..a679d4acee 100644 --- a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-voyager.dts +++ b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile-voyager.dts @@ -11,6 +11,7 @@ / { model = "Sony Xperia XA2 Plus"; compatible = "sony,voyager-row", "qcom,sdm630"; + chassis-type = "handset"; chosen { framebuffer@9d400000 { diff --git a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile.dtsi b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile.dtsi index 849900e8b8..42af1fade4 100644 --- a/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm630-sony-xperia-nile.dtsi @@ -90,7 +90,6 @@ cam_vana_rear_vreg: cam_vana_rear_vreg { gpio_keys { status = "okay"; compatible = "gpio-keys"; - input-name = "gpio-keys"; #address-cells = <1>; #size-cells = <0>; @@ -215,14 +214,14 @@ &blsp2_uart1 { /* HCI Bluetooth */ }; -&pon { - volup { - compatible = "qcom,pm8941-resin"; - interrupts = <0x0 0x8 1 IRQ_TYPE_EDGE_BOTH>; - debounce = <15625>; - bias-pull-up; - linux,code = ; - }; +&pon_pwrkey { + status = "okay"; +}; + +&pon_resin { + status = "okay"; + + linux,code = ; }; &qusb2phy { @@ -308,7 +307,7 @@ vreg_l3b_3p0: l3 { regulator-allow-set-load; }; - vreg_l4b_29p5: l4 { + vreg_l4b_2p95: l4 { regulator-min-microvolt = <2944000>; regulator-max-microvolt = <2952000>; regulator-enable-ramp-delay = <250>; @@ -327,7 +326,7 @@ vreg_l4b_29p5: l4 { * Tighten the range to 1.8-3.328 (closest to 3.3) to * make the mmc driver happy. */ - vreg_l5b_29p5: l5 { + vreg_l5b_2p95: l5 { regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3328000>; regulator-enable-ramp-delay = <250>; @@ -559,14 +558,14 @@ &sdhc_1 { mmc-hs400-1_8v; mmc-hs400-enhanced-strobe; - vmmc-supply = <&vreg_l4b_29p5>; + vmmc-supply = <&vreg_l4b_2p95>; vqmmc-supply = <&vreg_l8a_1p8>; }; &sdhc_2 { status = "okay"; - vmmc-supply = <&vreg_l5b_29p5>; + vmmc-supply = <&vreg_l5b_2p95>; vqmmc-supply = <&vreg_l2b_2p95>; }; diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi index a8724fd606..9217c3a51f 100644 --- a/arch/arm64/boot/dts/qcom/sdm630.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi @@ -19,6 +19,11 @@ / { #address-cells = <2>; #size-cells = <2>; + aliases { + mmc1 = &sdhc_1; + mmc2 = &sdhc_2; + }; + chosen { }; clocks { @@ -1014,7 +1019,6 @@ sd-cd { adreno_gpu: gpu@5000000 { compatible = "qcom,adreno-508.0", "qcom,adreno"; - #stream-id-cells = <16>; reg = <0x05000000 0x40000>; reg-names = "kgsl_3d0_reg_memory"; @@ -1176,6 +1180,11 @@ lpass_smmu: iommu@5100000 { status = "disabled"; }; + sram@290000 { + compatible = "qcom,rpm-stats"; + reg = <0x00290000 0x10000>; + }; + spmi_bus: spmi@800f000 { compatible = "qcom,spmi-pmic-arb"; reg = <0x0800f000 0x1000>, @@ -2020,6 +2029,57 @@ cci_i2c1: i2c-bus@1 { }; }; + venus: video-codec@cc00000 { + compatible = "qcom,sdm660-venus"; + reg = <0x0cc00000 0xff000>; + clocks = <&mmcc VIDEO_CORE_CLK>, + <&mmcc VIDEO_AHB_CLK>, + <&mmcc VIDEO_AXI_CLK>, + <&mmcc THROTTLE_VIDEO_AXI_CLK>; + clock-names = "core", "iface", "bus", "bus_throttle"; + interconnects = <&gnoc 0 &mnoc 13>, + <&mnoc 4 &bimc 5>; + interconnect-names = "cpu-cfg", "video-mem"; + interrupts = ; + iommus = <&mmss_smmu 0x400>, + <&mmss_smmu 0x401>, + <&mmss_smmu 0x40a>, + <&mmss_smmu 0x407>, + <&mmss_smmu 0x40e>, + <&mmss_smmu 0x40f>, + <&mmss_smmu 0x408>, + <&mmss_smmu 0x409>, + <&mmss_smmu 0x40b>, + <&mmss_smmu 0x40c>, + <&mmss_smmu 0x40d>, + <&mmss_smmu 0x410>, + <&mmss_smmu 0x421>, + <&mmss_smmu 0x428>, + <&mmss_smmu 0x429>, + <&mmss_smmu 0x42b>, + <&mmss_smmu 0x42c>, + <&mmss_smmu 0x42d>, + <&mmss_smmu 0x411>, + <&mmss_smmu 0x431>; + memory-region = <&venus_region>; + power-domains = <&mmcc VENUS_GDSC>; + status = "disabled"; + + video-decoder { + compatible = "venus-decoder"; + clocks = <&mmcc VIDEO_SUBCORE0_CLK>; + clock-names = "vcodec0_core"; + power-domains = <&mmcc VENUS_CORE0_GDSC>; + }; + + video-encoder { + compatible = "venus-encoder"; + clocks = <&mmcc VIDEO_SUBCORE0_CLK>; + clock-names = "vcodec0_core"; + power-domains = <&mmcc VENUS_CORE0_GDSC>; + }; + }; + mmss_smmu: iommu@cd00000 { compatible = "qcom,sdm630-smmu-v2", "qcom,smmu-v2"; reg = <0x0cd00000 0x40000>; diff --git a/arch/arm64/boot/dts/qcom/sdm636-sony-xperia-ganges-mermaid.dts b/arch/arm64/boot/dts/qcom/sdm636-sony-xperia-ganges-mermaid.dts index bba1c2bce2..b96da53f2f 100644 --- a/arch/arm64/boot/dts/qcom/sdm636-sony-xperia-ganges-mermaid.dts +++ b/arch/arm64/boot/dts/qcom/sdm636-sony-xperia-ganges-mermaid.dts @@ -11,6 +11,7 @@ / { model = "Sony Xperia 10 Plus"; compatible = "sony,mermaid-row", "qcom,sdm636"; + chassis-type = "handset"; /* SDM636 v1 */ qcom,msm-id = <345 0>; diff --git a/arch/arm64/boot/dts/qcom/sdm660-xiaomi-lavender.dts b/arch/arm64/boot/dts/qcom/sdm660-xiaomi-lavender.dts index 3e677fb7cf..dcbaacf18f 100644 --- a/arch/arm64/boot/dts/qcom/sdm660-xiaomi-lavender.dts +++ b/arch/arm64/boot/dts/qcom/sdm660-xiaomi-lavender.dts @@ -1,22 +1,62 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2020, Alexey Minnekhanov + * Copyright (c) 2021, Dang Huynh */ /dts-v1/; #include "sdm660.dtsi" +#include "pm660.dtsi" +#include "pm660l.dtsi" +#include +#include / { model = "Xiaomi Redmi Note 7"; compatible = "xiaomi,lavender", "qcom,sdm660"; + chassis-type = "handset"; aliases { serial0 = &blsp1_uart2; }; chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + stdout-path = "serial0:115200n8"; + + framebuffer0: framebuffer@9d400000 { + compatible = "simple-framebuffer"; + reg = <0 0x9d400000 0 (1080 * 2340 * 4)>; + width = <1080>; + height = <2340>; + stride = <(1080 * 4)>; + format = "a8r8g8b8"; + }; + }; + + vph_pwr: vph-pwr-regulator { + compatible = "regulator-fixed"; + regulator-name = "vph_pwr"; + regulator-min-microvolt = <3700000>; + regulator-max-microvolt = <3700000>; + + regulator-always-on; + regulator-boot-on; + }; + + gpio-keys { + compatible = "gpio-keys"; + + volup { + label = "Volume Up"; + gpios = <&pm660l_gpios 7 GPIO_ACTIVE_LOW>; + linux,code = ; + debounce-interval = <15>; + }; }; reserved-memory { @@ -32,6 +72,20 @@ ramoops@a0000000 { ftrace-size = <0x0>; pmsg-size = <0x20000>; }; + + framebuffer_mem: memory@9d400000 { + reg = <0x0 0x9d400000 0x0 0x23ff000>; + no-map; + }; + }; + + /* + * Until we hook up type-c detection, we + * have to stick with this. But it works. + */ + extcon_usb: extcon-usb { + compatible = "linux,extcon-usb-gpio"; + id-gpio = <&tlmm 58 GPIO_ACTIVE_HIGH>; }; }; @@ -39,6 +93,304 @@ &blsp1_uart2 { status = "okay"; }; +&pon_pwrkey { + status = "okay"; +}; + +&pon_resin { + status = "okay"; + + linux,code = ; +}; + +&qusb2phy { + status = "okay"; + + vdd-supply = <&vreg_l1b_0p925>; + vdda-phy-dpdm-supply = <&vreg_l7b_3p125>; +}; + +&rpm_requests { + pm660l-regulators { + compatible = "qcom,rpm-pm660l-regulators"; + + vdd_s1-supply = <&vph_pwr>; + vdd_s2-supply = <&vph_pwr>; + vdd_s3_s4-supply = <&vph_pwr>; + vdd_s5-supply = <&vph_pwr>; + vdd_s6-supply = <&vph_pwr>; + + vdd_l1_l9_l10-supply = <&vreg_s2b_1p05>; + vdd_l2-supply = <&vreg_bob>; + vdd_l3_l5_l7_l8-supply = <&vreg_bob>; + vdd_l4_l6-supply = <&vreg_bob>; + vdd_bob-supply = <&vph_pwr>; + + vreg_s1b_1p125: s1 { + regulator-min-microvolt = <1125000>; + regulator-max-microvolt = <1125000>; + regulator-enable-ramp-delay = <200>; + }; + + vreg_s2b_1p05: s2 { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-enable-ramp-delay = <200>; + }; + + /* LDOs */ + vreg_l1b_0p925: l1 { + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <925000>; + regulator-enable-ramp-delay = <250>; + regulator-allow-set-load; + }; + + /* SDHCI 3.3V signal doesn't seem to be supported. */ + vreg_l2b_2p95: l2 { + regulator-min-microvolt = <1648000>; + regulator-max-microvolt = <2696000>; + regulator-enable-ramp-delay = <250>; + regulator-allow-set-load; + }; + + vreg_l3b_3p3: l3 { + regulator-min-microvolt = <1700000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <250>; + regulator-allow-set-load; + }; + + vreg_l4b_2p95: l4 { + regulator-min-microvolt = <2944000>; + regulator-max-microvolt = <2952000>; + regulator-enable-ramp-delay = <250>; + + regulator-min-microamp = <200>; + regulator-max-microamp = <600000>; + regulator-system-load = <570000>; + regulator-allow-set-load; + }; + + /* + * Downstream specifies a range of 1721-3600mV, + * but the only assigned consumers are SDHCI2 VMMC + * and Coresight QPDI that both request pinned 2.95V. + * Tighten the range to 1.8-3.328 (closest to 3.3) to + * make the mmc driver happy. + */ + vreg_l5b_2p95: l5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3328000>; + regulator-enable-ramp-delay = <250>; + regulator-allow-set-load; + regulator-system-load = <800000>; + }; + + vreg_l7b_3p125: l7 { + regulator-min-microvolt = <2700000>; + regulator-max-microvolt = <3125000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l8b_3p3: l8 { + regulator-min-microvolt = <3200000>; + regulator-max-microvolt = <3400000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_bob: bob { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3600000>; + regulator-enable-ramp-delay = <500>; + }; + }; + + pm660-regulators { + compatible = "qcom,rpm-pm660-regulators"; + + vdd_s1-supply = <&vph_pwr>; + vdd_s2-supply = <&vph_pwr>; + vdd_s3-supply = <&vph_pwr>; + vdd_s4-supply = <&vph_pwr>; + vdd_s5-supply = <&vph_pwr>; + vdd_s6-supply = <&vph_pwr>; + + vdd_l1_l6_l7-supply = <&vreg_s5a_1p35>; + vdd_l2_l3-supply = <&vreg_s2b_1p05>; + vdd_l5-supply = <&vreg_s2b_1p05>; + vdd_l8_l9_l10_l11_l12_l13_l14-supply = <&vreg_s4a_2p04>; + vdd_l15_l16_l17_l18_l19-supply = <&vreg_bob>; + + /* + * S1A (FTAPC0), S2A (FTAPC1), S3A (HFAPC1) are managed + * by the Core Power Reduction hardened (CPRh) and the + * Operating State Manager (OSM) HW automatically. + */ + + vreg_s4a_2p04: s4 { + regulator-min-microvolt = <1805000>; + regulator-max-microvolt = <2040000>; + regulator-enable-ramp-delay = <200>; + regulator-always-on; + }; + + vreg_s5a_1p35: s5 { + regulator-min-microvolt = <1224000>; + regulator-max-microvolt = <1350000>; + regulator-enable-ramp-delay = <200>; + }; + + vreg_s6a_0p87: s6 { + regulator-min-microvolt = <504000>; + regulator-max-microvolt = <992000>; + regulator-enable-ramp-delay = <150>; + }; + + /* LDOs */ + vreg_l1a_1p225: l1 { + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1250000>; + regulator-enable-ramp-delay = <250>; + regulator-allow-set-load; + }; + + vreg_l2a_1p0: l2 { + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1010000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l3a_1p0: l3 { + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1010000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l5a_0p848: l5 { + regulator-min-microvolt = <525000>; + regulator-max-microvolt = <950000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l6a_1p3: l6 { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1370000>; + regulator-allow-set-load; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l7a_1p2: l7 { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l8a_1p8: l8 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <1800000>; + regulator-enable-ramp-delay = <250>; + regulator-system-load = <325000>; + regulator-allow-set-load; + }; + + vreg_l9a_1p8: l9 { + regulator-min-microvolt = <1750000>; + regulator-max-microvolt = <1900000>; + regulator-enable-ramp-delay = <250>; + regulator-allow-set-load; + }; + + vreg_l10a_1p8: l10 { + regulator-min-microvolt = <1780000>; + regulator-max-microvolt = <1950000>; + regulator-enable-ramp-delay = <250>; + regulator-allow-set-load; + }; + + vreg_l11a_1p8: l11 { + regulator-min-microvolt = <1780000>; + regulator-max-microvolt = <1950000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l12a_1p8: l12 { + regulator-min-microvolt = <1780000>; + regulator-max-microvolt = <1950000>; + regulator-enable-ramp-delay = <250>; + }; + + /* This gives power to the LPDDR4: never turn it off! */ + vreg_l13a_1p8: l13 { + regulator-min-microvolt = <1780000>; + regulator-max-microvolt = <1950000>; + regulator-enable-ramp-delay = <250>; + regulator-boot-on; + regulator-always-on; + }; + + vreg_l14a_1p8: l14 { + regulator-min-microvolt = <1710000>; + regulator-max-microvolt = <1900000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l15a_1p8: l15 { + regulator-min-microvolt = <1650000>; + regulator-max-microvolt = <2950000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l16a_2p7: l16 { + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-enable-ramp-delay = <250>; + regulator-always-on; + }; + + vreg_l17a_1p8: l17 { + regulator-min-microvolt = <1648000>; + regulator-max-microvolt = <2952000>; + regulator-enable-ramp-delay = <250>; + }; + + vreg_l19a_3p3: l19 { + regulator-min-microvolt = <3312000>; + regulator-max-microvolt = <3400000>; + regulator-enable-ramp-delay = <250>; + regulator-allow-set-load; + }; + }; +}; + +&sdhc_1 { + status = "okay"; + supports-cqe; + + mmc-hs200-1_8v; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + + vmmc-supply = <&vreg_l4b_2p95>; + vqmmc-supply = <&vreg_l8a_1p8>; +}; + +&sdhc_2 { + status = "okay"; + + vmmc-supply = <&vreg_l5b_2p95>; + vqmmc-supply = <&vreg_l2b_2p95>; +}; + &tlmm { gpio-reserved-ranges = <8 4>; }; + +&usb3 { + status = "okay"; +}; + +&usb3_dwc3 { + dr_mode = "peripheral"; + extcon = <&extcon_usb>; +}; diff --git a/arch/arm64/boot/dts/qcom/sdm845-cheza.dtsi b/arch/arm64/boot/dts/qcom/sdm845-cheza.dtsi index dfd1b42c07..4a6285a25f 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-cheza.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-cheza.dtsi @@ -5,7 +5,6 @@ * Copyright 2018 Google LLC. */ -#include #include #include #include "sdm845.dtsi" @@ -616,6 +615,14 @@ touchscreen@10 { }; }; +&gmu { + status = "okay"; +}; + +&gpu { + status = "okay"; +}; + &ipa { status = "okay"; modem-init; @@ -629,10 +636,6 @@ &mdss { status = "okay"; }; -&mdss_mdp { - status = "okay"; -}; - /* * Cheza fw does not properly program the GPU aperture to allow the * GPU to update the SMMU pagetables for context switches. Work @@ -643,6 +646,8 @@ &adreno_smmu { }; &mss_pil { + status = "okay"; + iommus = <&apps_smmu 0x781 0x0>, <&apps_smmu 0x724 0x3>; }; @@ -1317,6 +1322,8 @@ config { }; &venus { + status = "okay"; + video-firmware { iommus = <&apps_smmu 0x10b2 0x0>; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts index 2d5533dd4e..13f80a0b6f 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts @@ -5,7 +5,6 @@ /dts-v1/; -#include #include #include #include @@ -17,6 +16,8 @@ / { model = "Thundercomm Dragonboard 845c"; compatible = "thundercomm,db845c", "qcom,sdm845"; + qcom,msm-id = <341 0x20001>; + qcom,board-id = <8 0>; aliases { serial0 = &uart9; @@ -420,7 +421,12 @@ &gcc { ; }; +&gmu { + status = "okay"; +}; + &gpu { + status = "okay"; zap-shader { memory-region = <&gpu_mem>; firmware-name = "qcom/sdm845/a630_zap.mbn"; @@ -485,10 +491,6 @@ &mdss { status = "okay"; }; -&mdss_mdp { - status = "okay"; -}; - &mss_pil { status = "okay"; firmware-name = "qcom/sdm845/mba.mbn", "qcom/sdm845/modem.mbn"; @@ -999,6 +1001,10 @@ &ufs_mem_phy { vdda-pll-supply = <&vreg_l26a_1p2>; }; +&venus { + status = "okay"; +}; + &wcd9340{ pinctrl-0 = <&wcd_intr_default>; pinctrl-names = "default"; diff --git a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts index 52dd7a8582..7713e8060c 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts @@ -7,9 +7,10 @@ /dts-v1/; -#include #include #include "sdm845.dtsi" +#include "pm8998.dtsi" +#include "pmi8998.dtsi" / { model = "Qualcomm Technologies, Inc. SDM845 MTP"; @@ -46,6 +47,68 @@ vreg_s4a_1p8: pm8998-smps4 { vin-supply = <&vph_pwr>; }; + + thermal-zones { + xo_thermal: xo-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + + thermal-sensors = <&pm8998_adc_tm 1>; + + trips { + trip-point { + temperature = <125000>; + hysteresis = <10000>; + type = "passive"; + }; + }; + }; + + msm_thermal: msm-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + + thermal-sensors = <&pm8998_adc_tm 2>; + + trips { + trip-point { + temperature = <125000>; + hysteresis = <10000>; + type = "passive"; + }; + }; + }; + + pa_thermal: pa-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + + thermal-sensors = <&pm8998_adc_tm 3>; + + trips { + trip-point { + temperature = <125000>; + hysteresis = <10000>; + type = "passive"; + }; + }; + }; + + quiet_thermal: quiet-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + + thermal-sensors = <&pm8998_adc_tm 4>; + + trips { + trip-point { + temperature = <125000>; + hysteresis = <10000>; + type = "passive"; + }; + }; + }; + }; }; &adsp_pas { @@ -361,9 +424,6 @@ &dsi0 { qcom,dual-dsi-mode; qcom,master-dsi; - #address-cells = <1>; - #size-cells = <0>; - ports { port@1 { endpoint { @@ -439,7 +499,13 @@ &gcc { ; }; +&gmu { + status = "okay"; +}; + &gpu { + status = "okay"; + zap-shader { memory-region = <&gpu_mem>; firmware-name = "qcom/sdm845/a630_zap.mbn"; @@ -460,15 +526,87 @@ &mdss { status = "okay"; }; -&mdss_mdp { - status = "okay"; -}; - &mss_pil { status = "okay"; firmware-name = "qcom/sdm845/mba.mbn", "qcom/sdm845/modem.mbn"; }; +&pm8998_adc { + adc-chan@4c { + reg = ; + label = "xo_therm"; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + + adc-chan@4d { + reg = ; + label = "msm_therm"; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + + adc-chan@4f { + reg = ; + label = "pa_therm1"; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + + adc-chan@51 { + reg = ; + label = "quiet_therm"; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + + adc-chan@83 { + reg = ; + label = "vph_pwr"; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + + adc-chan@85 { + reg = ; + label = "vcoin"; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; +}; + +&pm8998_adc_tm { + status = "okay"; + + xo-thermistor@1 { + reg = <1>; + io-channels = <&pm8998_adc ADC5_XO_THERM_100K_PU>; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + + msm-thermistor@2 { + reg = <2>; + io-channels = <&pm8998_adc ADC5_AMUX_THM1_100K_PU>; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + + pa-thermistor@3 { + reg = <3>; + io-channels = <&pm8998_adc ADC5_AMUX_THM3_100K_PU>; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + + quiet-thermistor@4 { + reg = <4>; + io-channels = <&pm8998_adc ADC5_AMUX_THM5_100K_PU>; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; +}; + &qupv3_id_1 { status = "okay"; }; @@ -566,6 +704,10 @@ &usb_2_qmpphy { vdda-pll-supply = <&vdda_usb2_ss_core>; }; +&venus { + status = "okay"; +}; + &wifi { status = "okay"; vdd-0.8-cx-mx-supply = <&vreg_l5a_0p8>; diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi index d435552237..7f42e5315e 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi @@ -7,7 +7,6 @@ /dts-v1/; -#include #include #include @@ -93,7 +92,6 @@ ramoops: ramoops@ac300000 { console-size = <0x40000>; ftrace-size = <0x40000>; pmsg-size = <0x200000>; - devinfo-size = <0x1000>; ecc-size = <16>; }; }; @@ -248,6 +246,12 @@ vreg_l20a_2p95: ldo20 { regulator-initial-mode = ; }; + vreg_l23a_3p3: ldo23 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3312000>; + regulator-initial-mode = ; + }; + vdda_qusb_hs0_3p1: vreg_l24a_3p075: ldo24 { regulator-min-microvolt = <3088000>; @@ -315,9 +319,6 @@ &dsi0 { status = "okay"; vdda-supply = <&vdda_mipi_dsi0_1p2>; - #address-cells = <1>; - #size-cells = <0>; - /* * Both devices use different panels but all other properties * are common. Compatible line is declared in device dts. @@ -362,7 +363,13 @@ &gcc { ; }; +&gmu { + status = "okay"; +}; + &gpu { + status = "okay"; + zap-shader { memory-region = <&gpu_mem>; firmware-name = "qcom/sdm845/oneplus6/a630_zap.mbn"; @@ -415,10 +422,6 @@ &mdss { status = "okay"; }; -&mdss_mdp { - status = "okay"; -}; - /* Modem/wifi*/ &mss_pil { status = "okay"; @@ -641,12 +644,18 @@ mux { }; }; +&venus { + status = "okay"; + firmware-name = "qcom/sdm845/oneplus6/venus.mbn"; +}; + &wifi { status = "okay"; vdd-0.8-cx-mx-supply = <&vreg_l5a_0p8>; vdd-1.8-xo-supply = <&vreg_l7a_1p8>; vdd-1.3-rfa-supply = <&vreg_l17a_1p3>; vdd-3.3-ch0-supply = <&vreg_l25a_3p3>; + vdd-3.3-ch1-supply = <&vreg_l23a_3p3>; qcom,snoc-host-cap-8bit-quirk; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts b/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts index 72842c8876..5936b47dee 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts @@ -10,6 +10,9 @@ / { model = "OnePlus 6"; compatible = "oneplus,enchilada", "qcom,sdm845"; + chassis-type = "handset"; + qcom,msm-id = <0x141 0x20001>; + qcom,board-id = <8 0 17819 22>; }; &display_panel { diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-fajita.dts b/arch/arm64/boot/dts/qcom/sdm845-oneplus-fajita.dts index 969b36dc9e..78a0b99144 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-fajita.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-fajita.dts @@ -10,6 +10,9 @@ / { model = "OnePlus 6T"; compatible = "oneplus,fajita", "qcom,sdm845"; + chassis-type = "handset"; + qcom,msm-id = <0x141 0x20001>; + qcom,board-id = <8 0 18801 41>; }; &display_panel { diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts index c60c8c640e..367389526b 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts @@ -2,7 +2,6 @@ /dts-v1/; -#include #include #include #include @@ -29,6 +28,7 @@ / { model = "Xiaomi Pocophone F1"; compatible = "xiaomi,beryllium", "qcom,sdm845"; + chassis-type = "handset"; /* required for bootloader to select correct board */ qcom,board-id = <69 0>; @@ -215,9 +215,6 @@ &dsi0 { status = "okay"; vdda-supply = <&vreg_l26a_1p2>; - #address-cells = <1>; - #size-cells = <0>; - panel@0 { compatible = "tianma,fhd-video"; reg = <0>; @@ -256,7 +253,13 @@ &gcc { ; }; +&gmu { + status = "okay"; +}; + &gpu { + status = "okay"; + zap-shader { memory-region = <&gpu_mem>; firmware-name = "qcom/sdm845/a630_zap.mbn"; @@ -284,10 +287,6 @@ &mdss { status = "okay"; }; -&mdss_mdp { - status = "okay"; -}; - &mss_pil { status = "okay"; firmware-name = "qcom/sdm845/mba.mbn", "qcom/sdm845/modem.mdt"; @@ -513,6 +512,11 @@ &usb_1_qmpphy { vdda-pll-supply = <&vreg_l1a_0p875>; }; +&venus { + status = "okay"; + firmware-name = "qcom/sdm845/beryllium/venus.mbn"; +}; + &wcd9340{ pinctrl-0 = <&wcd_intr_default>; pinctrl-names = "default"; diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi index 519ca9a705..cfdeaa81f1 100644 --- a/arch/arm64/boot/dts/qcom/sdm845.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -99,9 +100,11 @@ aop_cmd_db_mem: memory@85fe0000 { no-map; }; - smem_mem: memory@86000000 { + smem@86000000 { + compatible = "qcom,smem"; reg = <0x0 0x86000000 0 0x200000>; no-map; + hwlocks = <&tcsr_mutex 3>; }; tz_mem: memory@86200000 { @@ -768,6 +771,8 @@ adsp_pas: remoteproc-adsp { memory-region = <&adsp_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&adsp_smp2p_out 0>; qcom,smem-state-names = "stop"; @@ -867,6 +872,8 @@ cdsp_pas: remoteproc-cdsp { memory-region = <&cdsp_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&cdsp_smp2p_out 0>; qcom,smem-state-names = "stop"; @@ -941,12 +948,6 @@ tcsr_mutex: hwlock { #hwlock-cells = <1>; }; - smem { - compatible = "qcom,smem"; - memory-region = <&smem_mem>; - hwlocks = <&tcsr_mutex 3>; - }; - smp2p-cdsp { compatible = "qcom,smp2p"; qcom,smem = <94>, <432>; @@ -1077,7 +1078,7 @@ gcc: clock-controller@100000 { }; qfprom@784000 { - compatible = "qcom,qfprom"; + compatible = "qcom,sdm845-qfprom", "qcom,qfprom"; reg = <0 0x00784000 0 0x8ff>; #address-cells = <1>; #size-cells = <1>; @@ -1995,10 +1996,10 @@ pcie0: pci@1c00000 { interrupt-names = "msi"; #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc 0 149 IRQ_TYPE_LEVEL_HIGH>, /* int_a */ - <0 0 0 2 &intc 0 150 IRQ_TYPE_LEVEL_HIGH>, /* int_b */ - <0 0 0 3 &intc 0 151 IRQ_TYPE_LEVEL_HIGH>, /* int_c */ - <0 0 0 4 &intc 0 152 IRQ_TYPE_LEVEL_HIGH>; /* int_d */ + interrupt-map = <0 0 0 1 &intc 0 0 0 149 IRQ_TYPE_LEVEL_HIGH>, /* int_a */ + <0 0 0 2 &intc 0 0 0 150 IRQ_TYPE_LEVEL_HIGH>, /* int_b */ + <0 0 0 3 &intc 0 0 0 151 IRQ_TYPE_LEVEL_HIGH>, /* int_c */ + <0 0 0 4 &intc 0 0 0 152 IRQ_TYPE_LEVEL_HIGH>; /* int_d */ clocks = <&gcc GCC_PCIE_0_PIPE_CLK>, <&gcc GCC_PCIE_0_AUX_CLK>, @@ -2064,7 +2065,7 @@ pcie0_phy: phy@1c06000 { status = "disabled"; - pcie0_lane: lanes@1c06200 { + pcie0_lane: phy@1c06200 { reg = <0 0x01c06200 0 0x128>, <0 0x01c06400 0 0x1fc>, <0 0x01c06800 0 0x218>, @@ -2100,10 +2101,10 @@ pcie1: pci@1c08000 { interrupt-names = "msi"; #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc 0 434 IRQ_TYPE_LEVEL_HIGH>, /* int_a */ - <0 0 0 2 &intc 0 435 IRQ_TYPE_LEVEL_HIGH>, /* int_b */ - <0 0 0 3 &intc 0 438 IRQ_TYPE_LEVEL_HIGH>, /* int_c */ - <0 0 0 4 &intc 0 439 IRQ_TYPE_LEVEL_HIGH>; /* int_d */ + interrupt-map = <0 0 0 1 &intc 0 0 0 434 IRQ_TYPE_LEVEL_HIGH>, /* int_a */ + <0 0 0 2 &intc 0 0 0 435 IRQ_TYPE_LEVEL_HIGH>, /* int_b */ + <0 0 0 3 &intc 0 0 0 438 IRQ_TYPE_LEVEL_HIGH>, /* int_c */ + <0 0 0 4 &intc 0 0 0 439 IRQ_TYPE_LEVEL_HIGH>; /* int_d */ clocks = <&gcc GCC_PCIE_1_PIPE_CLK>, <&gcc GCC_PCIE_1_AUX_CLK>, @@ -2174,7 +2175,7 @@ pcie1_phy: phy@1c0a000 { status = "disabled"; - pcie1_lane: lanes@1c06200 { + pcie1_lane: phy@1c06200 { reg = <0 0x01c0a800 0 0x800>, <0 0x01c0a800 0 0x800>, <0 0x01c0b800 0 0x400>; @@ -2302,7 +2303,7 @@ ufs_mem_phy: phy@1d87000 { reset-names = "ufsphy"; status = "disabled"; - ufs_mem_phy_lanes: lanes@1d87400 { + ufs_mem_phy_lanes: phy@1d87400 { reg = <0 0x01d87400 0 0x108>, <0 0x01d87600 0 0x1e0>, <0 0x01d87c00 0 0x1dc>, @@ -2312,7 +2313,7 @@ ufs_mem_phy_lanes: lanes@1d87400 { }; }; - cryptobam: dma@1dc4000 { + cryptobam: dma-controller@1dc4000 { compatible = "qcom,bam-v1.7.0"; reg = <0 0x01dc4000 0 0x24000>; interrupts = ; @@ -2984,6 +2985,8 @@ mss_pil: remoteproc@4080000 { clock-names = "iface", "bus", "mem", "gpll0_mss", "snoc_axi", "mnoc_axi", "prng", "xo"; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&modem_smp2p_out 0>; qcom,smem-state-names = "stop"; @@ -2993,11 +2996,12 @@ mss_pil: remoteproc@4080000 { qcom,halt-regs = <&tcsr_mutex_regs 0x23000 0x25000 0x24000>; - power-domains = <&aoss_qmp 2>, - <&rpmhpd SDM845_CX>, + power-domains = <&rpmhpd SDM845_CX>, <&rpmhpd SDM845_MX>, <&rpmhpd SDM845_MSS>; - power-domain-names = "load_state", "cx", "mx", "mss"; + power-domain-names = "cx", "mx", "mss"; + + status = "disabled"; mba { memory-region = <&mba_region>; @@ -3497,8 +3501,9 @@ sdhc_2: sdhci@8804000 { interrupt-names = "hc_irq", "pwr_irq"; clocks = <&gcc GCC_SDCC2_AHB_CLK>, - <&gcc GCC_SDCC2_APPS_CLK>; - clock-names = "iface", "core"; + <&gcc GCC_SDCC2_APPS_CLK>, + <&rpmhcc RPMH_CXO_CLK>; + clock-names = "iface", "core", "xo"; iommus = <&apps_smmu 0xa0 0xf>; power-domains = <&rpmhpd SDM845_CX>; operating-points-v2 = <&sdhc2_opp_table>; @@ -3646,6 +3651,30 @@ swm: swm@c85 { }; }; + lmh_cluster1: lmh@17d70800 { + compatible = "qcom,sdm845-lmh"; + reg = <0 0x17d70800 0 0x400>; + interrupts = ; + cpus = <&CPU4>; + qcom,lmh-temp-arm-millicelsius = <65000>; + qcom,lmh-temp-low-millicelsius = <94500>; + qcom,lmh-temp-high-millicelsius = <95000>; + interrupt-controller; + #interrupt-cells = <1>; + }; + + lmh_cluster0: lmh@17d78800 { + compatible = "qcom,sdm845-lmh"; + reg = <0 0x17d78800 0 0x400>; + interrupts = ; + cpus = <&CPU0>; + qcom,lmh-temp-arm-millicelsius = <65000>; + qcom,lmh-temp-low-millicelsius = <94500>; + qcom,lmh-temp-high-millicelsius = <95000>; + interrupt-controller; + #interrupt-cells = <1>; + }; + sound: sound { }; @@ -3683,7 +3712,6 @@ usb_1_qmpphy: phy@88e9000 { compatible = "qcom,sdm845-qmp-usb3-phy"; reg = <0 0x088e9000 0 0x18c>, <0 0x088e8000 0 0x10>; - reg-names = "reg-base", "dp_com"; status = "disabled"; #address-cells = <2>; #size-cells = <2>; @@ -3699,7 +3727,7 @@ usb_1_qmpphy: phy@88e9000 { <&gcc GCC_USB3_PHY_PRIM_BCR>; reset-names = "phy", "common"; - usb_1_ssphy: lanes@88e9200 { + usb_1_ssphy: phy@88e9200 { reg = <0 0x088e9200 0 0x128>, <0 0x088e9400 0 0x200>, <0 0x088e9c00 0 0x218>, @@ -3732,7 +3760,7 @@ usb_2_qmpphy: phy@88eb000 { <&gcc GCC_USB3_PHY_SEC_BCR>; reset-names = "phy", "common"; - usb_2_ssphy: lane@88eb200 { + usb_2_ssphy: phy@88eb200 { reg = <0 0x088eb200 0 0x128>, <0 0x088eb400 0 0x1fc>, <0 0x088eb800 0 0x218>, @@ -3868,6 +3896,8 @@ venus: video-codec@aa00000 { <&gladiator_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_VENUS_CFG 0>; interconnect-names = "video-mem", "cpu-cfg"; + status = "disabled"; + video-core0 { compatible = "venus-decoder"; }; @@ -4194,8 +4224,6 @@ mdss_mdp: mdp@ae01000 { interrupt-parent = <&mdss>; interrupts = <0 IRQ_TYPE_LEVEL_HIGH>; - status = "disabled"; - ports { #address-cells = <1>; #size-cells = <0>; @@ -4271,6 +4299,9 @@ dsi0: dsi@ae94000 { status = "disabled"; + #address-cells = <1>; + #size-cells = <0>; + ports { #address-cells = <1>; #size-cells = <0>; @@ -4340,6 +4371,9 @@ dsi1: dsi@ae96000 { status = "disabled"; + #address-cells = <1>; + #size-cells = <0>; + ports { #address-cells = <1>; #size-cells = <0>; @@ -4381,7 +4415,6 @@ dsi1_phy: dsi-phy@ae96400 { gpu: gpu@5000000 { compatible = "qcom,adreno-630.2", "qcom,adreno"; - #stream-id-cells = <16>; reg = <0 0x5000000 0 0x40000>, <0 0x509e000 0 0x10>; reg-names = "kgsl_3d0_reg_memory", "cx_mem"; @@ -4402,6 +4435,8 @@ gpu: gpu@5000000 { interconnects = <&mem_noc MASTER_GFX3D 0 &mem_noc SLAVE_EBI1 0>; interconnect-names = "gfx-mem"; + status = "disabled"; + gpu_opp_table: opp-table { compatible = "operating-points-v2"; @@ -4497,6 +4532,8 @@ gmu: gmu@506a000 { operating-points-v2 = <&gmu_opp_table>; + status = "disabled"; + gmu_opp_table: opp-table { compatible = "operating-points-v2"; @@ -4588,7 +4625,6 @@ aoss_qmp: power-controller@c300000 { mboxes = <&apss_shared 0>; #clock-cells = <0>; - #power-domain-cells = <1>; cx_cdev: cx { #cooling-cells = <2>; @@ -4917,6 +4953,8 @@ cpufreq_hw: cpufreq@17d43000 { reg = <0 0x17d43000 0 0x1400>, <0 0x17d45800 0 0x1400>; reg-names = "freq-domain0", "freq-domain1"; + interrupts-extended = <&lmh_cluster0 0>, <&lmh_cluster1 0>; + clocks = <&rpmhcc RPMH_CXO_CLK>, <&gcc GPLL0>; clock-names = "xo", "alternate"; @@ -4974,23 +5012,6 @@ cpu0_crit: cpu_crit { type = "critical"; }; }; - - cooling-maps { - map0 { - trip = <&cpu0_alert0>; - cooling-device = <&CPU0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - map1 { - trip = <&cpu0_alert1>; - cooling-device = <&CPU0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; }; cpu1-thermal { @@ -5018,23 +5039,6 @@ cpu1_crit: cpu_crit { type = "critical"; }; }; - - cooling-maps { - map0 { - trip = <&cpu1_alert0>; - cooling-device = <&CPU0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - map1 { - trip = <&cpu1_alert1>; - cooling-device = <&CPU0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; }; cpu2-thermal { @@ -5062,23 +5066,6 @@ cpu2_crit: cpu_crit { type = "critical"; }; }; - - cooling-maps { - map0 { - trip = <&cpu2_alert0>; - cooling-device = <&CPU0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - map1 { - trip = <&cpu2_alert1>; - cooling-device = <&CPU0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; }; cpu3-thermal { @@ -5106,23 +5093,6 @@ cpu3_crit: cpu_crit { type = "critical"; }; }; - - cooling-maps { - map0 { - trip = <&cpu3_alert0>; - cooling-device = <&CPU0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - map1 { - trip = <&cpu3_alert1>; - cooling-device = <&CPU0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; }; cpu4-thermal { @@ -5150,23 +5120,6 @@ cpu4_crit: cpu_crit { type = "critical"; }; }; - - cooling-maps { - map0 { - trip = <&cpu4_alert0>; - cooling-device = <&CPU4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU5 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU6 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU7 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - map1 { - trip = <&cpu4_alert1>; - cooling-device = <&CPU4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU5 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU6 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU7 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; }; cpu5-thermal { @@ -5194,23 +5147,6 @@ cpu5_crit: cpu_crit { type = "critical"; }; }; - - cooling-maps { - map0 { - trip = <&cpu5_alert0>; - cooling-device = <&CPU4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU5 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU6 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU7 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - map1 { - trip = <&cpu5_alert1>; - cooling-device = <&CPU4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU5 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU6 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU7 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; }; cpu6-thermal { @@ -5238,23 +5174,6 @@ cpu6_crit: cpu_crit { type = "critical"; }; }; - - cooling-maps { - map0 { - trip = <&cpu6_alert0>; - cooling-device = <&CPU4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU5 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU6 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU7 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - map1 { - trip = <&cpu6_alert1>; - cooling-device = <&CPU4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU5 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU6 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU7 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; }; cpu7-thermal { @@ -5282,23 +5201,6 @@ cpu7_crit: cpu_crit { type = "critical"; }; }; - - cooling-maps { - map0 { - trip = <&cpu7_alert0>; - cooling-device = <&CPU4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU5 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU6 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU7 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - map1 { - trip = <&cpu7_alert1>; - cooling-device = <&CPU4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU5 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU6 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&CPU7 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; }; aoss0-thermal { diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts index 617a634ac9..e2b9ec134c 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts @@ -7,7 +7,6 @@ /dts-v1/; -#include #include #include #include @@ -30,6 +29,7 @@ / { model = "Lenovo Yoga C630"; compatible = "lenovo,yoga-c630", "qcom,sdm845"; + chassis-type = "convertible"; aliases { hsuart0 = &uart6; @@ -264,6 +264,9 @@ vreg_l22a_2p85: ldo22 { }; vreg_l23a_3p3: ldo23 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3312000>; + regulator-initial-mode = ; }; vdda_qusb_hs0_3p1: @@ -341,7 +344,12 @@ &gcc { ; }; +&gmu { + status = "okay"; +}; + &gpu { + status = "okay"; zap-shader { memory-region = <&gpu_mem>; firmware-name = "qcom/LENOVO/81JL/qcdxkmsuc850.mbn"; @@ -458,11 +466,8 @@ &mdss { status = "okay"; }; -&mdss_mdp { - status = "okay"; -}; - &mss_pil { + status = "okay"; firmware-name = "qcom/LENOVO/81JL/qcdsp1v2850.mbn", "qcom/LENOVO/81JL/qcdsp2850.mbn"; }; @@ -676,6 +681,7 @@ bluetooth { vddxo-supply = <&vreg_l7a_1p8>; vddrf-supply = <&vreg_l17a_1p3>; vddch0-supply = <&vreg_l25a_3p3>; + vddch1-supply = <&vreg_l23a_3p3>; max-speed = <3200000>; }; }; @@ -750,6 +756,10 @@ &usb_2_qmpphy { vdda-pll-supply = <&vdda_usb2_ss_core>; }; +&venus { + status = "okay"; +}; + &wcd9340{ pinctrl-0 = <&wcd_intr_default>; pinctrl-names = "default"; @@ -761,6 +771,9 @@ &wcd9340{ vdd-tx-supply = <&vreg_s4a_1p8>; vdd-rx-supply = <&vreg_s4a_1p8>; vdd-io-supply = <&vreg_s4a_1p8>; + qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 500000 500000 500000 500000>; + qcom,mbhc-headset-vthreshold-microvolt = <1700000>; + qcom,mbhc-headphone-vthreshold-microvolt = <50000>; swm: swm@c85 { left_spkr: wsa8810-left{ @@ -790,6 +803,12 @@ &wifi { vdd-1.8-xo-supply = <&vreg_l7a_1p8>; vdd-1.3-rfa-supply = <&vreg_l17a_1p3>; vdd-3.3-ch0-supply = <&vreg_l25a_3p3>; + vdd-3.3-ch1-supply = <&vreg_l23a_3p3>; qcom,snoc-host-cap-8bit-quirk; }; + +&crypto { + /* FIXME: qce_start triggers an SError */ + status= "disable"; +}; diff --git a/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts b/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts index 58b6b2742d..871ccbba44 100644 --- a/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts +++ b/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts @@ -17,6 +17,7 @@ / { model = "Sony Xperia 10 II"; compatible = "sony,pdx201", "qcom,sm6125"; + chassis-type = "handset"; chosen { #address-cells = <2>; @@ -41,7 +42,6 @@ extcon_usb: extcon-usb { gpio-keys { status = "okay"; compatible = "gpio-keys"; - input-name = "gpio-keys"; #address-cells = <1>; #size-cells = <0>; autorepeat; diff --git a/arch/arm64/boot/dts/qcom/sm6125.dtsi b/arch/arm64/boot/dts/qcom/sm6125.dtsi index 9f476e3d07..49e6bca646 100644 --- a/arch/arm64/boot/dts/qcom/sm6125.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6125.dtsi @@ -302,6 +302,56 @@ rpmcc: clock-controller { compatible = "qcom,rpmcc-sm6125", "qcom,rpmcc"; #clock-cells = <1>; }; + + rpmpd: power-controller { + compatible = "qcom,sm6125-rpmpd"; + #power-domain-cells = <1>; + operating-points-v2 = <&rpmpd_opp_table>; + + rpmpd_opp_table: opp-table { + compatible = "operating-points-v2"; + + rpmpd_opp_ret: opp1 { + opp-level = ; + }; + + rpmpd_opp_ret_plus: opp2 { + opp-level = ; + }; + + rpmpd_opp_min_svs: opp3 { + opp-level = ; + }; + + rpmpd_opp_low_svs: opp4 { + opp-level = ; + }; + + rpmpd_opp_svs: opp5 { + opp-level = ; + }; + + rpmpd_opp_svs_plus: opp6 { + opp-level = ; + }; + + rpmpd_opp_nom: opp7 { + opp-level = ; + }; + + rpmpd_opp_nom_plus: opp8 { + opp-level = ; + }; + + rpmpd_opp_turbo: opp9 { + opp-level = ; + }; + + rpmpd_opp_turbo_no_cpr: opp10 { + opp-level = ; + }; + }; + }; }; }; @@ -326,8 +376,8 @@ tcsr_mutex: hwlock@340000 { tlmm: pinctrl@500000 { compatible = "qcom,sm6125-tlmm"; reg = <0x00500000 0x400000>, - <0x00900000 0x400000>, - <0x00d00000 0x400000>; + <0x00900000 0x400000>, + <0x00d00000 0x400000>; reg-names = "west", "south", "east"; interrupts = ; gpio-controller; @@ -391,13 +441,16 @@ sdhc_1: sdhci@4744000 { reg-names = "hc", "core"; interrupts = , - ; + ; interrupt-names = "hc_irq", "pwr_irq"; clocks = <&gcc GCC_SDCC1_AHB_CLK>, - <&gcc GCC_SDCC1_APPS_CLK>, - <&xo_board>; + <&gcc GCC_SDCC1_APPS_CLK>, + <&xo_board>; clock-names = "iface", "core", "xo"; + + power-domains = <&rpmpd 0>; + bus-width = <8>; non-removable; status = "disabled"; @@ -409,18 +462,20 @@ sdhc_2: sdhci@4784000 { reg-names = "hc"; interrupts = , - ; + ; interrupt-names = "hc_irq", "pwr_irq"; clocks = <&gcc GCC_SDCC2_AHB_CLK>, - <&gcc GCC_SDCC2_APPS_CLK>, - <&xo_board>; + <&gcc GCC_SDCC2_APPS_CLK>, + <&xo_board>; clock-names = "iface", "core", "xo"; pinctrl-0 = <&sdc2_state_on>; pinctrl-1 = <&sdc2_state_off>; pinctrl-names = "default", "sleep"; + power-domains = <&rpmpd 0>; + bus-width = <4>; status = "disabled"; }; @@ -433,11 +488,11 @@ usb3: usb@4ef8800 { ranges; clocks = <&gcc GCC_USB30_PRIM_MASTER_CLK>, - <&gcc GCC_SYS_NOC_USB3_PRIM_AXI_CLK>, - <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>, - <&gcc GCC_USB3_PRIM_CLKREF_CLK>, - <&gcc GCC_USB30_PRIM_SLEEP_CLK>, - <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>; + <&gcc GCC_SYS_NOC_USB3_PRIM_AXI_CLK>, + <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>, + <&gcc GCC_USB3_PRIM_CLKREF_CLK>, + <&gcc GCC_USB30_PRIM_SLEEP_CLK>, + <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>; assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>, <&gcc GCC_USB30_PRIM_MASTER_CLK>; @@ -460,13 +515,18 @@ usb3_dwc3: usb@4e00000 { }; }; + sram@4690000 { + compatible = "qcom,rpm-stats"; + reg = <0x04690000 0x10000>; + }; + spmi_bus: spmi@1c40000 { compatible = "qcom,spmi-pmic-arb"; - reg = <0x01c40000 0x1100>, - <0x01e00000 0x2000000>, - <0x03e00000 0x100000>, - <0x03f00000 0xa0000>, - <0x01c0a000 0x26000>; + reg = <0x01c40000 0x1100>, + <0x01e00000 0x2000000>, + <0x03e00000 0x100000>, + <0x03f00000 0xa0000>, + <0x01c0a000 0x26000>; reg-names = "core", "chnls", "obsrvr", "intr", "cnfg"; interrupt-names = "periph_irq"; interrupts = ; @@ -494,22 +554,22 @@ timer@f120000 { reg = <0x0f120000 0x1000>; clock-frequency = <19200000>; - frame@0f121000 { + frame@f121000 { frame-number = <0>; interrupts = , - ; + ; reg = <0x0f121000 0x1000>, <0x0f122000 0x1000>; }; - frame@0f123000 { + frame@f123000 { frame-number = <1>; interrupts = ; reg = <0x0f123000 0x1000>; status = "disabled"; }; - frame@0f124000 { + frame@f124000 { frame-number = <2>; interrupts = ; reg = <0x0f124000 0x1000>; @@ -548,7 +608,7 @@ frame@f128000 { intc: interrupt-controller@f200000 { compatible = "arm,gic-v3"; reg = <0x0f200000 0x20000>, - <0x0f300000 0x100000>; + <0x0f300000 0x100000>; #interrupt-cells = <3>; interrupt-controller; interrupts = ; @@ -558,9 +618,9 @@ intc: interrupt-controller@f200000 { timer { compatible = "arm,armv8-timer"; interrupts = ; + GIC_PPI 2 0xf08 + GIC_PPI 3 0xf08 + GIC_PPI 0 0xf08>; clock-frequency = <19200000>; }; }; diff --git a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts index 335aa0753f..37ddca0f02 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-hdk.dts @@ -419,9 +419,7 @@ &ufs_mem_phy { status = "okay"; vdda-phy-supply = <&vdda_ufs_2ln_core_1>; - vdda-max-microamp = <90200>; vdda-pll-supply = <&vreg_l3c_1p2>; - vdda-pll-max-microamp = <19000>; }; &usb_1_hsphy { diff --git a/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts b/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts index 736da9af44..5901c28e66 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-microsoft-surface-duo.dts @@ -15,6 +15,7 @@ / { model = "Microsoft Surface Duo"; compatible = "microsoft,surface-duo", "qcom,sm8150"; + chassis-type = "handset"; aliases { serial0 = &uart2; @@ -507,9 +508,7 @@ &ufs_mem_phy { status = "okay"; vdda-phy-supply = <&vdda_ufs_2ln_core_1>; - vdda-max-microamp = <90200>; vdda-pll-supply = <&vreg_l3c_1p2>; - vdda-pll-max-microamp = <19000>; }; &usb_1_hsphy { diff --git a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts index b484371a60..46b5cf9a11 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-mtp.dts @@ -416,9 +416,7 @@ &ufs_mem_phy { status = "okay"; vdda-phy-supply = <&vdda_ufs_2ln_core_1>; - vdda-max-microamp = <90200>; vdda-pll-supply = <&vreg_l3c_1p2>; - vdda-pll-max-microamp = <19000>; }; &usb_1_hsphy { diff --git a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano-bahamut.dts b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano-bahamut.dts index 3b55fdda76..b278040325 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano-bahamut.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano-bahamut.dts @@ -10,6 +10,7 @@ / { model = "Sony Xperia 5"; compatible = "sony,bahamut-generic", "qcom,sm8150"; + chassis-type = "handset"; }; &framebuffer { diff --git a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano-griffin.dts b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano-griffin.dts index 6f490ec284..0d6deceedf 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano-griffin.dts +++ b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano-griffin.dts @@ -10,4 +10,5 @@ / { model = "Sony Xperia 1"; compatible = "sony,griffin-generic", "qcom,sm8150"; + chassis-type = "handset"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi index ef0232c2cf..6012322a59 100644 --- a/arch/arm64/boot/dts/qcom/sm8150.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -1692,7 +1691,7 @@ ufs_mem_phy: phy@1d87000 { reset-names = "ufsphy"; status = "disabled"; - ufs_mem_phy_lanes: lanes@1d87400 { + ufs_mem_phy_lanes: phy@1d87400 { reg = <0 0x01d87400 0 0x108>, <0 0x01d87600 0 0x1e0>, <0 0x01d87c00 0 0x1dc>, @@ -1729,13 +1728,14 @@ remoteproc_slpi: remoteproc@2400000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_SLPI>, - <&rpmhpd 3>, + power-domains = <&rpmhpd 3>, <&rpmhpd 2>; - power-domain-names = "load_state", "lcx", "lmx"; + power-domain-names = "lcx", "lmx"; memory-region = <&slpi_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&slpi_smp2p_out 0>; qcom,smem-state-names = "stop"; @@ -1746,6 +1746,33 @@ glink-edge { label = "dsps"; qcom,remote-pid = <3>; mboxes = <&apss_shared 24>; + + fastrpc { + compatible = "qcom,fastrpc"; + qcom,glink-channels = "fastrpcglink-apps-dsp"; + label = "sdsp"; + #address-cells = <1>; + #size-cells = <0>; + + compute-cb@1 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <1>; + iommus = <&apps_smmu 0x05a1 0x0>; + }; + + compute-cb@2 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <2>; + iommus = <&apps_smmu 0x05a2 0x0>; + }; + + compute-cb@3 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <3>; + iommus = <&apps_smmu 0x05a3 0x0>; + /* note: shared-cb = <4> in downstream */ + }; + }; }; }; @@ -1758,7 +1785,6 @@ gpu: gpu@2c00000 { compatible = "qcom,adreno-640.1", "qcom,adreno", "amd,imageon"; - #stream-id-cells = <16>; reg = <0 0x02c00000 0 0x40000>; reg-names = "kgsl_3d0_reg_memory"; @@ -2319,13 +2345,14 @@ remoteproc_mpss: remoteproc@4080000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_MODEM>, - <&rpmhpd 7>, + power-domains = <&rpmhpd 7>, <&rpmhpd 0>; - power-domain-names = "load_state", "cx", "mss"; + power-domain-names = "cx", "mss"; memory-region = <&mpss_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&modem_smp2p_out 0>; qcom,smem-state-names = "stop"; @@ -2945,12 +2972,12 @@ remoteproc_cdsp: remoteproc@8300000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_CDSP>, - <&rpmhpd 7>; - power-domain-names = "load_state", "cx"; + power-domains = <&rpmhpd 7>; memory-region = <&cdsp_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&cdsp_smp2p_out 0>; qcom,smem-state-names = "stop"; @@ -2961,6 +2988,72 @@ glink-edge { label = "cdsp"; qcom,remote-pid = <5>; mboxes = <&apss_shared 4>; + + fastrpc { + compatible = "qcom,fastrpc"; + qcom,glink-channels = "fastrpcglink-apps-dsp"; + label = "cdsp"; + #address-cells = <1>; + #size-cells = <0>; + + compute-cb@1 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <1>; + iommus = <&apps_smmu 0x1401 0x2040>, + <&apps_smmu 0x1421 0x0>, + <&apps_smmu 0x2001 0x420>, + <&apps_smmu 0x2041 0x0>; + }; + + compute-cb@2 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <2>; + iommus = <&apps_smmu 0x2 0x3440>, + <&apps_smmu 0x22 0x3400>; + }; + + compute-cb@3 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <3>; + iommus = <&apps_smmu 0x3 0x3440>, + <&apps_smmu 0x1423 0x0>, + <&apps_smmu 0x2023 0x0>; + }; + + compute-cb@4 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <4>; + iommus = <&apps_smmu 0x4 0x3440>, + <&apps_smmu 0x24 0x3400>; + }; + + compute-cb@5 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <5>; + iommus = <&apps_smmu 0x5 0x3440>, + <&apps_smmu 0x25 0x3400>; + }; + + compute-cb@6 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <6>; + iommus = <&apps_smmu 0x6 0x3460>; + }; + + compute-cb@7 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <7>; + iommus = <&apps_smmu 0x7 0x3460>; + }; + + compute-cb@8 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <8>; + iommus = <&apps_smmu 0x8 0x3460>; + }; + + /* note: secure cb9 in downstream */ + }; }; }; @@ -2994,7 +3087,6 @@ usb_1_qmpphy: phy@88e9000 { compatible = "qcom,sm8150-qmp-usb3-phy"; reg = <0 0x088e9000 0 0x18c>, <0 0x088e8000 0 0x10>; - reg-names = "reg-base", "dp_com"; status = "disabled"; #address-cells = <2>; #size-cells = <2>; @@ -3010,7 +3102,7 @@ usb_1_qmpphy: phy@88e9000 { <&gcc GCC_USB3_PHY_PRIM_BCR>; reset-names = "phy", "common"; - usb_1_ssphy: lanes@88e9200 { + usb_1_ssphy: phy@88e9200 { reg = <0 0x088e9200 0 0x200>, <0 0x088e9400 0 0x200>, <0 0x088e9c00 0 0x218>, @@ -3043,7 +3135,7 @@ usb_2_qmpphy: phy@88eb000 { <&gcc GCC_USB3_PHY_SEC_BCR>; reset-names = "phy", "common"; - usb_2_ssphy: lane@88eb200 { + usb_2_ssphy: phy@88eb200 { reg = <0 0x088eb200 0 0x200>, <0 0x088eb400 0 0x200>, <0 0x088eb800 0 0x800>, @@ -3169,12 +3261,16 @@ camnoc_virt: interconnect@ac00000 { aoss_qmp: power-controller@c300000 { compatible = "qcom,sm8150-aoss-qmp"; - reg = <0x0 0x0c300000 0x0 0x100000>; + reg = <0x0 0x0c300000 0x0 0x400>; interrupts = ; mboxes = <&apss_shared 0>; #clock-cells = <0>; - #power-domain-cells = <1>; + }; + + sram@c3f0000 { + compatible = "qcom,rpmh-stats"; + reg = <0 0x0c3f0000 0 0x400>; }; tsens0: thermal-sensor@c263000 { @@ -3321,12 +3417,12 @@ remoteproc_adsp: remoteproc@17300000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_LPASS>, - <&rpmhpd 7>; - power-domain-names = "load_state", "cx"; + power-domains = <&rpmhpd 7>; memory-region = <&adsp_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&adsp_smp2p_out 0>; qcom,smem-state-names = "stop"; @@ -3337,6 +3433,32 @@ glink-edge { label = "lpass"; qcom,remote-pid = <2>; mboxes = <&apss_shared 8>; + + fastrpc { + compatible = "qcom,fastrpc"; + qcom,glink-channels = "fastrpcglink-apps-dsp"; + label = "adsp"; + #address-cells = <1>; + #size-cells = <0>; + + compute-cb@3 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <3>; + iommus = <&apps_smmu 0x1b23 0x0>; + }; + + compute-cb@4 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <4>; + iommus = <&apps_smmu 0x1b24 0x0>; + }; + + compute-cb@5 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <5>; + iommus = <&apps_smmu 0x1b25 0x0>; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts index 47742816ac..3b08247206 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-hdk.dts @@ -409,9 +409,7 @@ &ufs_mem_phy { status = "okay"; vdda-phy-supply = <&vreg_l5a_0p88>; - vdda-max-microamp = <89900>; vdda-pll-supply = <&vreg_l9a_1p2>; - vdda-pll-max-microamp = <18800>; }; &usb_1_hsphy { diff --git a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts index 062b944be9..fb99cc2827 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts @@ -6,6 +6,9 @@ /dts-v1/; #include +#include +#include +#include #include "sm8250.dtsi" #include "pm8150.dtsi" #include "pm8150b.dtsi" @@ -624,8 +627,196 @@ &slpi { firmware-name = "qcom/sm8250/slpi.mbn"; }; +&soc { + wcd938x: codec { + compatible = "qcom,wcd9380-codec"; + #sound-dai-cells = <1>; + reset-gpios = <&tlmm 32 0>; + vdd-buck-supply = <&vreg_s4a_1p8>; + vdd-rxtx-supply = <&vreg_s4a_1p8>; + vdd-io-supply = <&vreg_s4a_1p8>; + vdd-mic-bias-supply = <&vreg_bob>; + qcom,micbias1-microvolt = <1800000>; + qcom,micbias2-microvolt = <1800000>; + qcom,micbias3-microvolt = <1800000>; + qcom,micbias4-microvolt = <1800000>; + qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 500000 500000 500000 500000>; + qcom,mbhc-headset-vthreshold-microvolt = <1700000>; + qcom,mbhc-headphone-vthreshold-microvolt = <50000>; + qcom,rx-device = <&wcd_rx>; + qcom,tx-device = <&wcd_tx>; + }; +}; + +&sound { + compatible = "qcom,sm8250-sndcard"; + model = "SM8250-MTP-WCD9380-WSA8810-VA-DMIC"; + audio-routing = + "SpkrLeft IN", "WSA_SPK1 OUT", + "SpkrRight IN", "WSA_SPK2 OUT", + "IN1_HPHL", "HPHL_OUT", + "IN2_HPHR", "HPHR_OUT", + "AMIC1", "MIC BIAS1", + "AMIC2", "MIC BIAS2", + "AMIC3", "MIC BIAS3", + "AMIC4", "MIC BIAS3", + "AMIC5", "MIC BIAS4", + "TX SWR_ADC0", "ADC1_OUTPUT", + "TX SWR_ADC1", "ADC2_OUTPUT", + "TX SWR_ADC2", "ADC3_OUTPUT", + "TX SWR_ADC3", "ADC4_OUTPUT", + "TX SWR_DMIC0", "DMIC1_OUTPUT", + "TX SWR_DMIC1", "DMIC2_OUTPUT", + "TX SWR_DMIC2", "DMIC3_OUTPUT", + "TX SWR_DMIC3", "DMIC4_OUTPUT", + "TX SWR_DMIC4", "DMIC5_OUTPUT", + "TX SWR_DMIC5", "DMIC6_OUTPUT", + "TX SWR_DMIC6", "DMIC7_OUTPUT", + "TX SWR_DMIC7", "DMIC8_OUTPUT"; + + mm1-dai-link { + link-name = "MultiMedia1"; + cpu { + sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA1>; + }; + }; + + mm2-dai-link { + link-name = "MultiMedia2"; + cpu { + sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA2>; + }; + }; + + mm3-dai-link { + link-name = "MultiMedia3"; + cpu { + sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA3>; + }; + }; + + wcd-playback-dai-link { + link-name = "WCD Playback"; + cpu { + sound-dai = <&q6afedai RX_CODEC_DMA_RX_0>; + }; + codec { + sound-dai = <&wcd938x 0>, <&swr1 0>, <&rxmacro 0>; + }; + platform { + sound-dai = <&q6routing>; + }; + }; + + wcd-capture-dai-link { + link-name = "WCD Capture"; + cpu { + sound-dai = <&q6afedai TX_CODEC_DMA_TX_3>; + }; + + codec { + sound-dai = <&wcd938x 1>, <&swr2 0>, <&txmacro 0>; + }; + platform { + sound-dai = <&q6routing>; + }; + }; + + wsa-dai-link { + link-name = "WSA Playback"; + cpu { + sound-dai = <&q6afedai WSA_CODEC_DMA_RX_0>; + }; + + codec { + sound-dai = <&left_spkr>, <&right_spkr>, <&swr0 0>, <&wsamacro 0>; + }; + platform { + sound-dai = <&q6routing>; + }; + }; + + va-dai-link { + link-name = "VA Capture"; + cpu { + sound-dai = <&q6afedai VA_CODEC_DMA_TX_0>; + }; + + platform { + sound-dai = <&q6routing>; + }; + + codec { + sound-dai = <&vamacro 0>; + }; + }; +}; + +&swr0 { + left_spkr: wsa8810-right@0,3{ + compatible = "sdw10217211000"; + reg = <0 3>; + powerdown-gpios = <&tlmm 26 GPIO_ACTIVE_HIGH>; + #thermal-sensor-cells = <0>; + sound-name-prefix = "SpkrLeft"; + #sound-dai-cells = <0>; + }; + + right_spkr: wsa8810-left@0,4{ + compatible = "sdw10217211000"; + reg = <0 4>; + powerdown-gpios = <&tlmm 127 GPIO_ACTIVE_HIGH>; + #thermal-sensor-cells = <0>; + sound-name-prefix = "SpkrRight"; + #sound-dai-cells = <0>; + }; +}; + +&swr1 { + wcd_rx: wcd9380-rx@0,4 { + compatible = "sdw20217010d00"; + reg = <0 4>; + qcom,rx-port-mapping = <1 2 3 4 5>; + }; +}; + +&swr2 { + wcd_tx: wcd9380-tx@0,3 { + compatible = "sdw20217010d00"; + reg = <0 3>; + qcom,tx-port-mapping = <2 3 4 5>; + }; +}; + &tlmm { gpio-reserved-ranges = <28 4>, <40 4>; + + wcd938x_reset_default: wcd938x_reset_default { + mux { + pins = "gpio32"; + function = "gpio"; + }; + + config { + pins = "gpio32"; + drive-strength = <16>; + output-high; + }; + }; + + wcd938x_reset_sleep: wcd938x_reset_sleep { + mux { + pins = "gpio32"; + function = "gpio"; + }; + + config { + pins = "gpio32"; + drive-strength = <16>; + bias-disable; + output-low; + }; + }; }; &uart12 { @@ -647,9 +838,7 @@ &ufs_mem_phy { status = "okay"; vdda-phy-supply = <&vreg_l5a_0p875>; - vdda-max-microamp = <90200>; vdda-pll-supply = <&vreg_l9a_1p2>; - vdda-pll-max-microamp = <19000>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo-pdx203.dts b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo-pdx203.dts index 79afeb07f4..356a816987 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo-pdx203.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo-pdx203.dts @@ -10,6 +10,7 @@ / { model = "Sony Xperia 1 II"; compatible = "sony,pdx203-generic", "qcom,sm8250"; + chassis-type = "handset"; }; /delete-node/ &vreg_l7f_1p8; diff --git a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo-pdx206.dts b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo-pdx206.dts index 16c96e8385..5ecf7dafb2 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo-pdx206.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo-pdx206.dts @@ -10,6 +10,7 @@ / { model = "Sony Xperia 5 II"; compatible = "sony,pdx206-generic", "qcom,sm8250"; + chassis-type = "handset"; }; &framebuffer { diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi index d12e4cbfc8..5617a46e5c 100644 --- a/arch/arm64/boot/dts/qcom/sm8250.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -100,6 +99,9 @@ CPU0: cpu@0 { dynamic-power-coefficient = <205>; next-level-cache = <&L2_0>; qcom,freq-domain = <&cpufreq_hw 0>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_AMPSS_M0 &mc_virt SLAVE_EBI_CH0>, + <&epss_l3 MASTER_OSM_L3_APPS &epss_l3 SLAVE_OSM_L3>; #cooling-cells = <2>; L2_0: l2-cache { compatible = "cache"; @@ -119,6 +121,9 @@ CPU1: cpu@100 { dynamic-power-coefficient = <205>; next-level-cache = <&L2_100>; qcom,freq-domain = <&cpufreq_hw 0>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_AMPSS_M0 &mc_virt SLAVE_EBI_CH0>, + <&epss_l3 MASTER_OSM_L3_APPS &epss_l3 SLAVE_OSM_L3>; #cooling-cells = <2>; L2_100: l2-cache { compatible = "cache"; @@ -135,6 +140,9 @@ CPU2: cpu@200 { dynamic-power-coefficient = <205>; next-level-cache = <&L2_200>; qcom,freq-domain = <&cpufreq_hw 0>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_AMPSS_M0 &mc_virt SLAVE_EBI_CH0>, + <&epss_l3 MASTER_OSM_L3_APPS &epss_l3 SLAVE_OSM_L3>; #cooling-cells = <2>; L2_200: l2-cache { compatible = "cache"; @@ -151,6 +159,9 @@ CPU3: cpu@300 { dynamic-power-coefficient = <205>; next-level-cache = <&L2_300>; qcom,freq-domain = <&cpufreq_hw 0>; + operating-points-v2 = <&cpu0_opp_table>; + interconnects = <&gem_noc MASTER_AMPSS_M0 &mc_virt SLAVE_EBI_CH0>, + <&epss_l3 MASTER_OSM_L3_APPS &epss_l3 SLAVE_OSM_L3>; #cooling-cells = <2>; L2_300: l2-cache { compatible = "cache"; @@ -167,6 +178,9 @@ CPU4: cpu@400 { dynamic-power-coefficient = <379>; next-level-cache = <&L2_400>; qcom,freq-domain = <&cpufreq_hw 1>; + operating-points-v2 = <&cpu4_opp_table>; + interconnects = <&gem_noc MASTER_AMPSS_M0 &mc_virt SLAVE_EBI_CH0>, + <&epss_l3 MASTER_OSM_L3_APPS &epss_l3 SLAVE_OSM_L3>; #cooling-cells = <2>; L2_400: l2-cache { compatible = "cache"; @@ -183,6 +197,9 @@ CPU5: cpu@500 { dynamic-power-coefficient = <379>; next-level-cache = <&L2_500>; qcom,freq-domain = <&cpufreq_hw 1>; + operating-points-v2 = <&cpu4_opp_table>; + interconnects = <&gem_noc MASTER_AMPSS_M0 &mc_virt SLAVE_EBI_CH0>, + <&epss_l3 MASTER_OSM_L3_APPS &epss_l3 SLAVE_OSM_L3>; #cooling-cells = <2>; L2_500: l2-cache { compatible = "cache"; @@ -200,6 +217,9 @@ CPU6: cpu@600 { dynamic-power-coefficient = <379>; next-level-cache = <&L2_600>; qcom,freq-domain = <&cpufreq_hw 1>; + operating-points-v2 = <&cpu4_opp_table>; + interconnects = <&gem_noc MASTER_AMPSS_M0 &mc_virt SLAVE_EBI_CH0>, + <&epss_l3 MASTER_OSM_L3_APPS &epss_l3 SLAVE_OSM_L3>; #cooling-cells = <2>; L2_600: l2-cache { compatible = "cache"; @@ -216,6 +236,9 @@ CPU7: cpu@700 { dynamic-power-coefficient = <444>; next-level-cache = <&L2_700>; qcom,freq-domain = <&cpufreq_hw 2>; + operating-points-v2 = <&cpu7_opp_table>; + interconnects = <&gem_noc MASTER_AMPSS_M0 &mc_virt SLAVE_EBI_CH0>, + <&epss_l3 MASTER_OSM_L3_APPS &epss_l3 SLAVE_OSM_L3>; #cooling-cells = <2>; L2_700: l2-cache { compatible = "cache"; @@ -260,6 +283,296 @@ core7 { }; }; + cpu0_opp_table: cpu0_opp_table { + compatible = "operating-points-v2"; + opp-shared; + + cpu0_opp1: opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-peak-kBps = <800000 9600000>; + }; + + cpu0_opp2: opp-403200000 { + opp-hz = /bits/ 64 <403200000>; + opp-peak-kBps = <800000 9600000>; + }; + + cpu0_opp3: opp-518400000 { + opp-hz = /bits/ 64 <518400000>; + opp-peak-kBps = <800000 16588800>; + }; + + cpu0_opp4: opp-614400000 { + opp-hz = /bits/ 64 <614400000>; + opp-peak-kBps = <800000 16588800>; + }; + + cpu0_opp5: opp-691200000 { + opp-hz = /bits/ 64 <691200000>; + opp-peak-kBps = <800000 19660800>; + }; + + cpu0_opp6: opp-787200000 { + opp-hz = /bits/ 64 <787200000>; + opp-peak-kBps = <1804000 19660800>; + }; + + cpu0_opp7: opp-883200000 { + opp-hz = /bits/ 64 <883200000>; + opp-peak-kBps = <1804000 23347200>; + }; + + cpu0_opp8: opp-979200000 { + opp-hz = /bits/ 64 <979200000>; + opp-peak-kBps = <1804000 26419200>; + }; + + cpu0_opp9: opp-1075200000 { + opp-hz = /bits/ 64 <1075200000>; + opp-peak-kBps = <1804000 29491200>; + }; + + cpu0_opp10: opp-1171200000 { + opp-hz = /bits/ 64 <1171200000>; + opp-peak-kBps = <1804000 32563200>; + }; + + cpu0_opp11: opp-1248000000 { + opp-hz = /bits/ 64 <1248000000>; + opp-peak-kBps = <1804000 36249600>; + }; + + cpu0_opp12: opp-1344000000 { + opp-hz = /bits/ 64 <1344000000>; + opp-peak-kBps = <2188000 36249600>; + }; + + cpu0_opp13: opp-1420800000 { + opp-hz = /bits/ 64 <1420800000>; + opp-peak-kBps = <2188000 39321600>; + }; + + cpu0_opp14: opp-1516800000 { + opp-hz = /bits/ 64 <1516800000>; + opp-peak-kBps = <3072000 42393600>; + }; + + cpu0_opp15: opp-1612800000 { + opp-hz = /bits/ 64 <1612800000>; + opp-peak-kBps = <3072000 42393600>; + }; + + cpu0_opp16: opp-1708800000 { + opp-hz = /bits/ 64 <1708800000>; + opp-peak-kBps = <4068000 42393600>; + }; + + cpu0_opp17: opp-1804800000 { + opp-hz = /bits/ 64 <1804800000>; + opp-peak-kBps = <4068000 42393600>; + }; + }; + + cpu4_opp_table: cpu4_opp_table { + compatible = "operating-points-v2"; + opp-shared; + + cpu4_opp1: opp-710400000 { + opp-hz = /bits/ 64 <710400000>; + opp-peak-kBps = <1804000 19660800>; + }; + + cpu4_opp2: opp-825600000 { + opp-hz = /bits/ 64 <825600000>; + opp-peak-kBps = <2188000 23347200>; + }; + + cpu4_opp3: opp-940800000 { + opp-hz = /bits/ 64 <940800000>; + opp-peak-kBps = <2188000 26419200>; + }; + + cpu4_opp4: opp-1056000000 { + opp-hz = /bits/ 64 <1056000000>; + opp-peak-kBps = <3072000 26419200>; + }; + + cpu4_opp5: opp-1171200000 { + opp-hz = /bits/ 64 <1171200000>; + opp-peak-kBps = <3072000 29491200>; + }; + + cpu4_opp6: opp-1286400000 { + opp-hz = /bits/ 64 <1286400000>; + opp-peak-kBps = <4068000 29491200>; + }; + + cpu4_opp7: opp-1382400000 { + opp-hz = /bits/ 64 <1382400000>; + opp-peak-kBps = <4068000 32563200>; + }; + + cpu4_opp8: opp-1478400000 { + opp-hz = /bits/ 64 <1478400000>; + opp-peak-kBps = <4068000 32563200>; + }; + + cpu4_opp9: opp-1574400000 { + opp-hz = /bits/ 64 <1574400000>; + opp-peak-kBps = <5412000 39321600>; + }; + + cpu4_opp10: opp-1670400000 { + opp-hz = /bits/ 64 <1670400000>; + opp-peak-kBps = <5412000 42393600>; + }; + + cpu4_opp11: opp-1766400000 { + opp-hz = /bits/ 64 <1766400000>; + opp-peak-kBps = <5412000 45465600>; + }; + + cpu4_opp12: opp-1862400000 { + opp-hz = /bits/ 64 <1862400000>; + opp-peak-kBps = <6220000 45465600>; + }; + + cpu4_opp13: opp-1958400000 { + opp-hz = /bits/ 64 <1958400000>; + opp-peak-kBps = <6220000 48537600>; + }; + + cpu4_opp14: opp-2054400000 { + opp-hz = /bits/ 64 <2054400000>; + opp-peak-kBps = <7216000 48537600>; + }; + + cpu4_opp15: opp-2150400000 { + opp-hz = /bits/ 64 <2150400000>; + opp-peak-kBps = <7216000 51609600>; + }; + + cpu4_opp16: opp-2246400000 { + opp-hz = /bits/ 64 <2246400000>; + opp-peak-kBps = <7216000 51609600>; + }; + + cpu4_opp17: opp-2342400000 { + opp-hz = /bits/ 64 <2342400000>; + opp-peak-kBps = <8368000 51609600>; + }; + + cpu4_opp18: opp-2419200000 { + opp-hz = /bits/ 64 <2419200000>; + opp-peak-kBps = <8368000 51609600>; + }; + }; + + cpu7_opp_table: cpu7_opp_table { + compatible = "operating-points-v2"; + opp-shared; + + cpu7_opp1: opp-844800000 { + opp-hz = /bits/ 64 <844800000>; + opp-peak-kBps = <2188000 19660800>; + }; + + cpu7_opp2: opp-960000000 { + opp-hz = /bits/ 64 <960000000>; + opp-peak-kBps = <2188000 26419200>; + }; + + cpu7_opp3: opp-1075200000 { + opp-hz = /bits/ 64 <1075200000>; + opp-peak-kBps = <3072000 26419200>; + }; + + cpu7_opp4: opp-1190400000 { + opp-hz = /bits/ 64 <1190400000>; + opp-peak-kBps = <3072000 29491200>; + }; + + cpu7_opp5: opp-1305600000 { + opp-hz = /bits/ 64 <1305600000>; + opp-peak-kBps = <4068000 32563200>; + }; + + cpu7_opp6: opp-1401600000 { + opp-hz = /bits/ 64 <1401600000>; + opp-peak-kBps = <4068000 32563200>; + }; + + cpu7_opp7: opp-1516800000 { + opp-hz = /bits/ 64 <1516800000>; + opp-peak-kBps = <4068000 36249600>; + }; + + cpu7_opp8: opp-1632000000 { + opp-hz = /bits/ 64 <1632000000>; + opp-peak-kBps = <5412000 39321600>; + }; + + cpu7_opp9: opp-1747200000 { + opp-hz = /bits/ 64 <1708800000>; + opp-peak-kBps = <5412000 42393600>; + }; + + cpu7_opp10: opp-1862400000 { + opp-hz = /bits/ 64 <1862400000>; + opp-peak-kBps = <6220000 45465600>; + }; + + cpu7_opp11: opp-1977600000 { + opp-hz = /bits/ 64 <1977600000>; + opp-peak-kBps = <6220000 48537600>; + }; + + cpu7_opp12: opp-2073600000 { + opp-hz = /bits/ 64 <2073600000>; + opp-peak-kBps = <7216000 48537600>; + }; + + cpu7_opp13: opp-2169600000 { + opp-hz = /bits/ 64 <2169600000>; + opp-peak-kBps = <7216000 51609600>; + }; + + cpu7_opp14: opp-2265600000 { + opp-hz = /bits/ 64 <2265600000>; + opp-peak-kBps = <7216000 51609600>; + }; + + cpu7_opp15: opp-2361600000 { + opp-hz = /bits/ 64 <2361600000>; + opp-peak-kBps = <8368000 51609600>; + }; + + cpu7_opp16: opp-2457600000 { + opp-hz = /bits/ 64 <2457600000>; + opp-peak-kBps = <8368000 51609600>; + }; + + cpu7_opp17: opp-2553600000 { + opp-hz = /bits/ 64 <2553600000>; + opp-peak-kBps = <8368000 51609600>; + }; + + cpu7_opp18: opp-2649600000 { + opp-hz = /bits/ 64 <2649600000>; + opp-peak-kBps = <8368000 51609600>; + }; + + cpu7_opp19: opp-2745600000 { + opp-hz = /bits/ 64 <2745600000>; + opp-peak-kBps = <8368000 51609600>; + }; + + cpu7_opp20: opp-2841600000 { + opp-hz = /bits/ 64 <2841600000>; + opp-peak-kBps = <8368000 51609600>; + }; + }; + firmware { scm: scm { compatible = "qcom,scm"; @@ -273,13 +586,6 @@ memory@80000000 { reg = <0x0 0x80000000 0x0 0x0>; }; - mmcx_reg: mmcx-reg { - compatible = "regulator-fixed-domain"; - power-domains = <&rpmhpd SM8250_MMCX>; - required-opps = <&rpmhpd_opp_low_svs>; - regulator-name = "MMCX"; - }; - pmu { compatible = "arm,armv8-pmuv3"; interrupts = ; @@ -1463,7 +1769,7 @@ pcie0_phy: phy@1c06000 { status = "disabled"; - pcie0_lane: lanes@1c06200 { + pcie0_lane: phy@1c06200 { reg = <0 0x1c06200 0 0x170>, /* tx */ <0 0x1c06400 0 0x200>, /* rx */ <0 0x1c06800 0 0x1f0>, /* pcs */ @@ -1567,7 +1873,7 @@ pcie1_phy: phy@1c0e000 { status = "disabled"; - pcie1_lane: lanes@1c0e200 { + pcie1_lane: phy@1c0e200 { reg = <0 0x1c0e200 0 0x170>, /* tx0 */ <0 0x1c0e400 0 0x200>, /* rx0 */ <0 0x1c0ea00 0 0x1f0>, /* pcs */ @@ -1673,7 +1979,7 @@ pcie2_phy: phy@1c16000 { status = "disabled"; - pcie2_lane: lanes@1c16200 { + pcie2_lane: phy@1c16200 { reg = <0 0x1c16200 0 0x170>, /* tx0 */ <0 0x1c16400 0 0x200>, /* rx0 */ <0 0x1c16a00 0 0x1f0>, /* pcs */ @@ -1750,7 +2056,7 @@ ufs_mem_phy: phy@1d87000 { reset-names = "ufsphy"; status = "disabled"; - ufs_mem_phy_lanes: lanes@1d87400 { + ufs_mem_phy_lanes: phy@1d87400 { reg = <0 0x01d87400 0 0x108>, <0 0x01d87600 0 0x1e0>, <0 0x01d87c00 0 0x1dc>, @@ -1839,6 +2145,101 @@ vamacro: codec@3370000 { #sound-dai-cells = <1>; }; + rxmacro: rxmacro@3200000 { + pinctrl-names = "default"; + pinctrl-0 = <&rx_swr_active>; + compatible = "qcom,sm8250-lpass-rx-macro"; + reg = <0 0x3200000 0 0x1000>; + + clocks = <&q6afecc LPASS_CLK_ID_TX_CORE_MCLK LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&q6afecc LPASS_CLK_ID_TX_CORE_NPL_MCLK LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&q6afecc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&q6afecc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&vamacro>; + + clock-names = "mclk", "npl", "macro", "dcodec", "fsgen"; + + #clock-cells = <0>; + clock-frequency = <9600000>; + clock-output-names = "mclk"; + #sound-dai-cells = <1>; + }; + + swr1: soundwire-controller@3210000 { + reg = <0 0x3210000 0 0x2000>; + compatible = "qcom,soundwire-v1.5.1"; + interrupts = ; + clocks = <&rxmacro>; + clock-names = "iface"; + label = "RX"; + qcom,din-ports = <0>; + qcom,dout-ports = <5>; + + qcom,ports-sinterval-low = /bits/ 8 <0x03 0x1F 0x1F 0x07 0x00>; + qcom,ports-offset1 = /bits/ 8 <0x00 0x00 0x0B 0x01 0x00>; + qcom,ports-offset2 = /bits/ 8 <0x00 0x00 0x0B 0x00 0x00>; + qcom,ports-hstart = /bits/ 8 <0xFF 0x03 0xFF 0xFF 0xFF>; + qcom,ports-hstop = /bits/ 8 <0xFF 0x06 0xFF 0xFF 0xFF>; + qcom,ports-word-length = /bits/ 8 <0x01 0x07 0x04 0xFF 0xFF>; + qcom,ports-block-pack-mode = /bits/ 8 <0xFF 0x00 0x01 0xFF 0xFF>; + qcom,ports-lane-control = /bits/ 8 <0x01 0x00 0x00 0x00 0x00>; + qcom,ports-block-group-count = /bits/ 8 <0xFF 0xFF 0xFF 0xFF 0x00>; + + #sound-dai-cells = <1>; + #address-cells = <2>; + #size-cells = <0>; + }; + + txmacro: txmacro@3220000 { + pinctrl-names = "default"; + pinctrl-0 = <&tx_swr_active>; + compatible = "qcom,sm8250-lpass-tx-macro"; + reg = <0 0x3220000 0 0x1000>; + + clocks = <&q6afecc LPASS_CLK_ID_TX_CORE_MCLK LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&q6afecc LPASS_CLK_ID_TX_CORE_NPL_MCLK LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&q6afecc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&q6afecc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&vamacro>; + + clock-names = "mclk", "npl", "macro", "dcodec", "fsgen"; + + #clock-cells = <0>; + clock-frequency = <9600000>; + clock-output-names = "mclk"; + #address-cells = <2>; + #size-cells = <2>; + #sound-dai-cells = <1>; + }; + + /* tx macro */ + swr2: soundwire-controller@3230000 { + reg = <0 0x3230000 0 0x2000>; + compatible = "qcom,soundwire-v1.5.1"; + interrupts-extended = <&intc GIC_SPI 297 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "core"; + + clocks = <&txmacro>; + clock-names = "iface"; + label = "TX"; + + qcom,din-ports = <5>; + qcom,dout-ports = <0>; + qcom,ports-sinterval-low = /bits/ 8 <0xFF 0x01 0x01 0x03 0x03>; + qcom,ports-offset1 = /bits/ 8 <0xFF 0x01 0x00 0x02 0x00>; + qcom,ports-offset2 = /bits/ 8 <0xFF 0x00 0x00 0x00 0x00>; + qcom,ports-block-pack-mode = /bits/ 8 <0xFF 0xFF 0xFF 0xFF 0xFF>; + qcom,ports-hstart = /bits/ 8 <0xFF 0xFF 0xFF 0xFF 0xFF>; + qcom,ports-hstop = /bits/ 8 <0xFF 0xFF 0xFF 0xFF 0xFF>; + qcom,ports-word-length = /bits/ 8 <0xFF 0xFF 0xFF 0xFF 0xFF>; + qcom,ports-block-group-count = /bits/ 8 <0xFF 0xFF 0xFF 0xFF 0xFF>; + qcom,ports-lane-control = /bits/ 8 <0xFF 0x00 0x01 0x00 0x01>; + qcom,port-offset = <1>; + #sound-dai-cells = <1>; + #address-cells = <2>; + #size-cells = <0>; + }; + aoncc: clock-controller@3380000 { compatible = "qcom,sm8250-lpass-aoncc"; reg = <0 0x03380000 0 0x40000>; @@ -1931,12 +2332,73 @@ data { input-enable; }; }; + + rx_swr_active: rx_swr-active-pins { + clk { + pins = "gpio3"; + function = "swr_rx_clk"; + drive-strength = <2>; + slew-rate = <1>; + bias-disable; + }; + + data { + pins = "gpio4", "gpio5"; + function = "swr_rx_data"; + drive-strength = <2>; + slew-rate = <1>; + bias-bus-hold; + }; + }; + + tx_swr_active: tx_swr-active-pins { + clk { + pins = "gpio0"; + function = "swr_tx_clk"; + drive-strength = <2>; + slew-rate = <1>; + bias-disable; + }; + + data { + pins = "gpio1", "gpio2"; + function = "swr_tx_data"; + drive-strength = <2>; + slew-rate = <1>; + bias-bus-hold; + }; + }; + + tx_swr_sleep: tx_swr-sleep-pins { + clk { + pins = "gpio0"; + function = "swr_tx_clk"; + drive-strength = <2>; + input-enable; + bias-pull-down; + }; + + data1 { + pins = "gpio1"; + function = "swr_tx_data"; + drive-strength = <2>; + input-enable; + bias-bus-hold; + }; + + data2 { + pins = "gpio2"; + function = "swr_tx_data"; + drive-strength = <2>; + input-enable; + bias-pull-down; + }; + }; }; gpu: gpu@3d00000 { compatible = "qcom,adreno-650.2", "qcom,adreno"; - #stream-id-cells = <16>; reg = <0 0x03d00000 0 0x40000>; reg-names = "kgsl_3d0_reg_memory"; @@ -2088,13 +2550,14 @@ slpi: remoteproc@5c00000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_SLPI>, - <&rpmhpd SM8250_LCX>, + power-domains = <&rpmhpd SM8250_LCX>, <&rpmhpd SM8250_LMX>; - power-domain-names = "load_state", "lcx", "lmx"; + power-domain-names = "lcx", "lmx"; memory-region = <&slpi_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&smp2p_slpi_out 0>; qcom,smem-state-names = "stop"; @@ -2154,12 +2617,12 @@ cdsp: remoteproc@8300000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_CDSP>, - <&rpmhpd SM8250_CX>; - power-domain-names = "load_state", "cx"; + power-domains = <&rpmhpd SM8250_CX>; memory-region = <&cdsp_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&smp2p_cdsp_out 0>; qcom,smem-state-names = "stop"; @@ -2330,7 +2793,7 @@ usb_2_qmpphy: phy@88eb000 { <&gcc GCC_USB3_PHY_SEC_BCR>; reset-names = "phy", "common"; - usb_2_ssphy: lanes@88eb200 { + usb_2_ssphy: phy@88eb200 { reg = <0 0x088eb200 0 0x200>, <0 0x088eb400 0 0x200>, <0 0x088eb800 0 0x800>; @@ -2571,7 +3034,8 @@ videocc: clock-controller@abf0000 { clocks = <&gcc GCC_VIDEO_AHB_CLK>, <&rpmhcc RPMH_CXO_CLK>, <&rpmhcc RPMH_CXO_CLK_A>; - mmcx-supply = <&mmcx_reg>; + power-domains = <&rpmhpd SM8250_MMCX>; + required-opps = <&rpmhpd_opp_low_svs>; clock-names = "iface", "bi_tcxo", "bi_tcxo_ao"; #clock-cells = <1>; #reset-cells = <1>; @@ -2846,7 +3310,8 @@ opp-358000000 { dispcc: clock-controller@af00000 { compatible = "qcom,sm8250-dispcc"; reg = <0 0x0af00000 0 0x10000>; - mmcx-supply = <&mmcx_reg>; + power-domains = <&rpmhpd SM8250_MMCX>; + required-opps = <&rpmhpd_opp_low_svs>; clocks = <&rpmhcc RPMH_CXO_CLK>, <&dsi0_phy 0>, <&dsi0_phy 1>, @@ -2900,7 +3365,7 @@ tsens1: thermal-sensor@c265000 { aoss_qmp: power-controller@c300000 { compatible = "qcom,sm8250-aoss-qmp"; - reg = <0 0x0c300000 0 0x100000>; + reg = <0 0x0c300000 0 0x400>; interrupts-extended = <&ipcc IPCC_CLIENT_AOP IPCC_MPROC_SIGNAL_GLINK_QMP IRQ_TYPE_EDGE_RISING>; @@ -2908,7 +3373,11 @@ IPCC_MPROC_SIGNAL_GLINK_QMP IPCC_MPROC_SIGNAL_GLINK_QMP>; #clock-cells = <0>; - #power-domain-cells = <1>; + }; + + sram@c3f0000 { + compatible = "qcom,rpmh-stats"; + reg = <0 0x0c3f0000 0 0x400>; }; spmi_bus: spmi@c440000 { @@ -3825,13 +4294,14 @@ adsp: remoteproc@17300000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_LPASS>, - <&rpmhpd SM8250_LCX>, + power-domains = <&rpmhpd SM8250_LCX>, <&rpmhpd SM8250_LMX>; - power-domain-names = "load_state", "lcx", "lmx"; + power-domain-names = "lcx", "lmx"; memory-region = <&adsp_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&smp2p_adsp_out 0>; qcom,smem-state-names = "stop"; diff --git a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts index 56093e260d..1e5e9405d8 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts @@ -5,7 +5,6 @@ /dts-v1/; -#include #include #include "sm8350.dtsi" @@ -251,9 +250,7 @@ &ufs_mem_phy { status = "okay"; vdda-phy-supply = <&vreg_l5b_0p88>; - vdda-max-microamp = <91600>; vdda-pll-supply = <&vreg_l6b_1p2>; - vdda-pll-max-microamp = <19000>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8350-mtp.dts b/arch/arm64/boot/dts/qcom/sm8350-mtp.dts index bd95009c18..52cf304560 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8350-mtp.dts @@ -5,7 +5,6 @@ /dts-v1/; -#include #include #include "sm8350.dtsi" #include "pm8350.dtsi" @@ -286,6 +285,19 @@ &mpss { firmware-name = "qcom/sm8350/modem.mbn"; }; +&pmk8350_rtc { + status = "okay"; +}; + +&pon_pwrkey { + status = "okay"; +}; + +&pon_resin { + status = "okay"; + linux,code = ; +}; + &qupv3_id_0 { status = "okay"; }; @@ -318,9 +330,7 @@ &ufs_mem_phy { status = "okay"; vdda-phy-supply = <&vreg_l5b_0p88>; - vdda-max-microamp = <91600>; vdda-pll-supply = <&vreg_l6b_1p2>; - vdda-pll-max-microamp = <19000>; }; &usb_1 { diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi index a8886adaaf..4b19744bcf 100644 --- a/arch/arm64/boot/dts/qcom/sm8350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi @@ -6,9 +6,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -66,6 +66,8 @@ CPU0: cpu@0 { enable-method = "psci"; next-level-cache = <&L2_0>; qcom,freq-domain = <&cpufreq_hw 0>; + power-domains = <&CPU_PD0>; + power-domain-names = "psci"; #cooling-cells = <2>; L2_0: l2-cache { compatible = "cache"; @@ -83,6 +85,8 @@ CPU1: cpu@100 { enable-method = "psci"; next-level-cache = <&L2_100>; qcom,freq-domain = <&cpufreq_hw 0>; + power-domains = <&CPU_PD1>; + power-domain-names = "psci"; #cooling-cells = <2>; L2_100: l2-cache { compatible = "cache"; @@ -97,6 +101,8 @@ CPU2: cpu@200 { enable-method = "psci"; next-level-cache = <&L2_200>; qcom,freq-domain = <&cpufreq_hw 0>; + power-domains = <&CPU_PD2>; + power-domain-names = "psci"; #cooling-cells = <2>; L2_200: l2-cache { compatible = "cache"; @@ -111,6 +117,8 @@ CPU3: cpu@300 { enable-method = "psci"; next-level-cache = <&L2_300>; qcom,freq-domain = <&cpufreq_hw 0>; + power-domains = <&CPU_PD3>; + power-domain-names = "psci"; #cooling-cells = <2>; L2_300: l2-cache { compatible = "cache"; @@ -125,6 +133,8 @@ CPU4: cpu@400 { enable-method = "psci"; next-level-cache = <&L2_400>; qcom,freq-domain = <&cpufreq_hw 1>; + power-domains = <&CPU_PD4>; + power-domain-names = "psci"; #cooling-cells = <2>; L2_400: l2-cache { compatible = "cache"; @@ -139,6 +149,8 @@ CPU5: cpu@500 { enable-method = "psci"; next-level-cache = <&L2_500>; qcom,freq-domain = <&cpufreq_hw 1>; + power-domains = <&CPU_PD5>; + power-domain-names = "psci"; #cooling-cells = <2>; L2_500: l2-cache { compatible = "cache"; @@ -154,6 +166,8 @@ CPU6: cpu@600 { enable-method = "psci"; next-level-cache = <&L2_600>; qcom,freq-domain = <&cpufreq_hw 1>; + power-domains = <&CPU_PD6>; + power-domain-names = "psci"; #cooling-cells = <2>; L2_600: l2-cache { compatible = "cache"; @@ -168,12 +182,86 @@ CPU7: cpu@700 { enable-method = "psci"; next-level-cache = <&L2_700>; qcom,freq-domain = <&cpufreq_hw 2>; + power-domains = <&CPU_PD7>; + power-domain-names = "psci"; #cooling-cells = <2>; L2_700: l2-cache { compatible = "cache"; next-level-cache = <&L3_0>; }; }; + + cpu-map { + cluster0 { + core0 { + cpu = <&CPU0>; + }; + + core1 { + cpu = <&CPU1>; + }; + + core2 { + cpu = <&CPU2>; + }; + + core3 { + cpu = <&CPU3>; + }; + + core4 { + cpu = <&CPU4>; + }; + + core5 { + cpu = <&CPU5>; + }; + + core6 { + cpu = <&CPU6>; + }; + + core7 { + cpu = <&CPU7>; + }; + }; + }; + + idle-states { + entry-method = "psci"; + + LITTLE_CPU_SLEEP_0: cpu-sleep-0-0 { + compatible = "arm,idle-state"; + idle-state-name = "silver-rail-power-collapse"; + arm,psci-suspend-param = <0x40000004>; + entry-latency-us = <355>; + exit-latency-us = <909>; + min-residency-us = <3934>; + local-timer-stop; + }; + + BIG_CPU_SLEEP_0: cpu-sleep-1-0 { + compatible = "arm,idle-state"; + idle-state-name = "gold-rail-power-collapse"; + arm,psci-suspend-param = <0x40000004>; + entry-latency-us = <241>; + exit-latency-us = <1461>; + min-residency-us = <4488>; + local-timer-stop; + }; + }; + + domain-idle-states { + CLUSTER_SLEEP_0: cluster-sleep-0 { + compatible = "domain-idle-state"; + idle-state-name = "cluster-power-collapse"; + arm,psci-suspend-param = <0x4100c344>; + entry-latency-us = <3263>; + exit-latency-us = <6562>; + min-residency-us = <9987>; + local-timer-stop; + }; + }; }; firmware { @@ -197,6 +285,59 @@ pmu { psci { compatible = "arm,psci-1.0"; method = "smc"; + + CPU_PD0: cpu0 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER_PD>; + domain-idle-states = <&LITTLE_CPU_SLEEP_0>; + }; + + CPU_PD1: cpu1 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER_PD>; + domain-idle-states = <&LITTLE_CPU_SLEEP_0>; + }; + + CPU_PD2: cpu2 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER_PD>; + domain-idle-states = <&LITTLE_CPU_SLEEP_0>; + }; + + CPU_PD3: cpu3 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER_PD>; + domain-idle-states = <&LITTLE_CPU_SLEEP_0>; + }; + + CPU_PD4: cpu4 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER_PD>; + domain-idle-states = <&BIG_CPU_SLEEP_0>; + }; + + CPU_PD5: cpu5 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER_PD>; + domain-idle-states = <&BIG_CPU_SLEEP_0>; + }; + + CPU_PD6: cpu6 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER_PD>; + domain-idle-states = <&BIG_CPU_SLEEP_0>; + }; + + CPU_PD7: cpu7 { + #power-domain-cells = <0>; + power-domains = <&CLUSTER_PD>; + domain-idle-states = <&BIG_CPU_SLEEP_0>; + }; + + CLUSTER_PD: cpu-cluster0 { + #power-domain-cells = <0>; + domain-idle-states = <&CLUSTER_SLEEP_0>; + }; }; reserved_memory: reserved-memory { @@ -496,17 +637,305 @@ ipcc: mailbox@408000 { #mbox-cells = <2>; }; + qup_opp_table_100mhz: qup-100mhz-opp-table { + compatible = "operating-points-v2"; + + opp-50000000 { + opp-hz = /bits/ 64 <50000000>; + required-opps = <&rpmhpd_opp_min_svs>; + }; + + opp-75000000 { + opp-hz = /bits/ 64 <75000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + }; + + qup_opp_table_120mhz: qup-120mhz-opp-table { + compatible = "operating-points-v2"; + + opp-50000000 { + opp-hz = /bits/ 64 <50000000>; + required-opps = <&rpmhpd_opp_min_svs>; + }; + + opp-75000000 { + opp-hz = /bits/ 64 <75000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-120000000 { + opp-hz = /bits/ 64 <120000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + }; + + qupv3_id_2: geniqup@8c0000 { + compatible = "qcom,geni-se-qup"; + reg = <0x0 0x008c0000 0x0 0x6000>; + clock-names = "m-ahb", "s-ahb"; + clocks = <&gcc GCC_QUPV3_WRAP_2_M_AHB_CLK>, + <&gcc GCC_QUPV3_WRAP_2_S_AHB_CLK>; + iommus = <&apps_smmu 0x5e3 0x0>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + status = "disabled"; + + i2c14: i2c@880000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00880000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S0_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c14_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi14: spi@880000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00880000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S0_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_120mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c15: i2c@884000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00884000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S1_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c15_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi15: spi@884000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00884000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S1_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_120mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c16: i2c@888000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00888000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S2_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c16_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi16: spi@888000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00888000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S2_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c17: i2c@88c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x0088c000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c17_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi17: spi@88c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x0088c000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + /* QUP no. 18 seems to be strictly SPI/UART-only */ + + spi18: spi@890000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00890000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S4_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart18: serial@890000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00890000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S4_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart18_default>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + status = "disabled"; + }; + + i2c19: i2c@894000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00894000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S5_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c19_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi19: spi@894000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00894000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP2_S5_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + }; + qupv3_id_0: geniqup@9c0000 { compatible = "qcom,geni-se-qup"; reg = <0x0 0x009c0000 0x0 0x6000>; clock-names = "m-ahb", "s-ahb"; clocks = <&gcc GCC_QUPV3_WRAP_0_M_AHB_CLK>, <&gcc GCC_QUPV3_WRAP_0_S_AHB_CLK>; + iommus = <&apps_smmu 0x5a3 0>; #address-cells = <2>; #size-cells = <2>; ranges; status = "disabled"; + i2c0: i2c@980000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00980000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c0_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi0: spi@980000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00980000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c1: i2c@984000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00984000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c1_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi1: spi@984000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00984000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c2: i2c@988000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00988000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c2_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi2: spi@988000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00988000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + uart2: serial@98c000 { compatible = "qcom,geni-debug-uart"; reg = <0 0x0098c000 0 0x4000>; @@ -515,6 +944,140 @@ uart2: serial@98c000 { pinctrl-names = "default"; pinctrl-0 = <&qup_uart3_default_state>; interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + /* QUP no. 3 seems to be strictly SPI-only */ + + spi3: spi@98c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x0098c000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c4: i2c@990000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00990000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c4_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi4: spi@990000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00990000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c5: i2c@994000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00994000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c5_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi5: spi@994000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00994000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c6: i2c@998000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00998000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c6_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi6: spi@998000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00998000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart6: serial@998000 { + compatible = "qcom,geni-uart"; + reg = <0 0x00998000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_uart6_default>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + status = "disabled"; + }; + + i2c7: i2c@99c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x0099c000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c7_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi7: spi@99c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x0099c000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; #address-cells = <1>; #size-cells = <0>; status = "disabled"; @@ -527,23 +1090,167 @@ qupv3_id_1: geniqup@ac0000 { clock-names = "m-ahb", "s-ahb"; clocks = <&gcc GCC_QUPV3_WRAP_1_M_AHB_CLK>, <&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>; + iommus = <&apps_smmu 0x43 0>; #address-cells = <2>; #size-cells = <2>; ranges; status = "disabled"; + i2c8: i2c@a80000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a80000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c8_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi8: spi@a80000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a80000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_120mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c9: i2c@a84000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a84000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c9_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi9: spi@a84000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a84000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c10: i2c@a88000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a88000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c10_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi10: spi@a88000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a88000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c11: i2c@a8c000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a8c000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c11_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi11: spi@a8c000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a8c000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c12: i2c@a90000 { + compatible = "qcom,geni-i2c"; + reg = <0 0x00a90000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; + pinctrl-names = "default"; + pinctrl-0 = <&qup_i2c12_default>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi12: spi@a90000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a90000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + i2c13: i2c@a94000 { compatible = "qcom,geni-i2c"; reg = <0 0x00a94000 0 0x4000>; clock-names = "se"; clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>; pinctrl-names = "default"; - pinctrl-0 = <&qup_i2c13_default_state>; + pinctrl-0 = <&qup_i2c13_default>; interrupts = ; #address-cells = <1>; #size-cells = <0>; status = "disabled"; }; + + spi13: spi@a94000 { + compatible = "qcom,geni-spi"; + reg = <0 0x00a94000 0 0x4000>; + clock-names = "se"; + clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>; + interrupts = ; + power-domains = <&rpmhpd SM8350_CX>; + operating-points-v2 = <&qup_opp_table_100mhz>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; }; apps_smmu: iommu@15000000 { @@ -766,15 +1473,16 @@ mpss: remoteproc@4080000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_MODEM>, - <&rpmhpd 0>, + power-domains = <&rpmhpd 0>, <&rpmhpd 12>; - power-domain-names = "load_state", "cx", "mss"; + power-domain-names = "cx", "mss"; interconnects = <&mc_virt MASTER_LLCC &mc_virt SLAVE_EBI1>; memory-region = <&pil_modem_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&smp2p_modem_out 0>; qcom,smem-state-names = "stop"; @@ -809,7 +1517,7 @@ tsens0: thermal-sensor@c263000 { reg = <0 0x0c263000 0 0x1ff>, /* TM */ <0 0x0c222000 0 0x8>; /* SROT */ #qcom,sensors = <15>; - interrupts = <&pdc 26 IRQ_TYPE_LEVEL_HIGH>, + interrupts-extended = <&pdc 26 IRQ_TYPE_LEVEL_HIGH>, <&pdc 28 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "uplow", "critical"; #thermal-sensor-cells = <1>; @@ -820,7 +1528,7 @@ tsens1: thermal-sensor@c265000 { reg = <0 0x0c265000 0 0x1ff>, /* TM */ <0 0x0c223000 0 0x8>; /* SROT */ #qcom,sensors = <14>; - interrupts = <&pdc 27 IRQ_TYPE_LEVEL_HIGH>, + interrupts-extended = <&pdc 27 IRQ_TYPE_LEVEL_HIGH>, <&pdc 29 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "uplow", "critical"; #thermal-sensor-cells = <1>; @@ -828,13 +1536,17 @@ tsens1: thermal-sensor@c265000 { aoss_qmp: power-controller@c300000 { compatible = "qcom,sm8350-aoss-qmp"; - reg = <0 0x0c300000 0 0x100000>; + reg = <0 0x0c300000 0 0x400>; interrupts-extended = <&ipcc IPCC_CLIENT_AOP IPCC_MPROC_SIGNAL_GLINK_QMP IRQ_TYPE_EDGE_RISING>; mboxes = <&ipcc IPCC_CLIENT_AOP IPCC_MPROC_SIGNAL_GLINK_QMP>; #clock-cells = <0>; - #power-domain-cells = <1>; + }; + + sram@c3f0000 { + compatible = "qcom,rpmh-stats"; + reg = <0 0x0c3f0000 0 0x400>; }; spmi_bus: spmi@c440000 { @@ -877,17 +1589,144 @@ tx { }; }; - qup_i2c13_default_state: qup-i2c13-default-state { - mux { - pins = "gpio0", "gpio1"; - function = "qup13"; - }; + qup_uart6_default: qup-uart6-default { + pins = "gpio30", "gpio31"; + function = "qup6"; + drive-strength = <2>; + bias-disable; + }; - config { - pins = "gpio0", "gpio1"; - drive-strength = <2>; - bias-pull-up; - }; + qup_uart18_default: qup-uart18-default { + pins = "gpio58", "gpio59"; + function = "qup18"; + drive-strength = <2>; + bias-disable; + }; + + qup_i2c0_default: qup-i2c0-default { + pins = "gpio4", "gpio5"; + function = "qup0"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c1_default: qup-i2c1-default { + pins = "gpio8", "gpio9"; + function = "qup1"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c2_default: qup-i2c2-default { + pins = "gpio12", "gpio13"; + function = "qup2"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c4_default: qup-i2c4-default { + pins = "gpio20", "gpio21"; + function = "qup4"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c5_default: qup-i2c5-default { + pins = "gpio24", "gpio25"; + function = "qup5"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c6_default: qup-i2c6-default { + pins = "gpio28", "gpio29"; + function = "qup6"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c7_default: qup-i2c7-default { + pins = "gpio32", "gpio33"; + function = "qup7"; + drive-strength = <2>; + bias-disable; + }; + + qup_i2c8_default: qup-i2c8-default { + pins = "gpio36", "gpio37"; + function = "qup8"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c9_default: qup-i2c9-default { + pins = "gpio40", "gpio41"; + function = "qup9"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c10_default: qup-i2c10-default { + pins = "gpio44", "gpio45"; + function = "qup10"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c11_default: qup-i2c11-default { + pins = "gpio48", "gpio49"; + function = "qup11"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c12_default: qup-i2c12-default { + pins = "gpio52", "gpio53"; + function = "qup12"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c13_default: qup-i2c13-default { + pins = "gpio0", "gpio1"; + function = "qup13"; + drive-strength = <2>; + bias-pull-up; + }; + + qup_i2c14_default: qup-i2c14-default { + pins = "gpio56", "gpio57"; + function = "qup14"; + drive-strength = <2>; + bias-disable; + }; + + qup_i2c15_default: qup-i2c15-default { + pins = "gpio60", "gpio61"; + function = "qup15"; + drive-strength = <2>; + bias-disable; + }; + + qup_i2c16_default: qup-i2c16-default { + pins = "gpio64", "gpio65"; + function = "qup16"; + drive-strength = <2>; + bias-disable; + }; + + qup_i2c17_default: qup-i2c17-default { + pins = "gpio72", "gpio73"; + function = "qup17"; + drive-strength = <2>; + bias-disable; + }; + + qup_i2c19_default: qup-i2c19-default { + pins = "gpio76", "gpio77"; + function = "qup19"; + drive-strength = <2>; + bias-disable; }; }; @@ -902,6 +1741,8 @@ intc: interrupt-controller@17a00000 { compatible = "arm,gic-v3"; #interrupt-cells = <3>; interrupt-controller; + #redistributor-regions = <1>; + redistributor-stride = <0 0x20000>; reg = <0x0 0x17a00000 0x0 0x10000>, /* GICD */ <0x0 0x17a60000 0x0 0x100000>; /* GICR * 8 */ interrupts = ; @@ -1110,7 +1951,6 @@ ufs_mem_phy: phy@1d87000 { reg = <0 0x01d87000 0 0xe10>; #address-cells = <2>; #size-cells = <2>; - #clock-cells = <1>; ranges; clock-names = "ref", "ref_aux"; @@ -1121,7 +1961,7 @@ ufs_mem_phy: phy@1d87000 { reset-names = "ufsphy"; status = "disabled"; - ufs_mem_phy_lanes: lanes@1d87400 { + ufs_mem_phy_lanes: phy@1d87400 { reg = <0 0x01d87400 0 0x108>, <0 0x01d87600 0 0x1e0>, <0 0x01d87c00 0 0x1dc>, @@ -1147,13 +1987,14 @@ slpi: remoteproc@5c00000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_SLPI>, - <&rpmhpd 4>, + power-domains = <&rpmhpd 4>, <&rpmhpd 5>; - power-domain-names = "load_state", "lcx", "lmx"; + power-domain-names = "lcx", "lmx"; memory-region = <&pil_slpi_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&smp2p_slpi_out 0>; qcom,smem-state-names = "stop"; @@ -1169,6 +2010,32 @@ IPCC_MPROC_SIGNAL_GLINK_QMP label = "slpi"; qcom,remote-pid = <3>; + fastrpc { + compatible = "qcom,fastrpc"; + qcom,glink-channels = "fastrpcglink-apps-dsp"; + label = "sdsp"; + #address-cells = <1>; + #size-cells = <0>; + + compute-cb@1 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <1>; + iommus = <&apps_smmu 0x0541 0x0>; + }; + + compute-cb@2 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <2>; + iommus = <&apps_smmu 0x0542 0x0>; + }; + + compute-cb@3 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <3>; + iommus = <&apps_smmu 0x0543 0x0>; + /* note: shared-cb = <4> in downstream */ + }; + }; }; }; @@ -1187,15 +2054,16 @@ cdsp: remoteproc@98900000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_CDSP>, - <&rpmhpd 0>, + power-domains = <&rpmhpd 0>, <&rpmhpd 10>; - power-domain-names = "load_state", "cx", "mxc"; + power-domain-names = "cx", "mxc"; interconnects = <&compute_noc MASTER_CDSP_PROC &mc_virt SLAVE_EBI1>; memory-region = <&pil_cdsp_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&smp2p_cdsp_out 0>; qcom,smem-state-names = "stop"; @@ -1210,6 +2078,72 @@ IPCC_MPROC_SIGNAL_GLINK_QMP label = "cdsp"; qcom,remote-pid = <5>; + + fastrpc { + compatible = "qcom,fastrpc"; + qcom,glink-channels = "fastrpcglink-apps-dsp"; + label = "cdsp"; + #address-cells = <1>; + #size-cells = <0>; + + compute-cb@1 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <1>; + iommus = <&apps_smmu 0x2161 0x0400>, + <&apps_smmu 0x1181 0x0420>; + }; + + compute-cb@2 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <2>; + iommus = <&apps_smmu 0x2162 0x0400>, + <&apps_smmu 0x1182 0x0420>; + }; + + compute-cb@3 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <3>; + iommus = <&apps_smmu 0x2163 0x0400>, + <&apps_smmu 0x1183 0x0420>; + }; + + compute-cb@4 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <4>; + iommus = <&apps_smmu 0x2164 0x0400>, + <&apps_smmu 0x1184 0x0420>; + }; + + compute-cb@5 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <5>; + iommus = <&apps_smmu 0x2165 0x0400>, + <&apps_smmu 0x1185 0x0420>; + }; + + compute-cb@6 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <6>; + iommus = <&apps_smmu 0x2166 0x0400>, + <&apps_smmu 0x1186 0x0420>; + }; + + compute-cb@7 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <7>; + iommus = <&apps_smmu 0x2167 0x0400>, + <&apps_smmu 0x1187 0x0420>; + }; + + compute-cb@8 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <8>; + iommus = <&apps_smmu 0x2168 0x0400>, + <&apps_smmu 0x1188 0x0420>; + }; + + /* note: secure cb9 in downstream */ + }; }; }; @@ -1243,9 +2177,7 @@ usb_1_qmpphy: phy-wrapper@88e9000 { compatible = "qcom,sm8350-qmp-usb3-phy"; reg = <0 0x088e9000 0 0x200>, <0 0x088e8000 0 0x20>; - reg-names = "reg-base", "dp_com"; status = "disabled"; - #clock-cells = <1>; #address-cells = <2>; #size-cells = <2>; ranges; @@ -1278,7 +2210,6 @@ usb_2_qmpphy: phy-wrapper@88eb000 { compatible = "qcom,sm8350-qmp-usb3-uni-phy"; reg = <0 0x088eb000 0 0x200>; status = "disabled"; - #clock-cells = <1>; #address-cells = <2>; #size-cells = <2>; ranges; @@ -1319,6 +2250,12 @@ gem_noc: interconnect@9100000 { qcom,bcm-voters = <&apps_bcm_voter>; }; + system-cache-controller@9200000 { + compatible = "qcom,sm8350-llcc"; + reg = <0 0x09200000 0 0x1d0000>, <0 0x09600000 0 0x50000>; + reg-names = "llcc_base", "llcc_broadcast_base"; + }; + usb_1: usb@a6f8800 { compatible = "qcom,sm8350-dwc3", "qcom,dwc3"; reg = <0 0x0a6f8800 0 0x400>; @@ -1421,13 +2358,14 @@ adsp: remoteproc@17300000 { clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "xo"; - power-domains = <&aoss_qmp AOSS_QMP_LS_LPASS>, - <&rpmhpd 4>, + power-domains = <&rpmhpd 4>, <&rpmhpd 5>; - power-domain-names = "load_state", "lcx", "lmx"; + power-domain-names = "lcx", "lmx"; memory-region = <&pil_adsp_mem>; + qcom,qmp = <&aoss_qmp>; + qcom,smem-states = <&smp2p_adsp_out 0>; qcom,smem-state-names = "stop"; @@ -1442,6 +2380,32 @@ IPCC_MPROC_SIGNAL_GLINK_QMP label = "lpass"; qcom,remote-pid = <2>; + + fastrpc { + compatible = "qcom,fastrpc"; + qcom,glink-channels = "fastrpcglink-apps-dsp"; + label = "adsp"; + #address-cells = <1>; + #size-cells = <0>; + + compute-cb@3 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <3>; + iommus = <&apps_smmu 0x1803 0x0>; + }; + + compute-cb@4 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <4>; + iommus = <&apps_smmu 0x1804 0x0>; + }; + + compute-cb@5 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <5>; + iommus = <&apps_smmu 0x1805 0x0>; + }; + }; }; }; }; diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 10c25ad2d0..02b97e838c 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -726,7 +726,7 @@ apps_smmu: iommu@15000000 { compatible = "qcom,sm8450-smmu-500", "arm,mmu-500"; reg = <0 0x15000000 0 0x100000>; #iommu-cells = <2>; - #global-interrupts = <2>; + #global-interrupts = <1>; interrupts = , , , @@ -813,6 +813,7 @@ apps_smmu: iommu@15000000 { , , , + , , , , @@ -1072,9 +1073,10 @@ usb_1: usb@a6f8800 { <&gcc GCC_USB30_PRIM_MASTER_CLK>, <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>, <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>, - <&gcc GCC_USB30_PRIM_SLEEP_CLK>; + <&gcc GCC_USB30_PRIM_SLEEP_CLK>, + <&gcc GCC_USB3_0_CLKREF_EN>; clock-names = "cfg_noc", "core", "iface", "mock_utmi", - "sleep"; + "sleep", "xo"; assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>, <&gcc GCC_USB30_PRIM_MASTER_CLK>; diff --git a/arch/arm64/boot/dts/renesas/Makefile b/arch/arm64/boot/dts/renesas/Makefile index 15a53b5139..5bc8065a78 100644 --- a/arch/arm64/boot/dts/renesas/Makefile +++ b/arch/arm64/boot/dts/renesas/Makefile @@ -63,6 +63,8 @@ dtb-$(CONFIG_ARCH_R8A77995) += r8a77995-draak.dtb dtb-$(CONFIG_ARCH_R8A779A0) += r8a779a0-falcon.dtb +dtb-$(CONFIG_ARCH_R8A779F0) += r8a779f0-spider.dtb + dtb-$(CONFIG_ARCH_R8A77951) += r8a779m1-salvator-xs.dtb dtb-$(CONFIG_ARCH_R8A77951) += r8a779m1-ulcb.dtb dtb-$(CONFIG_ARCH_R8A77951) += r8a779m1-ulcb-kf.dtb @@ -71,4 +73,6 @@ dtb-$(CONFIG_ARCH_R8A77961) += r8a779m3-salvator-xs.dtb dtb-$(CONFIG_ARCH_R8A77961) += r8a779m3-ulcb.dtb dtb-$(CONFIG_ARCH_R8A77961) += r8a779m3-ulcb-kf.dtb +dtb-$(CONFIG_ARCH_R8A77965) += r8a779m5-salvator-xs.dtb + dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-smarc.dtb diff --git a/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi b/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi index 937d17a426..0d136809eb 100644 --- a/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi +++ b/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi @@ -59,6 +59,8 @@ &avb { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id004d.d074", + "ethernet-phy-ieee802.3-c22"; reg = <0>; interrupt-parent = <&gpio2>; interrupts = <11 IRQ_TYPE_LEVEL_LOW>; diff --git a/arch/arm64/boot/dts/renesas/cat875.dtsi b/arch/arm64/boot/dts/renesas/cat875.dtsi index 20f8adc635..8c9da8b4bd 100644 --- a/arch/arm64/boot/dts/renesas/cat875.dtsi +++ b/arch/arm64/boot/dts/renesas/cat875.dtsi @@ -22,6 +22,8 @@ &avb { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id001c.c915", + "ethernet-phy-ieee802.3-c22"; reg = <0>; interrupt-parent = <&gpio2>; interrupts = <21 IRQ_TYPE_LEVEL_LOW>; diff --git a/arch/arm64/boot/dts/renesas/hihope-rzg2-ex.dtsi b/arch/arm64/boot/dts/renesas/hihope-rzg2-ex.dtsi index dde3a07bc4..ad898c6db4 100644 --- a/arch/arm64/boot/dts/renesas/hihope-rzg2-ex.dtsi +++ b/arch/arm64/boot/dts/renesas/hihope-rzg2-ex.dtsi @@ -24,6 +24,8 @@ &avb { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id001c.c915", + "ethernet-phy-ieee802.3-c22"; reg = <0>; interrupt-parent = <&gpio2>; interrupts = <11 IRQ_TYPE_LEVEL_LOW>; diff --git a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi index e70aa5a087..eda6a84037 100644 --- a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi @@ -58,7 +58,7 @@ can_clk: can { clock-frequency = <0>; }; - cluster0_opp: opp_table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -80,7 +80,7 @@ opp-1500000000 { }; }; - cluster1_opp: opp_table1 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; @@ -2276,7 +2276,8 @@ sdhi0: mmc@ee100000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee100000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A774A1_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774A1_PD_ALWAYS_ON>; resets = <&cpg 314>; @@ -2288,7 +2289,8 @@ sdhi1: mmc@ee120000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee120000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 313>; + clocks = <&cpg CPG_MOD 313>, <&cpg CPG_CORE R8A774A1_CLK_SD1H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774A1_PD_ALWAYS_ON>; resets = <&cpg 313>; @@ -2300,7 +2302,8 @@ sdhi2: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 312>; + clocks = <&cpg CPG_MOD 312>, <&cpg CPG_CORE R8A774A1_CLK_SD2H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774A1_PD_ALWAYS_ON>; resets = <&cpg 312>; @@ -2312,7 +2315,8 @@ sdhi3: mmc@ee160000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee160000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 311>; + clocks = <&cpg CPG_MOD 311>, <&cpg CPG_CORE R8A774A1_CLK_SD3H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774A1_PD_ALWAYS_ON>; resets = <&cpg 311>; diff --git a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi index 6c5694fa66..44f79fbd75 100644 --- a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi @@ -47,7 +47,7 @@ can_clk: can { clock-frequency = <0>; }; - cluster0_opp: opp_table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -2133,7 +2133,8 @@ sdhi0: mmc@ee100000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee100000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A774B1_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774B1_PD_ALWAYS_ON>; resets = <&cpg 314>; @@ -2145,7 +2146,8 @@ sdhi1: mmc@ee120000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee120000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 313>; + clocks = <&cpg CPG_MOD 313>, <&cpg CPG_CORE R8A774B1_CLK_SD1H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774B1_PD_ALWAYS_ON>; resets = <&cpg 313>; @@ -2157,7 +2159,8 @@ sdhi2: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 312>; + clocks = <&cpg CPG_MOD 312>, <&cpg CPG_CORE R8A774B1_CLK_SD2H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774B1_PD_ALWAYS_ON>; resets = <&cpg 312>; @@ -2169,7 +2172,8 @@ sdhi3: mmc@ee160000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee160000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 311>; + clocks = <&cpg CPG_MOD 311>, <&cpg CPG_CORE R8A774B1_CLK_SD3H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774B1_PD_ALWAYS_ON>; resets = <&cpg 311>; diff --git a/arch/arm64/boot/dts/renesas/r8a774c0.dtsi b/arch/arm64/boot/dts/renesas/r8a774c0.dtsi index d597772c4c..b8dcbbbf3d 100644 --- a/arch/arm64/boot/dts/renesas/r8a774c0.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a774c0.dtsi @@ -44,7 +44,7 @@ can_clk: can { clock-frequency = <0>; }; - cluster1_opp: opp_table10 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; opp-800000000 { @@ -1626,7 +1626,8 @@ sdhi0: mmc@ee100000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee100000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A774C0_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774C0_PD_ALWAYS_ON>; resets = <&cpg 314>; @@ -1638,7 +1639,8 @@ sdhi1: mmc@ee120000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee120000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 313>; + clocks = <&cpg CPG_MOD 313>, <&cpg CPG_CORE R8A774C0_CLK_SD1H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774C0_PD_ALWAYS_ON>; resets = <&cpg 313>; @@ -1650,7 +1652,8 @@ sdhi3: mmc@ee160000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee160000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 311>; + clocks = <&cpg CPG_MOD 311>, <&cpg CPG_CORE R8A774C0_CLK_SD3H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774C0_PD_ALWAYS_ON>; resets = <&cpg 311>; diff --git a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi index 62209ab6de..e6d8610730 100644 --- a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi @@ -47,7 +47,7 @@ can_clk: can { clock-frequency = <0>; }; - cluster0_opp: opp_table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -69,7 +69,7 @@ opp-1500000000 { }; }; - cluster1_opp: opp_table1 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; @@ -2362,7 +2362,8 @@ sdhi0: mmc@ee100000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee100000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A774E1_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774E1_PD_ALWAYS_ON>; resets = <&cpg 314>; @@ -2375,7 +2376,8 @@ sdhi1: mmc@ee120000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee120000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 313>; + clocks = <&cpg CPG_MOD 313>, <&cpg CPG_CORE R8A774E1_CLK_SD1H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774E1_PD_ALWAYS_ON>; resets = <&cpg 313>; @@ -2388,7 +2390,8 @@ sdhi2: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 312>; + clocks = <&cpg CPG_MOD 312>, <&cpg CPG_CORE R8A774E1_CLK_SD2H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774E1_PD_ALWAYS_ON>; resets = <&cpg 312>; @@ -2401,7 +2404,8 @@ sdhi3: mmc@ee160000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee160000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 311>; + clocks = <&cpg CPG_MOD 311>, <&cpg CPG_CORE R8A774E1_CLK_SD3H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A774E1_PD_ALWAYS_ON>; resets = <&cpg 311>; diff --git a/arch/arm64/boot/dts/renesas/r8a77951.dtsi b/arch/arm64/boot/dts/renesas/r8a77951.dtsi index 193d81be40..9265a57027 100644 --- a/arch/arm64/boot/dts/renesas/r8a77951.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77951.dtsi @@ -62,7 +62,7 @@ can_clk: can { clock-frequency = <0>; }; - cluster0_opp: opp_table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -96,7 +96,7 @@ opp-1700000000 { }; }; - cluster1_opp: opp_table1 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; @@ -2668,7 +2668,8 @@ sdhi0: mmc@ee100000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee100000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A7795_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A7795_PD_ALWAYS_ON>; resets = <&cpg 314>; @@ -2681,7 +2682,8 @@ sdhi1: mmc@ee120000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee120000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 313>; + clocks = <&cpg CPG_MOD 313>, <&cpg CPG_CORE R8A7795_CLK_SD1H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A7795_PD_ALWAYS_ON>; resets = <&cpg 313>; @@ -2694,7 +2696,8 @@ sdhi2: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 312>; + clocks = <&cpg CPG_MOD 312>, <&cpg CPG_CORE R8A7795_CLK_SD2H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A7795_PD_ALWAYS_ON>; resets = <&cpg 312>; @@ -2707,7 +2710,8 @@ sdhi3: mmc@ee160000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee160000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 311>; + clocks = <&cpg CPG_MOD 311>, <&cpg CPG_CORE R8A7795_CLK_SD3H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A7795_PD_ALWAYS_ON>; resets = <&cpg 311>; diff --git a/arch/arm64/boot/dts/renesas/r8a77960.dtsi b/arch/arm64/boot/dts/renesas/r8a77960.dtsi index b526e4f0ee..26f7103d11 100644 --- a/arch/arm64/boot/dts/renesas/r8a77960.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77960.dtsi @@ -57,7 +57,7 @@ can_clk: can { clock-frequency = <0>; }; - cluster0_opp: opp_table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -97,7 +97,7 @@ opp-1800000000 { }; }; - cluster1_opp: opp_table1 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; @@ -2468,7 +2468,8 @@ sdhi0: mmc@ee100000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee100000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A7796_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A7796_PD_ALWAYS_ON>; resets = <&cpg 314>; @@ -2481,7 +2482,8 @@ sdhi1: mmc@ee120000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee120000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 313>; + clocks = <&cpg CPG_MOD 313>, <&cpg CPG_CORE R8A7796_CLK_SD1H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A7796_PD_ALWAYS_ON>; resets = <&cpg 313>; @@ -2494,7 +2496,8 @@ sdhi2: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 312>; + clocks = <&cpg CPG_MOD 312>, <&cpg CPG_CORE R8A7796_CLK_SD2H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A7796_PD_ALWAYS_ON>; resets = <&cpg 312>; @@ -2507,7 +2510,8 @@ sdhi3: mmc@ee160000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee160000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 311>; + clocks = <&cpg CPG_MOD 311>, <&cpg CPG_CORE R8A7796_CLK_SD3H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A7796_PD_ALWAYS_ON>; resets = <&cpg 311>; diff --git a/arch/arm64/boot/dts/renesas/r8a77961.dtsi b/arch/arm64/boot/dts/renesas/r8a77961.dtsi index 21fc95397c..ac9b587f6f 100644 --- a/arch/arm64/boot/dts/renesas/r8a77961.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77961.dtsi @@ -46,7 +46,7 @@ can_clk: can { clock-frequency = <0>; }; - cluster0_opp: opp_table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -86,7 +86,7 @@ opp-1800000000 { }; }; - cluster1_opp: opp_table1 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; @@ -1392,6 +1392,17 @@ scif5: serial@e6f30000 { status = "disabled"; }; + tpu: pwm@e6e80000 { + compatible = "renesas,tpu-r8a77961", "renesas,tpu"; + reg = <0 0xe6e80000 0 0x148>; + interrupts = ; + clocks = <&cpg CPG_MOD 304>; + power-domains = <&sysc R8A77961_PD_ALWAYS_ON>; + resets = <&cpg 304>; + #pwm-cells = <3>; + status = "disabled"; + }; + msiof0: spi@e6e90000 { compatible = "renesas,msiof-r8a77961", "renesas,rcar-gen3-msiof"; @@ -2301,7 +2312,8 @@ sdhi0: mmc@ee100000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee100000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A77961_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77961_PD_ALWAYS_ON>; resets = <&cpg 314>; @@ -2314,7 +2326,8 @@ sdhi1: mmc@ee120000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee120000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 313>; + clocks = <&cpg CPG_MOD 313>, <&cpg CPG_CORE R8A77961_CLK_SD1H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77961_PD_ALWAYS_ON>; resets = <&cpg 313>; @@ -2327,7 +2340,8 @@ sdhi2: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 312>; + clocks = <&cpg CPG_MOD 312>, <&cpg CPG_CORE R8A77961_CLK_SD2H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77961_PD_ALWAYS_ON>; resets = <&cpg 312>; @@ -2340,7 +2354,8 @@ sdhi3: mmc@ee160000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee160000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 311>; + clocks = <&cpg CPG_MOD 311>, <&cpg CPG_CORE R8A77961_CLK_SD3H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77961_PD_ALWAYS_ON>; resets = <&cpg 311>; diff --git a/arch/arm64/boot/dts/renesas/r8a77965.dtsi b/arch/arm64/boot/dts/renesas/r8a77965.dtsi index f9679a4dd8..f898aad72b 100644 --- a/arch/arm64/boot/dts/renesas/r8a77965.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77965.dtsi @@ -62,7 +62,7 @@ can_clk: can { clock-frequency = <0>; }; - cluster0_opp: opp_table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -2315,7 +2315,8 @@ sdhi0: mmc@ee100000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee100000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A77965_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77965_PD_ALWAYS_ON>; resets = <&cpg 314>; @@ -2328,7 +2329,8 @@ sdhi1: mmc@ee120000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee120000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 313>; + clocks = <&cpg CPG_MOD 313>, <&cpg CPG_CORE R8A77965_CLK_SD1H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77965_PD_ALWAYS_ON>; resets = <&cpg 313>; @@ -2341,7 +2343,8 @@ sdhi2: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 312>; + clocks = <&cpg CPG_MOD 312>, <&cpg CPG_CORE R8A77965_CLK_SD2H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77965_PD_ALWAYS_ON>; resets = <&cpg 312>; @@ -2354,7 +2357,8 @@ sdhi3: mmc@ee160000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee160000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 311>; + clocks = <&cpg CPG_MOD 311>, <&cpg CPG_CORE R8A77965_CLK_SD3H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77965_PD_ALWAYS_ON>; resets = <&cpg 311>; diff --git a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts index d24da54f31..b579d3189a 100644 --- a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts +++ b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts @@ -8,6 +8,7 @@ /dts-v1/; #include "r8a77970.dtsi" +#include / { model = "Renesas Eagle board based on r8a77970"; @@ -92,10 +93,13 @@ &avb { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id0022.1622", + "ethernet-phy-ieee802.3-c22"; rxc-skew-ps = <1500>; reg = <0>; interrupt-parent = <&gpio1>; interrupts = <17 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&gpio1 16 GPIO_ACTIVE_LOW>; }; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts index 2426e53312..39f3e6cbba 100644 --- a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts +++ b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts @@ -8,6 +8,7 @@ /dts-v1/; #include "r8a77970.dtsi" +#include / { model = "Renesas V3M Starter Kit board"; @@ -107,10 +108,13 @@ &avb { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id0022.1622", + "ethernet-phy-ieee802.3-c22"; rxc-skew-ps = <1500>; reg = <0>; interrupt-parent = <&gpio1>; interrupts = <17 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&gpio1 16 GPIO_ACTIVE_LOW>; }; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77980-condor.dts b/arch/arm64/boot/dts/renesas/r8a77980-condor.dts index edf7f2a2f9..3d6d10c82f 100644 --- a/arch/arm64/boot/dts/renesas/r8a77980-condor.dts +++ b/arch/arm64/boot/dts/renesas/r8a77980-condor.dts @@ -8,6 +8,7 @@ /dts-v1/; #include "r8a77980.dtsi" +#include / { model = "Renesas Condor board based on r8a77980"; @@ -132,10 +133,13 @@ &gether { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id0022.1622", + "ethernet-phy-ieee802.3-c22"; rxc-skew-ps = <1500>; reg = <0>; interrupt-parent = <&gpio4>; interrupts = <23 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&gpio4 22 GPIO_ACTIVE_LOW>; }; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts index 7838dcee31..1d09d88676 100644 --- a/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts +++ b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts @@ -8,6 +8,7 @@ /dts-v1/; #include "r8a77980.dtsi" +#include / { model = "Renesas V3H Starter Kit board"; @@ -113,9 +114,12 @@ &gether { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id0022.1622", + "ethernet-phy-ieee802.3-c22"; reg = <0>; interrupt-parent = <&gpio4>; interrupts = <23 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&gpio4 22 GPIO_ACTIVE_LOW>; }; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77980.dtsi b/arch/arm64/boot/dts/renesas/r8a77980.dtsi index 21fe602bd2..347c068ff2 100644 --- a/arch/arm64/boot/dts/renesas/r8a77980.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77980.dtsi @@ -1339,7 +1339,8 @@ mmc0: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A77980_CLK_SD0H>; + clock-names = "core", "clkh"; power-domains = <&sysc R8A77980_PD_ALWAYS_ON>; resets = <&cpg 314>; max-frequency = <200000000>; diff --git a/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts b/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts index 9c7146084e..9da0fd08f8 100644 --- a/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts +++ b/arch/arm64/boot/dts/renesas/r8a77990-ebisu.dts @@ -7,795 +7,9 @@ /dts-v1/; #include "r8a77990.dtsi" -#include -#include +#include "ebisu.dtsi" / { model = "Renesas Ebisu board based on r8a77990"; compatible = "renesas,ebisu", "renesas,r8a77990"; - - aliases { - serial0 = &scif2; - ethernet0 = &avb; - mmc0 = &sdhi3; - mmc1 = &sdhi0; - mmc2 = &sdhi1; - }; - - chosen { - bootargs = "ignore_loglevel rw root=/dev/nfs ip=on"; - stdout-path = "serial0:115200n8"; - }; - - audio_clkout: audio-clkout { - /* - * This is same as <&rcar_sound 0> - * but needed to avoid cs2000/rcar_sound probe dead-lock - */ - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <11289600>; - }; - - backlight: backlight { - compatible = "pwm-backlight"; - pwms = <&pwm3 0 50000>; - - brightness-levels = <512 511 505 494 473 440 392 327 241 133 0>; - default-brightness-level = <10>; - - power-supply = <®_12p0v>; - }; - - cvbs-in { - compatible = "composite-video-connector"; - label = "CVBS IN"; - - port { - cvbs_con: endpoint { - remote-endpoint = <&adv7482_ain7>; - }; - }; - }; - - hdmi-in { - compatible = "hdmi-connector"; - label = "HDMI IN"; - type = "a"; - - port { - hdmi_in_con: endpoint { - remote-endpoint = <&adv7482_hdmi>; - }; - }; - }; - - hdmi-out { - compatible = "hdmi-connector"; - type = "a"; - - port { - hdmi_con_out: endpoint { - remote-endpoint = <&adv7511_out>; - }; - }; - }; - - keys { - compatible = "gpio-keys"; - - pinctrl-0 = <&keys_pins>; - pinctrl-names = "default"; - - key-1 { - gpios = <&gpio5 10 GPIO_ACTIVE_LOW>; - linux,code = ; - label = "SW4-1"; - wakeup-source; - debounce-interval = <20>; - }; - key-2 { - gpios = <&gpio5 11 GPIO_ACTIVE_LOW>; - linux,code = ; - label = "SW4-2"; - wakeup-source; - debounce-interval = <20>; - }; - key-3 { - gpios = <&gpio5 12 GPIO_ACTIVE_LOW>; - linux,code = ; - label = "SW4-3"; - wakeup-source; - debounce-interval = <20>; - }; - key-4 { - gpios = <&gpio5 13 GPIO_ACTIVE_LOW>; - linux,code = ; - label = "SW4-4"; - wakeup-source; - debounce-interval = <20>; - }; - }; - - lvds-decoder { - compatible = "thine,thc63lvd1024"; - vcc-supply = <®_3p3v>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - thc63lvd1024_in: endpoint { - remote-endpoint = <&lvds0_out>; - }; - }; - - port@2 { - reg = <2>; - thc63lvd1024_out: endpoint { - remote-endpoint = <&adv7511_in>; - }; - }; - }; - }; - - memory@48000000 { - device_type = "memory"; - /* first 128MB is reserved for secure area. */ - reg = <0x0 0x48000000 0x0 0x38000000>; - }; - - reg_1p8v: regulator0 { - compatible = "regulator-fixed"; - regulator-name = "fixed-1.8V"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-boot-on; - regulator-always-on; - }; - - reg_3p3v: regulator1 { - compatible = "regulator-fixed"; - regulator-name = "fixed-3.3V"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - regulator-boot-on; - regulator-always-on; - }; - - reg_12p0v: regulator2 { - compatible = "regulator-fixed"; - regulator-name = "D12.0V"; - regulator-min-microvolt = <12000000>; - regulator-max-microvolt = <12000000>; - regulator-boot-on; - regulator-always-on; - }; - - rsnd_ak4613: sound { - compatible = "simple-audio-card"; - - simple-audio-card,name = "rsnd-ak4613"; - simple-audio-card,format = "left_j"; - simple-audio-card,bitclock-master = <&sndcpu>; - simple-audio-card,frame-master = <&sndcpu>; - - sndcodec: simple-audio-card,codec { - sound-dai = <&ak4613>; - }; - - sndcpu: simple-audio-card,cpu { - sound-dai = <&rcar_sound>; - }; - }; - - vbus0_usb2: regulator-vbus0-usb2 { - compatible = "regulator-fixed"; - - regulator-name = "USB20_VBUS_CN"; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - - gpio = <&gpio6 4 GPIO_ACTIVE_HIGH>; - enable-active-high; - }; - - vcc_sdhi0: regulator-vcc-sdhi0 { - compatible = "regulator-fixed"; - - regulator-name = "SDHI0 Vcc"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - - gpio = <&gpio5 17 GPIO_ACTIVE_HIGH>; - enable-active-high; - }; - - vccq_sdhi0: regulator-vccq-sdhi0 { - compatible = "regulator-gpio"; - - regulator-name = "SDHI0 VccQ"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <3300000>; - - gpios = <&gpio5 18 GPIO_ACTIVE_HIGH>; - gpios-states = <1>; - states = <3300000 1>, <1800000 0>; - }; - - vcc_sdhi1: regulator-vcc-sdhi1 { - compatible = "regulator-fixed"; - - regulator-name = "SDHI1 Vcc"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - - gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; - enable-active-high; - }; - - vccq_sdhi1: regulator-vccq-sdhi1 { - compatible = "regulator-gpio"; - - regulator-name = "SDHI1 VccQ"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <3300000>; - - gpios = <&gpio3 15 GPIO_ACTIVE_HIGH>; - gpios-states = <1>; - states = <3300000 1>, <1800000 0>; - }; - - vga { - compatible = "vga-connector"; - - port { - vga_in: endpoint { - remote-endpoint = <&adv7123_out>; - }; - }; - }; - - vga-encoder { - compatible = "adi,adv7123"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - adv7123_in: endpoint { - remote-endpoint = <&du_out_rgb>; - }; - }; - port@1 { - reg = <1>; - adv7123_out: endpoint { - remote-endpoint = <&vga_in>; - }; - }; - }; - }; - - x12_clk: x12 { - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <24576000>; - }; - - x13_clk: x13 { - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <74250000>; - }; -}; - -&audio_clk_a { - clock-frequency = <22579200>; -}; - -&avb { - pinctrl-0 = <&avb_pins>; - pinctrl-names = "default"; - phy-handle = <&phy0>; - status = "okay"; - - phy0: ethernet-phy@0 { - rxc-skew-ps = <1500>; - reg = <0>; - interrupt-parent = <&gpio2>; - interrupts = <21 IRQ_TYPE_LEVEL_LOW>; - reset-gpios = <&gpio1 20 GPIO_ACTIVE_LOW>; - /* - * TX clock internal delay mode is required for reliable - * 1Gbps communication using the KSZ9031RNX phy present on - * the Ebisu board, however, TX clock internal delay mode - * isn't supported on r8a77990. Thus, limit speed to - * 100Mbps for reliable communication. - */ - max-speed = <100>; - }; -}; - -&canfd { - pinctrl-0 = <&canfd0_pins>; - pinctrl-names = "default"; - status = "okay"; - - channel0 { - status = "okay"; - }; -}; - -&csi40 { - status = "okay"; - - ports { - port@0 { - csi40_in: endpoint { - clock-lanes = <0>; - data-lanes = <1 2>; - remote-endpoint = <&adv7482_txa>; - }; - }; - }; -}; - -&du { - pinctrl-0 = <&du_pins>; - pinctrl-names = "default"; - status = "okay"; - - clocks = <&cpg CPG_MOD 724>, - <&cpg CPG_MOD 723>, - <&x13_clk>; - clock-names = "du.0", "du.1", "dclkin.0"; - - ports { - port@0 { - endpoint { - remote-endpoint = <&adv7123_in>; - }; - }; - }; -}; - -&ehci0 { - dr_mode = "otg"; - status = "okay"; -}; - -&extal_clk { - clock-frequency = <48000000>; -}; - -&hsusb { - dr_mode = "otg"; - status = "okay"; -}; - -&i2c0 { - status = "okay"; - - io_expander: gpio@20 { - compatible = "onnn,pca9654"; - reg = <0x20>; - gpio-controller; - #gpio-cells = <2>; - interrupt-parent = <&gpio2>; - interrupts = <22 IRQ_TYPE_LEVEL_LOW>; - }; - - hdmi-encoder@39 { - compatible = "adi,adv7511w"; - reg = <0x39>; - interrupt-parent = <&gpio1>; - interrupts = <1 IRQ_TYPE_LEVEL_LOW>; - - adi,input-depth = <8>; - adi,input-colorspace = "rgb"; - adi,input-clock = "1x"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - adv7511_in: endpoint { - remote-endpoint = <&thc63lvd1024_out>; - }; - }; - - port@1 { - reg = <1>; - adv7511_out: endpoint { - remote-endpoint = <&hdmi_con_out>; - }; - }; - }; - }; - - video-receiver@70 { - compatible = "adi,adv7482"; - reg = <0x70>; - - #address-cells = <1>; - #size-cells = <0>; - - interrupt-parent = <&gpio0>; - interrupt-names = "intrq1", "intrq2"; - interrupts = <7 IRQ_TYPE_LEVEL_LOW>, - <17 IRQ_TYPE_LEVEL_LOW>; - - port@7 { - reg = <7>; - - adv7482_ain7: endpoint { - remote-endpoint = <&cvbs_con>; - }; - }; - - port@8 { - reg = <8>; - - adv7482_hdmi: endpoint { - remote-endpoint = <&hdmi_in_con>; - }; - }; - - port@a { - reg = <10>; - - adv7482_txa: endpoint { - clock-lanes = <0>; - data-lanes = <1 2>; - remote-endpoint = <&csi40_in>; - }; - }; - }; -}; - -&i2c3 { - status = "okay"; - - ak4613: codec@10 { - compatible = "asahi-kasei,ak4613"; - #sound-dai-cells = <0>; - reg = <0x10>; - clocks = <&rcar_sound 3>; - - asahi-kasei,in1-single-end; - asahi-kasei,in2-single-end; - asahi-kasei,out1-single-end; - asahi-kasei,out2-single-end; - asahi-kasei,out3-single-end; - asahi-kasei,out4-single-end; - asahi-kasei,out5-single-end; - asahi-kasei,out6-single-end; - }; - - cs2000: clk-multiplier@4f { - #clock-cells = <0>; - compatible = "cirrus,cs2000-cp"; - reg = <0x4f>; - clocks = <&audio_clkout>, <&x12_clk>; - clock-names = "clk_in", "ref_clk"; - - assigned-clocks = <&cs2000>; - assigned-clock-rates = <24576000>; /* 1/1 divide */ - }; -}; - -&i2c_dvfs { - status = "okay"; - - clock-frequency = <400000>; - - pmic: pmic@30 { - pinctrl-0 = <&irq0_pins>; - pinctrl-names = "default"; - - compatible = "rohm,bd9571mwv"; - reg = <0x30>; - interrupt-parent = <&intc_ex>; - interrupts = <0 IRQ_TYPE_LEVEL_LOW>; - interrupt-controller; - #interrupt-cells = <2>; - gpio-controller; - #gpio-cells = <2>; - rohm,ddr-backup-power = <0x1>; - rohm,rstbmode-level; - }; - - eeprom@50 { - compatible = "rohm,br24t01", "atmel,24c01"; - reg = <0x50>; - pagesize = <8>; - }; -}; - -&lvds0 { - status = "okay"; - - clocks = <&cpg CPG_MOD 727>, - <&x13_clk>, - <&extal_clk>; - clock-names = "fck", "dclkin.0", "extal"; - - ports { - port@1 { - lvds0_out: endpoint { - remote-endpoint = <&thc63lvd1024_in>; - }; - }; - }; -}; - -&lvds1 { - /* - * Even though the LVDS1 output is not connected, the encoder must be - * enabled to supply a pixel clock to the DU for the DPAD output when - * LVDS0 is in use. - */ - status = "okay"; - - clocks = <&cpg CPG_MOD 727>, - <&x13_clk>, - <&extal_clk>; - clock-names = "fck", "dclkin.0", "extal"; -}; - -&ohci0 { - dr_mode = "otg"; - status = "okay"; -}; - -&pcie_bus_clk { - clock-frequency = <100000000>; -}; - -&pciec0 { - status = "okay"; -}; - -&pfc { - avb_pins: avb { - groups = "avb_link", "avb_mii"; - function = "avb"; - }; - - canfd0_pins: canfd0 { - groups = "canfd0_data"; - function = "canfd0"; - }; - - du_pins: du { - groups = "du_rgb888", "du_sync", "du_disp", "du_clk_out_0"; - function = "du"; - }; - - irq0_pins: irq0 { - groups = "intc_ex_irq0"; - function = "intc_ex"; - }; - - keys_pins: keys { - pins = "GP_5_10", "GP_5_11", "GP_5_12", "GP_5_13"; - bias-pull-up; - }; - - pwm3_pins: pwm3 { - groups = "pwm3_b"; - function = "pwm3"; - }; - - pwm5_pins: pwm5 { - groups = "pwm5_a"; - function = "pwm5"; - }; - - scif2_pins: scif2 { - groups = "scif2_data_a"; - function = "scif2"; - }; - - sdhi0_pins: sd0 { - groups = "sdhi0_data4", "sdhi0_ctrl"; - function = "sdhi0"; - power-source = <3300>; - }; - - sdhi0_pins_uhs: sd0_uhs { - groups = "sdhi0_data4", "sdhi0_ctrl"; - function = "sdhi0"; - power-source = <1800>; - }; - - sdhi1_pins: sd1 { - groups = "sdhi1_data4", "sdhi1_ctrl"; - function = "sdhi1"; - power-source = <3300>; - }; - - sdhi1_pins_uhs: sd1_uhs { - groups = "sdhi1_data4", "sdhi1_ctrl"; - function = "sdhi1"; - power-source = <1800>; - }; - - sdhi3_pins: sd3 { - groups = "sdhi3_data8", "sdhi3_ctrl", "sdhi3_ds"; - function = "sdhi3"; - power-source = <1800>; - }; - - sound_clk_pins: sound_clk { - groups = "audio_clk_a", "audio_clk_b_a", "audio_clk_c_a", - "audio_clkout_a", "audio_clkout1_a"; - function = "audio_clk"; - }; - - sound_pins: sound { - groups = "ssi01239_ctrl", "ssi0_data", "ssi1_data"; - function = "ssi"; - }; - - usb0_pins: usb { - groups = "usb0_b", "usb0_id"; - function = "usb0"; - }; - - usb30_pins: usb30 { - groups = "usb30"; - function = "usb30"; - }; -}; - -&pwm3 { - pinctrl-0 = <&pwm3_pins>; - pinctrl-names = "default"; - - status = "okay"; -}; - -&pwm5 { - pinctrl-0 = <&pwm5_pins>; - pinctrl-names = "default"; - - status = "okay"; -}; - -&rcar_sound { - pinctrl-0 = <&sound_pins>, <&sound_clk_pins>; - pinctrl-names = "default"; - - /* Single DAI */ - #sound-dai-cells = <0>; - - /* audio_clkout0/1/2/3 */ - #clock-cells = <1>; - clock-frequency = <12288000 11289600>; - - status = "okay"; - - /* update to */ - clocks = <&cpg CPG_MOD 1005>, - <&cpg CPG_MOD 1006>, <&cpg CPG_MOD 1007>, - <&cpg CPG_MOD 1008>, <&cpg CPG_MOD 1009>, - <&cpg CPG_MOD 1010>, <&cpg CPG_MOD 1011>, - <&cpg CPG_MOD 1012>, <&cpg CPG_MOD 1013>, - <&cpg CPG_MOD 1014>, <&cpg CPG_MOD 1015>, - <&cpg CPG_MOD 1022>, <&cpg CPG_MOD 1023>, - <&cpg CPG_MOD 1024>, <&cpg CPG_MOD 1025>, - <&cpg CPG_MOD 1026>, <&cpg CPG_MOD 1027>, - <&cpg CPG_MOD 1028>, <&cpg CPG_MOD 1029>, - <&cpg CPG_MOD 1030>, <&cpg CPG_MOD 1031>, - <&cpg CPG_MOD 1020>, <&cpg CPG_MOD 1021>, - <&cpg CPG_MOD 1020>, <&cpg CPG_MOD 1021>, - <&cpg CPG_MOD 1019>, <&cpg CPG_MOD 1018>, - <&audio_clk_a>, <&cs2000>, <&audio_clk_c>, - <&cpg CPG_CORE R8A77990_CLK_ZA2>; - - rcar_sound,dai { - dai0 { - playback = <&ssi0>, <&src0>, <&dvc0>; - capture = <&ssi1>, <&src1>, <&dvc1>; - }; - }; - -}; - -&rwdt { - timeout-sec = <60>; - status = "okay"; -}; - -&scif2 { - pinctrl-0 = <&scif2_pins>; - pinctrl-names = "default"; - - status = "okay"; -}; - -&sdhi0 { - pinctrl-0 = <&sdhi0_pins>; - pinctrl-1 = <&sdhi0_pins_uhs>; - pinctrl-names = "default", "state_uhs"; - - vmmc-supply = <&vcc_sdhi0>; - vqmmc-supply = <&vccq_sdhi0>; - cd-gpios = <&gpio3 12 GPIO_ACTIVE_LOW>; - wp-gpios = <&gpio3 13 GPIO_ACTIVE_HIGH>; - bus-width = <4>; - sd-uhs-sdr50; - sd-uhs-sdr104; - status = "okay"; -}; - -&sdhi1 { - pinctrl-0 = <&sdhi1_pins>; - pinctrl-1 = <&sdhi1_pins_uhs>; - pinctrl-names = "default", "state_uhs"; - - vmmc-supply = <&vcc_sdhi1>; - vqmmc-supply = <&vccq_sdhi1>; - cd-gpios = <&gpio3 14 GPIO_ACTIVE_LOW>; - bus-width = <4>; - sd-uhs-sdr50; - sd-uhs-sdr104; - status = "okay"; -}; - -&sdhi3 { - /* used for on-board 8bit eMMC */ - pinctrl-0 = <&sdhi3_pins>; - pinctrl-1 = <&sdhi3_pins>; - pinctrl-names = "default", "state_uhs"; - - vmmc-supply = <®_3p3v>; - vqmmc-supply = <®_1p8v>; - mmc-hs200-1_8v; - mmc-hs400-1_8v; - bus-width = <8>; - no-sd; - no-sdio; - non-removable; - full-pwr-cycle-in-suspend; - status = "okay"; -}; - -&ssi1 { - shared-pin; -}; - -&usb2_phy0 { - pinctrl-0 = <&usb0_pins>; - pinctrl-names = "default"; - - vbus-supply = <&vbus0_usb2>; - status = "okay"; -}; - -&usb3_peri0 { - companion = <&xhci0>; - status = "okay"; -}; - -&vin4 { - status = "okay"; -}; - -&vin5 { - status = "okay"; -}; - -&xhci0 { - pinctrl-0 = <&usb30_pins>; - pinctrl-names = "default"; - - status = "okay"; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77990.dtsi b/arch/arm64/boot/dts/renesas/r8a77990.dtsi index 0ea300a814..14caedd0c9 100644 --- a/arch/arm64/boot/dts/renesas/r8a77990.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77990.dtsi @@ -55,7 +55,7 @@ can_clk: can { clock-frequency = <0>; }; - cluster1_opp: opp_table10 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; opp-800000000 { @@ -1788,7 +1788,8 @@ sdhi0: mmc@ee100000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee100000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 314>; + clocks = <&cpg CPG_MOD 314>, <&cpg CPG_CORE R8A77990_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77990_PD_ALWAYS_ON>; resets = <&cpg 314>; @@ -1801,7 +1802,8 @@ sdhi1: mmc@ee120000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee120000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 313>; + clocks = <&cpg CPG_MOD 313>, <&cpg CPG_CORE R8A77990_CLK_SD1H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77990_PD_ALWAYS_ON>; resets = <&cpg 313>; @@ -1814,7 +1816,8 @@ sdhi3: mmc@ee160000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee160000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 311>; + clocks = <&cpg CPG_MOD 311>, <&cpg CPG_CORE R8A77990_CLK_SD3H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77990_PD_ALWAYS_ON>; resets = <&cpg 311>; diff --git a/arch/arm64/boot/dts/renesas/r8a77995-draak.dts b/arch/arm64/boot/dts/renesas/r8a77995-draak.dts index 1ac15aa05b..384825617f 100644 --- a/arch/arm64/boot/dts/renesas/r8a77995-draak.dts +++ b/arch/arm64/boot/dts/renesas/r8a77995-draak.dts @@ -8,678 +8,9 @@ /dts-v1/; #include "r8a77995.dtsi" -#include -#include +#include "draak.dtsi" / { model = "Renesas Draak board based on r8a77995"; compatible = "renesas,draak", "renesas,r8a77995"; - - aliases { - serial0 = &scif2; - ethernet0 = &avb; - }; - - audio_clkout: audio-clkout { - /* - * This is same as <&rcar_sound 0> - * but needed to avoid cs2000/rcar_sound probe dead-lock - */ - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <12288000>; - }; - - backlight: backlight { - compatible = "pwm-backlight"; - pwms = <&pwm1 0 50000>; - - brightness-levels = <512 511 505 494 473 440 392 327 241 133 0>; - default-brightness-level = <10>; - - power-supply = <®_12p0v>; - enable-gpios = <&gpio4 0 GPIO_ACTIVE_HIGH>; - }; - - chosen { - bootargs = "ignore_loglevel rw root=/dev/nfs ip=on"; - stdout-path = "serial0:115200n8"; - }; - - composite-in { - compatible = "composite-video-connector"; - - port { - composite_con_in: endpoint { - remote-endpoint = <&adv7180_in>; - }; - }; - }; - - hdmi-in { - compatible = "hdmi-connector"; - type = "a"; - - port { - hdmi_con_in: endpoint { - remote-endpoint = <&adv7612_in>; - }; - }; - }; - - hdmi-out { - compatible = "hdmi-connector"; - type = "a"; - - port { - hdmi_con_out: endpoint { - remote-endpoint = <&adv7511_out>; - }; - }; - }; - - keys { - compatible = "gpio-keys"; - - pinctrl-0 = <&keys_pins>; - pinctrl-names = "default"; - - key-1 { - gpios = <&gpio4 12 GPIO_ACTIVE_LOW>; - linux,code = ; - label = "SW56-1"; - wakeup-source; - debounce-interval = <20>; - }; - key-2 { - gpios = <&gpio4 13 GPIO_ACTIVE_LOW>; - linux,code = ; - label = "SW56-2"; - wakeup-source; - debounce-interval = <20>; - }; - key-3 { - gpios = <&gpio4 14 GPIO_ACTIVE_LOW>; - linux,code = ; - label = "SW56-3"; - wakeup-source; - debounce-interval = <20>; - }; - key-4 { - gpios = <&gpio4 15 GPIO_ACTIVE_LOW>; - linux,code = ; - label = "SW56-4"; - wakeup-source; - debounce-interval = <20>; - }; - }; - - lvds-decoder { - compatible = "thine,thc63lvd1024"; - vcc-supply = <®_3p3v>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - thc63lvd1024_in: endpoint { - remote-endpoint = <&lvds0_out>; - }; - }; - - port@2 { - reg = <2>; - thc63lvd1024_out: endpoint { - remote-endpoint = <&adv7511_in>; - }; - }; - }; - }; - - memory@48000000 { - device_type = "memory"; - /* first 128MB is reserved for secure area. */ - reg = <0x0 0x48000000 0x0 0x18000000>; - }; - - reg_1p8v: regulator-1p8v { - compatible = "regulator-fixed"; - regulator-name = "fixed-1.8V"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-boot-on; - regulator-always-on; - }; - - reg_3p3v: regulator-3p3v { - compatible = "regulator-fixed"; - regulator-name = "fixed-3.3V"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - regulator-boot-on; - regulator-always-on; - }; - - reg_12p0v: regulator-12p0v { - compatible = "regulator-fixed"; - regulator-name = "D12.0V"; - regulator-min-microvolt = <12000000>; - regulator-max-microvolt = <12000000>; - regulator-boot-on; - regulator-always-on; - }; - - sound_card: sound { - compatible = "audio-graph-card"; - - dais = <&rsnd_port0 /* ak4613 */ - /* HDMI is not yet supported */ - >; - }; - - vga { - compatible = "vga-connector"; - - port { - vga_in: endpoint { - remote-endpoint = <&adv7123_out>; - }; - }; - }; - - vga-encoder { - compatible = "adi,adv7123"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - adv7123_in: endpoint { - remote-endpoint = <&du_out_rgb>; - }; - }; - port@1 { - reg = <1>; - adv7123_out: endpoint { - remote-endpoint = <&vga_in>; - }; - }; - }; - }; - - x12_clk: x12 { - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <74250000>; - }; - - x19_clk: x19 { - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <24576000>; - }; -}; - -&audio_clk_b { - /* - * X11 is connected to VI4_FIELD/SCIF_CLK/AUDIO_CLKB, - * and R-Car Sound uses AUDIO_CLKB. - * Note is that schematic indicates VI4_FIELD conection only - * not AUDIO_CLKB at SoC page. - * And this VI4_FIELD/SCIF_CLK/AUDIO_CLKB is connected to SW60. - * SW60 should be 1-2. - */ - - clock-frequency = <22579200>; -}; - -&avb { - pinctrl-0 = <&avb0_pins>; - pinctrl-names = "default"; - renesas,no-ether-link; - phy-handle = <&phy0>; - status = "okay"; - - phy0: ethernet-phy@0 { - rxc-skew-ps = <1500>; - reg = <0>; - interrupt-parent = <&gpio5>; - interrupts = <19 IRQ_TYPE_LEVEL_LOW>; - /* - * TX clock internal delay mode is required for reliable - * 1Gbps communication using the KSZ9031RNX phy present on - * the Draak board, however, TX clock internal delay mode - * isn't supported on r8a77995. Thus, limit speed to - * 100Mbps for reliable communication. - */ - max-speed = <100>; - }; -}; - -&can0 { - pinctrl-0 = <&can0_pins>; - pinctrl-names = "default"; - status = "okay"; -}; - -&can1 { - pinctrl-0 = <&can1_pins>; - pinctrl-names = "default"; - status = "okay"; -}; - -&du { - pinctrl-0 = <&du_pins>; - pinctrl-names = "default"; - status = "okay"; - - clocks = <&cpg CPG_MOD 724>, - <&cpg CPG_MOD 723>, - <&x12_clk>; - clock-names = "du.0", "du.1", "dclkin.0"; - - ports { - port@0 { - endpoint { - remote-endpoint = <&adv7123_in>; - }; - }; - }; -}; - -&ehci0 { - dr_mode = "host"; - status = "okay"; -}; - -&extal_clk { - clock-frequency = <48000000>; -}; - -&hsusb { - dr_mode = "host"; - status = "okay"; -}; - -&i2c0 { - pinctrl-0 = <&i2c0_pins>; - pinctrl-names = "default"; - status = "okay"; - - ak4613: codec@10 { - compatible = "asahi-kasei,ak4613"; - #sound-dai-cells = <0>; - reg = <0x10>; - clocks = <&rcar_sound 0>; /* audio_clkout */ - - asahi-kasei,in1-single-end; - asahi-kasei,in2-single-end; - asahi-kasei,out1-single-end; - asahi-kasei,out2-single-end; - asahi-kasei,out3-single-end; - asahi-kasei,out4-single-end; - asahi-kasei,out5-single-end; - asahi-kasei,out6-single-end; - - port { - ak4613_endpoint: endpoint { - remote-endpoint = <&rsnd_for_ak4613>; - }; - }; - }; - - composite-in@20 { - compatible = "adi,adv7180cp"; - reg = <0x20>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - adv7180_in: endpoint { - remote-endpoint = <&composite_con_in>; - }; - }; - - port@3 { - reg = <3>; - - /* - * The VIN4 video input path is shared between - * CVBS and HDMI inputs through SW[49-53] - * switches. - * - * CVBS is the default selection, link it to - * VIN4 here. - */ - adv7180_out: endpoint { - remote-endpoint = <&vin4_in>; - }; - }; - }; - - }; - - hdmi-encoder@39 { - compatible = "adi,adv7511w"; - reg = <0x39>, <0x3f>, <0x3c>, <0x38>; - reg-names = "main", "edid", "cec", "packet"; - interrupt-parent = <&gpio1>; - interrupts = <28 IRQ_TYPE_LEVEL_LOW>; - - adi,input-depth = <8>; - adi,input-colorspace = "rgb"; - adi,input-clock = "1x"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - adv7511_in: endpoint { - remote-endpoint = <&thc63lvd1024_out>; - }; - }; - - port@1 { - reg = <1>; - adv7511_out: endpoint { - remote-endpoint = <&hdmi_con_out>; - }; - }; - }; - }; - - hdmi-decoder@4c { - compatible = "adi,adv7612"; - reg = <0x4c>; - default-input = <0>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - - adv7612_in: endpoint { - remote-endpoint = <&hdmi_con_in>; - }; - }; - - port@2 { - reg = <2>; - - /* - * The VIN4 video input path is shared between - * CVBS and HDMI inputs through SW[49-53] - * switches. - * - * CVBS is the default selection, leave HDMI - * not connected here. - */ - adv7612_out: endpoint { - pclk-sample = <0>; - hsync-active = <0>; - vsync-active = <0>; - }; - }; - }; - }; - - cs2000: clk-multiplier@4f { - #clock-cells = <0>; - compatible = "cirrus,cs2000-cp"; - reg = <0x4f>; - clocks = <&audio_clkout>, <&x19_clk>; /* audio_clkout_1, x19 */ - clock-names = "clk_in", "ref_clk"; - - assigned-clocks = <&cs2000>; - assigned-clock-rates = <24576000>; /* 1/1 divide */ - }; - - eeprom@50 { - compatible = "rohm,br24t01", "atmel,24c01"; - reg = <0x50>; - pagesize = <8>; - }; -}; - -&i2c1 { - pinctrl-0 = <&i2c1_pins>; - pinctrl-names = "default"; - status = "okay"; -}; - -&lvds0 { - status = "okay"; - - clocks = <&cpg CPG_MOD 727>, - <&x12_clk>, - <&extal_clk>; - clock-names = "fck", "dclkin.0", "extal"; - - ports { - port@1 { - lvds0_out: endpoint { - remote-endpoint = <&thc63lvd1024_in>; - }; - }; - }; -}; - -&lvds1 { - /* - * Even though the LVDS1 output is not connected, the encoder must be - * enabled to supply a pixel clock to the DU for the DPAD output when - * LVDS0 is in use. - */ - status = "okay"; - - clocks = <&cpg CPG_MOD 727>, - <&x12_clk>, - <&extal_clk>; - clock-names = "fck", "dclkin.0", "extal"; -}; - -&ohci0 { - dr_mode = "host"; - status = "okay"; -}; - -&pfc { - avb0_pins: avb { - groups = "avb0_link", "avb0_mdio", "avb0_mii"; - function = "avb0"; - }; - - can0_pins: can0 { - groups = "can0_data_a"; - function = "can0"; - }; - - can1_pins: can1 { - groups = "can1_data_a"; - function = "can1"; - }; - - du_pins: du { - groups = "du_rgb888", "du_sync", "du_disp", "du_clk_out_0"; - function = "du"; - }; - - i2c0_pins: i2c0 { - groups = "i2c0"; - function = "i2c0"; - }; - - i2c1_pins: i2c1 { - groups = "i2c1"; - function = "i2c1"; - }; - - keys_pins: keys { - pins = "GP_4_12", "GP_4_13", "GP_4_14", "GP_4_15"; - bias-pull-up; - }; - - pwm0_pins: pwm0 { - groups = "pwm0_c"; - function = "pwm0"; - }; - - pwm1_pins: pwm1 { - groups = "pwm1_c"; - function = "pwm1"; - }; - - scif2_pins: scif2 { - groups = "scif2_data"; - function = "scif2"; - }; - - sdhi2_pins: sd2 { - groups = "mmc_data8", "mmc_ctrl"; - function = "mmc"; - power-source = <1800>; - }; - - sdhi2_pins_uhs: sd2_uhs { - groups = "mmc_data8", "mmc_ctrl"; - function = "mmc"; - power-source = <1800>; - }; - - sound_pins: sound { - groups = "ssi34_ctrl", "ssi3_data", "ssi4_data_a"; - function = "ssi"; - }; - - sound_clk_pins: sound-clk { - groups = "audio_clk_a", "audio_clk_b", - "audio_clkout", "audio_clkout1"; - function = "audio_clk"; - }; - - usb0_pins: usb0 { - groups = "usb0"; - function = "usb0"; - }; - - vin4_pins_cvbs: vin4 { - groups = "vin4_data8", "vin4_sync", "vin4_clk"; - function = "vin4"; - }; -}; - -&pwm0 { - pinctrl-0 = <&pwm0_pins>; - pinctrl-names = "default"; - - status = "okay"; -}; - -&pwm1 { - pinctrl-0 = <&pwm1_pins>; - pinctrl-names = "default"; - - status = "okay"; -}; - -&rcar_sound { - pinctrl-0 = <&sound_pins>, <&sound_clk_pins>; - pinctrl-names = "default"; - - /* Single DAI */ - #sound-dai-cells = <0>; - - /* audio_clkout0/1 */ - #clock-cells = <1>; - clock-frequency = <12288000 11289600>; - - status = "okay"; - - clocks = <&cpg CPG_MOD 1005>, - <&cpg CPG_MOD 1011>, <&cpg CPG_MOD 1012>, - <&cpg CPG_MOD 1025>, <&cpg CPG_MOD 1026>, - <&cpg CPG_MOD 1020>, <&cpg CPG_MOD 1021>, - <&cpg CPG_MOD 1020>, <&cpg CPG_MOD 1021>, - <&cpg CPG_MOD 1019>, <&cpg CPG_MOD 1018>, - <&cs2000>, <&audio_clk_b>, - <&cpg CPG_CORE R8A77995_CLK_ZA2>; - - ports { - rsnd_port0: port { - rsnd_for_ak4613: endpoint { - remote-endpoint = <&ak4613_endpoint>; - dai-format = "left_j"; - bitclock-master = <&rsnd_for_ak4613>; - frame-master = <&rsnd_for_ak4613>; - playback = <&ssi3>, <&src5>, <&dvc0>; - capture = <&ssi4>, <&src6>, <&dvc1>; - }; - }; - }; -}; - -&rwdt { - timeout-sec = <60>; - status = "okay"; -}; - -&scif2 { - pinctrl-0 = <&scif2_pins>; - pinctrl-names = "default"; - - status = "okay"; -}; - -&sdhi2 { - /* used for on-board eMMC */ - pinctrl-0 = <&sdhi2_pins>; - pinctrl-1 = <&sdhi2_pins_uhs>; - pinctrl-names = "default", "state_uhs"; - - vmmc-supply = <®_3p3v>; - vqmmc-supply = <®_1p8v>; - bus-width = <8>; - mmc-hs200-1_8v; - no-sd; - no-sdio; - non-removable; - status = "okay"; -}; - -&ssi4 { - shared-pin; -}; - -&usb2_phy0 { - pinctrl-0 = <&usb0_pins>; - pinctrl-names = "default"; - - renesas,no-otg-pins; - status = "okay"; -}; - -&vin4 { - pinctrl-0 = <&vin4_pins_cvbs>; - pinctrl-names = "default"; - - status = "okay"; - - ports { - port { - vin4_in: endpoint { - remote-endpoint = <&adv7180_out>; - }; - }; - }; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77995.dtsi b/arch/arm64/boot/dts/renesas/r8a77995.dtsi index 16ad5fc23a..f29f3982a4 100644 --- a/arch/arm64/boot/dts/renesas/r8a77995.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77995.dtsi @@ -1216,7 +1216,8 @@ sdhi2: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 312>; + clocks = <&cpg CPG_MOD 312>, <&cpg CPG_CORE R8A77995_CLK_SD0H>; + clock-names = "core", "clkh"; max-frequency = <200000000>; power-domains = <&sysc R8A77995_PD_ALWAYS_ON>; resets = <&cpg 312>; diff --git a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi index a0a1a1da0d..6af3f4f4f2 100644 --- a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi @@ -6,6 +6,7 @@ */ #include +#include #include #include "r8a779a0.dtsi" @@ -22,6 +23,37 @@ chosen { stdout-path = "serial0:115200n8"; }; + keys { + compatible = "gpio-keys"; + + pinctrl-0 = <&keys_pins>; + pinctrl-names = "default"; + + key-1 { + gpios = <&gpio6 18 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "SW47"; + wakeup-source; + debounce-interval = <20>; + }; + + key-2 { + gpios = <&gpio6 19 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "SW48"; + wakeup-source; + debounce-interval = <20>; + }; + + key-3 { + gpios = <&gpio6 20 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "SW49"; + wakeup-source; + debounce-interval = <20>; + }; + }; + leds { compatible = "gpio-leds"; @@ -66,6 +98,27 @@ memory@700000000 { reg = <0x7 0x00000000 0x0 0x80000000>; }; + mini-dp-con { + compatible = "dp-connector"; + label = "CN5"; + type = "mini"; + + port { + mini_dp_con_in: endpoint { + remote-endpoint = <&sn65dsi86_out>; + }; + }; + }; + + reg_1p2v: regulator-1p2v { + compatible = "regulator-fixed"; + regulator-name = "fixed-1.2V"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-boot-on; + regulator-always-on; + }; + reg_1p8v: regulator-1p8v { compatible = "regulator-fixed"; regulator-name = "fixed-1.8V"; @@ -83,6 +136,29 @@ reg_3p3v: regulator-3p3v { regulator-boot-on; regulator-always-on; }; + + sn65dsi86_refclk: clk-x6 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <38400000>; + }; +}; + +&dsi0 { + status = "okay"; + + ports { + port@1 { + dsi0_out: endpoint { + remote-endpoint = <&sn65dsi86_in>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&du { + status = "okay"; }; &extal_clk { @@ -114,6 +190,41 @@ &i2c1 { status = "okay"; clock-frequency = <400000>; + + bridge@2c { + compatible = "ti,sn65dsi86"; + reg = <0x2c>; + + clocks = <&sn65dsi86_refclk>; + clock-names = "refclk"; + + interrupt-parent = <&gpio1>; + interrupts = <24 IRQ_TYPE_LEVEL_HIGH>; + + vccio-supply = <®_1p8v>; + vpll-supply = <®_1p8v>; + vcca-supply = <®_1p2v>; + vcc-supply = <®_1p2v>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + sn65dsi86_in: endpoint { + remote-endpoint = <&dsi0_out>; + }; + }; + + port@1 { + reg = <1>; + sn65dsi86_out: endpoint { + remote-endpoint = <&mini_dp_con_in>; + }; + }; + }; + }; }; &i2c6 { @@ -160,12 +271,22 @@ i2c6_pins: i2c6 { function = "i2c6"; }; + keys_pins: keys { + pins = "GP_6_18", "GP_6_19", "GP_6_20"; + bias-pull-up; + }; + mmc_pins: mmc { groups = "mmc_data8", "mmc_ctrl", "mmc_ds"; function = "mmc"; power-source = <1800>; }; + qspi0_pins: qspi0 { + groups = "qspi0_ctrl", "qspi0_data4"; + function = "qspi0"; + }; + scif0_pins: scif0 { groups = "scif0_data", "scif0_ctrl"; function = "scif0"; @@ -177,6 +298,34 @@ scif_clk_pins: scif_clk { }; }; +&rpc { + pinctrl-0 = <&qspi0_pins>; + pinctrl-names = "default"; + + status = "okay"; + + flash@0 { + compatible = "spansion,s25fs512s", "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <40000000>; + spi-rx-bus-width = <4>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + boot@0 { + reg = <0x0 0xcc0000>; + read-only; + }; + user@cc0000 { + reg = <0xcc0000 0x3340000>; + }; + }; + }; +}; + &rwdt { timeout-sec = <60>; status = "okay"; diff --git a/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts b/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts index dc671ff57e..e46dc9aa0a 100644 --- a/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts +++ b/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts @@ -27,6 +27,8 @@ &avb0 { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id0022.1622", + "ethernet-phy-ieee802.3-c22"; rxc-skew-ps = <1500>; reg = <0>; interrupt-parent = <&gpio4>; diff --git a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi index 26899fb768..1e7ed12ebc 100644 --- a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi @@ -87,7 +87,7 @@ rwdt: watchdog@e6020000 { status = "disabled"; }; - pfc: pin-controller@e6050000 { + pfc: pinctrl@e6050000 { compatible = "renesas,pfc-r8a779a0"; reg = <0 0xe6050000 0 0x16c>, <0 0xe6050800 0 0x16c>, <0 0xe6058000 0 0x16c>, <0 0xe6058800 0 0x16c>, @@ -933,6 +933,17 @@ scif4: serial@e6c40000 { status = "disabled"; }; + tpu: pwm@e6e80000 { + compatible = "renesas,tpu-r8a779a0", "renesas,tpu"; + reg = <0 0xe6e80000 0 0x148>; + interrupts = ; + clocks = <&cpg CPG_MOD 718>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 718>; + #pwm-cells = <3>; + status = "disabled"; + }; + msiof0: spi@e6e90000 { compatible = "renesas,msiof-r8a779a0", "renesas,rcar-gen3-msiof"; @@ -1023,6 +1034,902 @@ msiof5: spi@e6c28000 { status = "disabled"; }; + vin00: video@e6ef0000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef0000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 730>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 730>; + renesas,id = <0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin00isp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0vin00>; + }; + }; + }; + }; + + vin01: video@e6ef1000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef1000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 731>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 731>; + renesas,id = <1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin01isp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0vin01>; + }; + }; + }; + }; + + vin02: video@e6ef2000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef2000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 800>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 800>; + renesas,id = <2>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin02isp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0vin02>; + }; + }; + }; + }; + + vin03: video@e6ef3000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef3000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 801>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 801>; + renesas,id = <3>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin03isp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0vin03>; + }; + }; + }; + }; + + vin04: video@e6ef4000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef4000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 802>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 802>; + renesas,id = <4>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin04isp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0vin04>; + }; + }; + }; + }; + + vin05: video@e6ef5000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef5000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 803>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 803>; + renesas,id = <5>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin05isp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0vin05>; + }; + }; + }; + }; + + vin06: video@e6ef6000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef6000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 804>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 804>; + renesas,id = <6>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin06isp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0vin06>; + }; + }; + }; + }; + + vin07: video@e6ef7000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef7000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 805>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 805>; + renesas,id = <7>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin07isp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0vin07>; + }; + }; + }; + }; + + vin08: video@e6ef8000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef8000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 806>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 806>; + renesas,id = <8>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin08isp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp1vin08>; + }; + }; + }; + }; + + vin09: video@e6ef9000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ef9000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 807>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 807>; + renesas,id = <9>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin09isp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp1vin09>; + }; + }; + }; + }; + + vin10: video@e6efa000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6efa000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 808>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 808>; + renesas,id = <10>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin10isp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp1vin10>; + }; + }; + }; + }; + + vin11: video@e6efb000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6efb000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 809>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 809>; + renesas,id = <11>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin11isp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp1vin11>; + }; + }; + }; + }; + + vin12: video@e6efc000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6efc000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 810>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 810>; + renesas,id = <12>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin12isp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp1vin12>; + }; + }; + }; + }; + + vin13: video@e6efd000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6efd000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 811>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 811>; + renesas,id = <13>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin13isp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp1vin13>; + }; + }; + }; + }; + + vin14: video@e6efe000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6efe000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 812>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 812>; + renesas,id = <14>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin14isp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp1vin14>; + }; + }; + }; + }; + + vin15: video@e6eff000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6eff000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 813>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 813>; + renesas,id = <15>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin15isp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp1vin15>; + }; + }; + }; + }; + + vin16: video@e6ed0000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed0000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 814>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 814>; + renesas,id = <16>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin16isp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&isp2vin16>; + }; + }; + }; + }; + + vin17: video@e6ed1000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed1000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 815>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 815>; + renesas,id = <17>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin17isp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&isp2vin17>; + }; + }; + }; + }; + + vin18: video@e6ed2000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed2000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 816>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 816>; + renesas,id = <18>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin18isp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&isp2vin18>; + }; + }; + }; + }; + + vin19: video@e6ed3000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed3000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 817>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 817>; + renesas,id = <19>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin19isp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&isp2vin19>; + }; + }; + }; + }; + + vin20: video@e6ed4000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed4000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 818>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 818>; + renesas,id = <20>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin20isp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&isp2vin20>; + }; + }; + }; + }; + + vin21: video@e6ed5000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed5000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 819>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 819>; + renesas,id = <21>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin21isp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&isp2vin21>; + }; + }; + }; + }; + + vin22: video@e6ed6000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed6000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 820>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 820>; + renesas,id = <22>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin22isp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&isp2vin22>; + }; + }; + }; + }; + + vin23: video@e6ed7000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed7000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 821>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 821>; + renesas,id = <23>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin23isp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&isp2vin23>; + }; + }; + }; + }; + + vin24: video@e6ed8000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed8000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 822>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 822>; + renesas,id = <24>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin24isp3: endpoint@3 { + reg = <3>; + remote-endpoint = <&isp3vin24>; + }; + }; + }; + }; + + vin25: video@e6ed9000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ed9000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 823>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 823>; + renesas,id = <25>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin25isp3: endpoint@3 { + reg = <3>; + remote-endpoint = <&isp3vin25>; + }; + }; + }; + }; + + vin26: video@e6eda000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6eda000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 824>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 824>; + renesas,id = <26>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin26isp3: endpoint@3 { + reg = <3>; + remote-endpoint = <&isp3vin26>; + }; + }; + }; + }; + + vin27: video@e6edb000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6edb000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 825>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 825>; + renesas,id = <27>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin27isp3: endpoint@3 { + reg = <3>; + remote-endpoint = <&isp3vin27>; + }; + }; + }; + }; + + vin28: video@e6edc000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6edc000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 826>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 826>; + renesas,id = <28>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin28isp3: endpoint@3 { + reg = <3>; + remote-endpoint = <&isp3vin28>; + }; + }; + }; + }; + + vin29: video@e6edd000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6edd000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 827>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 827>; + renesas,id = <29>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin29isp3: endpoint@3 { + reg = <3>; + remote-endpoint = <&isp3vin29>; + }; + }; + }; + }; + + vin30: video@e6ede000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6ede000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 828>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 828>; + renesas,id = <30>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin30isp3: endpoint@3 { + reg = <3>; + remote-endpoint = <&isp3vin30>; + }; + }; + }; + }; + + vin31: video@e6edf000 { + compatible = "renesas,vin-r8a779a0"; + reg = <0 0xe6edf000 0 0x1000>; + interrupts = ; + clocks = <&cpg CPG_MOD 829>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 829>; + renesas,id = <31>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vin31isp3: endpoint@3 { + reg = <3>; + remote-endpoint = <&isp3vin31>; + }; + }; + }; + }; + dmac1: dma-controller@e7350000 { compatible = "renesas,dmac-r8a779a0"; reg = <0 0xe7350000 0 0x1000>, @@ -1086,13 +1993,129 @@ mmc0: mmc@ee140000 { "renesas,rcar-gen3-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; - clocks = <&cpg CPG_MOD 706>; + clocks = <&cpg CPG_MOD 706>, <&cpg CPG_CORE R8A779A0_CLK_SD0H>; + clock-names = "core", "clkh"; power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; resets = <&cpg 706>; max-frequency = <200000000>; + iommus = <&ipmmu_ds0 32>; status = "disabled"; }; + rpc: spi@ee200000 { + compatible = "renesas,r8a779a0-rpc-if", + "renesas,rcar-gen3-rpc-if"; + reg = <0 0xee200000 0 0x200>, + <0 0x08000000 0 0x04000000>, + <0 0xee208000 0 0x100>; + reg-names = "regs", "dirmap", "wbuf"; + interrupts = ; + clocks = <&cpg CPG_MOD 629>; + clock-names = "rpc"; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 629>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + ipmmu_rt0: iommu@ee480000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xee480000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 10>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_rt1: iommu@ee4c0000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xee4c0000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 19>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_ds0: iommu@eed00000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeed00000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 0>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_ds1: iommu@eed40000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeed40000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 1>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_ir: iommu@eed80000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeed80000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 3>; + power-domains = <&sysc R8A779A0_PD_A3IR>; + #iommu-cells = <1>; + }; + + ipmmu_vc0: iommu@eedc0000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeedc0000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 12>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_vi0: iommu@eee80000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeee80000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 14>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_vi1: iommu@eeec0000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeeec0000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 15>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_3dg: iommu@eee00000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeee00000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 6>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_vip0: iommu@eef00000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeef00000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 5>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_vip1: iommu@eef40000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeef40000 0 0x20000>; + renesas,ipmmu-main = <&ipmmu_mm 11>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + + ipmmu_mm: iommu@eefc0000 { + compatible = "renesas,ipmmu-r8a779a0"; + reg = <0 0xeefc0000 0 0x20000>; + interrupts = , + ; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + #iommu-cells = <1>; + }; + gic: interrupt-controller@f1000000 { compatible = "arm,gic-v3"; #interrupt-cells = <3>; @@ -1142,6 +2165,532 @@ vspd1: vsp@fea28000 { renesas,fcp = <&fcpvd1>; }; + csi40: csi2@feaa0000 { + compatible = "renesas,r8a779a0-csi2"; + reg = <0 0xfeaa0000 0 0x10000>; + interrupts = ; + clocks = <&cpg CPG_MOD 331>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 331>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + csi40isp0: endpoint { + remote-endpoint = <&isp0csi40>; + }; + }; + }; + }; + + csi41: csi2@feab0000 { + compatible = "renesas,r8a779a0-csi2"; + reg = <0 0xfeab0000 0 0x10000>; + interrupts = ; + clocks = <&cpg CPG_MOD 400>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 400>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + csi41isp1: endpoint { + remote-endpoint = <&isp1csi41>; + }; + }; + }; + }; + + csi42: csi2@fed60000 { + compatible = "renesas,r8a779a0-csi2"; + reg = <0 0xfed60000 0 0x10000>; + interrupts = ; + clocks = <&cpg CPG_MOD 401>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 401>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + csi42isp2: endpoint { + remote-endpoint = <&isp2csi42>; + }; + }; + }; + }; + + csi43: csi2@fed70000 { + compatible = "renesas,r8a779a0-csi2"; + reg = <0 0xfed70000 0 0x10000>; + interrupts = ; + clocks = <&cpg CPG_MOD 402>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 402>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + csi43isp3: endpoint { + remote-endpoint = <&isp3csi43>; + }; + }; + }; + }; + + du: display@feb00000 { + compatible = "renesas,du-r8a779a0"; + reg = <0 0xfeb00000 0 0x40000>; + interrupts = , + ; + clocks = <&cpg CPG_MOD 411>; + clock-names = "du.0"; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + resets = <&cpg 411>; + reset-names = "du.0"; + renesas,vsps = <&vspd0 0>, <&vspd1 0>; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + du_out_dsi0: endpoint { + remote-endpoint = <&dsi0_in>; + }; + }; + + port@1 { + reg = <1>; + du_out_dsi1: endpoint { + remote-endpoint = <&dsi1_in>; + }; + }; + }; + }; + + isp0: isp@fed00000 { + compatible = "renesas,r8a779a0-isp"; + reg = <0 0xfed00000 0 0x10000>; + interrupts = ; + clocks = <&cpg CPG_MOD 612>; + power-domains = <&sysc R8A779A0_PD_A3ISP01>; + resets = <&cpg 612>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <0>; + + isp0csi40: endpoint@0 { + reg = <0>; + remote-endpoint = <&csi40isp0>; + }; + }; + + port@1 { + reg = <1>; + isp0vin00: endpoint { + remote-endpoint = <&vin00isp0>; + }; + }; + + port@2 { + reg = <2>; + isp0vin01: endpoint { + remote-endpoint = <&vin01isp0>; + }; + }; + + port@3 { + reg = <3>; + isp0vin02: endpoint { + remote-endpoint = <&vin02isp0>; + }; + }; + + port@4 { + reg = <4>; + isp0vin03: endpoint { + remote-endpoint = <&vin03isp0>; + }; + }; + + port@5 { + reg = <5>; + isp0vin04: endpoint { + remote-endpoint = <&vin04isp0>; + }; + }; + + port@6 { + reg = <6>; + isp0vin05: endpoint { + remote-endpoint = <&vin05isp0>; + }; + }; + + port@7 { + reg = <7>; + isp0vin06: endpoint { + remote-endpoint = <&vin06isp0>; + }; + }; + + port@8 { + reg = <8>; + isp0vin07: endpoint { + remote-endpoint = <&vin07isp0>; + }; + }; + }; + }; + + isp1: isp@fed20000 { + compatible = "renesas,r8a779a0-isp"; + reg = <0 0xfed20000 0 0x10000>; + interrupts = ; + clocks = <&cpg CPG_MOD 613>; + power-domains = <&sysc R8A779A0_PD_A3ISP01>; + resets = <&cpg 613>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <0>; + + isp1csi41: endpoint@1 { + reg = <1>; + remote-endpoint = <&csi41isp1>; + }; + }; + + port@1 { + reg = <1>; + isp1vin08: endpoint { + remote-endpoint = <&vin08isp1>; + }; + }; + + port@2 { + reg = <2>; + isp1vin09: endpoint { + remote-endpoint = <&vin09isp1>; + }; + }; + + port@3 { + reg = <3>; + isp1vin10: endpoint { + remote-endpoint = <&vin10isp1>; + }; + }; + + port@4 { + reg = <4>; + isp1vin11: endpoint { + remote-endpoint = <&vin11isp1>; + }; + }; + + port@5 { + reg = <5>; + isp1vin12: endpoint { + remote-endpoint = <&vin12isp1>; + }; + }; + + port@6 { + reg = <6>; + isp1vin13: endpoint { + remote-endpoint = <&vin13isp1>; + }; + }; + + port@7 { + reg = <7>; + isp1vin14: endpoint { + remote-endpoint = <&vin14isp1>; + }; + }; + + port@8 { + reg = <8>; + isp1vin15: endpoint { + remote-endpoint = <&vin15isp1>; + }; + }; + }; + }; + + isp2: isp@fed30000 { + compatible = "renesas,r8a779a0-isp"; + reg = <0 0xfed30000 0 0x10000>; + interrupts = ; + clocks = <&cpg CPG_MOD 614>; + power-domains = <&sysc R8A779A0_PD_A3ISP23>; + resets = <&cpg 614>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <0>; + + isp2csi42: endpoint@0 { + reg = <0>; + remote-endpoint = <&csi42isp2>; + }; + }; + + port@1 { + reg = <1>; + isp2vin16: endpoint { + remote-endpoint = <&vin16isp2>; + }; + }; + + port@2 { + reg = <2>; + isp2vin17: endpoint { + remote-endpoint = <&vin17isp2>; + }; + }; + + port@3 { + reg = <3>; + isp2vin18: endpoint { + remote-endpoint = <&vin18isp2>; + }; + }; + + port@4 { + reg = <4>; + isp2vin19: endpoint { + remote-endpoint = <&vin19isp2>; + }; + }; + + port@5 { + reg = <5>; + isp2vin20: endpoint { + remote-endpoint = <&vin20isp2>; + }; + }; + + port@6 { + reg = <6>; + isp2vin21: endpoint { + remote-endpoint = <&vin21isp2>; + }; + }; + + port@7 { + reg = <7>; + isp2vin22: endpoint { + remote-endpoint = <&vin22isp2>; + }; + }; + + port@8 { + reg = <8>; + isp2vin23: endpoint { + remote-endpoint = <&vin23isp2>; + }; + }; + }; + }; + + isp3: isp@fed40000 { + compatible = "renesas,r8a779a0-isp"; + reg = <0 0xfed40000 0 0x10000>; + interrupts = ; + clocks = <&cpg CPG_MOD 615>; + power-domains = <&sysc R8A779A0_PD_A3ISP23>; + resets = <&cpg 615>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <0>; + + isp3csi43: endpoint@1 { + reg = <1>; + remote-endpoint = <&csi43isp3>; + }; + }; + + port@1 { + reg = <1>; + isp3vin24: endpoint { + remote-endpoint = <&vin24isp3>; + }; + }; + + port@2 { + reg = <2>; + isp3vin25: endpoint { + remote-endpoint = <&vin25isp3>; + }; + }; + + port@3 { + reg = <3>; + isp3vin26: endpoint { + remote-endpoint = <&vin26isp3>; + }; + }; + + port@4 { + reg = <4>; + isp3vin27: endpoint { + remote-endpoint = <&vin27isp3>; + }; + }; + + port@5 { + reg = <5>; + isp3vin28: endpoint { + remote-endpoint = <&vin28isp3>; + }; + }; + + port@6 { + reg = <6>; + isp3vin29: endpoint { + remote-endpoint = <&vin29isp3>; + }; + }; + + port@7 { + reg = <7>; + isp3vin30: endpoint { + remote-endpoint = <&vin30isp3>; + }; + }; + + port@8 { + reg = <8>; + isp3vin31: endpoint { + remote-endpoint = <&vin31isp3>; + }; + }; + }; + }; + + dsi0: dsi-encoder@fed80000 { + compatible = "renesas,r8a779a0-dsi-csi2-tx"; + reg = <0 0xfed80000 0 0x10000>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + clocks = <&cpg CPG_MOD 415>, + <&cpg CPG_CORE R8A779A0_CLK_DSI>, + <&cpg CPG_CORE R8A779A0_CLK_CL16MCK>; + clock-names = "fck", "dsi", "pll"; + resets = <&cpg 415>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dsi0_in: endpoint { + remote-endpoint = <&du_out_dsi0>; + }; + }; + + port@1 { + reg = <1>; + }; + }; + }; + + dsi1: dsi-encoder@fed90000 { + compatible = "renesas,r8a779a0-dsi-csi2-tx"; + reg = <0 0xfed90000 0 0x10000>; + power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; + clocks = <&cpg CPG_MOD 416>, + <&cpg CPG_CORE R8A779A0_CLK_DSI>, + <&cpg CPG_CORE R8A779A0_CLK_CL16MCK>; + clock-names = "fck", "dsi", "pll"; + resets = <&cpg 416>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dsi1_in: endpoint { + remote-endpoint = <&du_out_dsi1>; + }; + }; + + port@1 { + reg = <1>; + }; + }; + }; + prr: chipid@fff00044 { compatible = "renesas,prr"; reg = <0 0xfff00044 0 4>; diff --git a/arch/arm64/boot/dts/renesas/r8a779m1.dtsi b/arch/arm64/boot/dts/renesas/r8a779m1.dtsi index 0e9b04469b..b6e855f52a 100644 --- a/arch/arm64/boot/dts/renesas/r8a779m1.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779m1.dtsi @@ -10,3 +10,12 @@ / { compatible = "renesas,r8a779m1", "renesas,r8a7795"; }; + +&cluster0_opp { + opp-2000000000 { + opp-hz = /bits/ 64 <2000000000>; + opp-microvolt = <960000>; + clock-latency-ns = <300000>; + turbo-mode; + }; +}; diff --git a/arch/arm64/boot/dts/renesas/r8a779m3.dtsi b/arch/arm64/boot/dts/renesas/r8a779m3.dtsi index 65bb6188cc..6cff38a6d2 100644 --- a/arch/arm64/boot/dts/renesas/r8a779m3.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779m3.dtsi @@ -10,3 +10,12 @@ / { compatible = "renesas,r8a779m3", "renesas,r8a77961"; }; + +&cluster0_opp { + opp-2000000000 { + opp-hz = /bits/ 64 <2000000000>; + opp-microvolt = <960000>; + clock-latency-ns = <300000>; + turbo-mode; + }; +}; diff --git a/arch/arm64/boot/dts/renesas/r9a07g044.dtsi b/arch/arm64/boot/dts/renesas/r9a07g044.dtsi index 5f3bc2898d..19287cccb1 100644 --- a/arch/arm64/boot/dts/renesas/r9a07g044.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a07g044.dtsi @@ -13,6 +13,20 @@ / { #address-cells = <2>; #size-cells = <2>; + audio_clk1: audio_clk1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + /* This value must be overridden by boards that provide it */ + clock-frequency = <0>; + }; + + audio_clk2: audio_clk2 { + compatible = "fixed-clock"; + #clock-cells = <0>; + /* This value must be overridden by boards that provide it */ + clock-frequency = <0>; + }; + /* External CAN clock - to be overridden by boards that provide it */ can_clk: can { compatible = "fixed-clock"; @@ -28,9 +42,31 @@ extal_clk: extal { clock-frequency = <0>; }; - psci { - compatible = "arm,psci-1.0", "arm,psci-0.2"; - method = "smc"; + cluster0_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + + opp-150000000 { + opp-hz = /bits/ 64 <150000000>; + opp-microvolt = <1100000>; + clock-latency-ns = <300000>; + }; + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <1100000>; + clock-latency-ns = <300000>; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <1100000>; + clock-latency-ns = <300000>; + }; + opp-1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <1100000>; + clock-latency-ns = <300000>; + opp-suspend; + }; }; cpus { @@ -52,8 +88,11 @@ cpu0: cpu@0 { compatible = "arm,cortex-a55"; reg = <0>; device_type = "cpu"; + #cooling-cells = <2>; next-level-cache = <&L3_CA55>; enable-method = "psci"; + clocks = <&cpg CPG_CORE R9A07G044_CLK_I>; + operating-points-v2 = <&cluster0_opp>; }; cpu1: cpu@100 { @@ -62,6 +101,8 @@ cpu1: cpu@100 { device_type = "cpu"; next-level-cache = <&L3_CA55>; enable-method = "psci"; + clocks = <&cpg CPG_CORE R9A07G044_CLK_I>; + operating-points-v2 = <&cluster0_opp>; }; L3_CA55: cache-controller-0 { @@ -71,6 +112,55 @@ L3_CA55: cache-controller-0 { }; }; + gpu_opp_table: opp-table-1 { + compatible = "operating-points-v2"; + + opp-500000000 { + opp-hz = /bits/ 64 <500000000>; + opp-microvolt = <1100000>; + }; + + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <1100000>; + }; + + opp-250000000 { + opp-hz = /bits/ 64 <250000000>; + opp-microvolt = <1100000>; + }; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <1100000>; + }; + + opp-125000000 { + opp-hz = /bits/ 64 <125000000>; + opp-microvolt = <1100000>; + }; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + opp-microvolt = <1100000>; + }; + + opp-62500000 { + opp-hz = /bits/ 64 <62500000>; + opp-microvolt = <1100000>; + }; + + opp-50000000 { + opp-hz = /bits/ 64 <50000000>; + opp-microvolt = <1100000>; + }; + }; + + psci { + compatible = "arm,psci-1.0", "arm,psci-0.2"; + method = "smc"; + }; + soc: soc { compatible = "simple-bus"; interrupt-parent = <&gic>; @@ -78,6 +168,138 @@ soc: soc { #size-cells = <2>; ranges; + ssi0: ssi@10049c00 { + compatible = "renesas,r9a07g044-ssi", + "renesas,rz-ssi"; + reg = <0 0x10049c00 0 0x400>; + interrupts = , + , + , + ; + interrupt-names = "int_req", "dma_rx", "dma_tx", "dma_rt"; + clocks = <&cpg CPG_MOD R9A07G044_SSI0_PCLK2>, + <&cpg CPG_MOD R9A07G044_SSI0_PCLK_SFR>, + <&audio_clk1>, <&audio_clk2>; + clock-names = "ssi", "ssi_sfr", "audio_clk1", "audio_clk2"; + resets = <&cpg R9A07G044_SSI0_RST_M2_REG>; + dmas = <&dmac 0x2655>, <&dmac 0x2656>; + dma-names = "tx", "rx"; + power-domains = <&cpg>; + #sound-dai-cells = <0>; + status = "disabled"; + }; + + ssi1: ssi@1004a000 { + compatible = "renesas,r9a07g044-ssi", + "renesas,rz-ssi"; + reg = <0 0x1004a000 0 0x400>; + interrupts = , + , + , + ; + interrupt-names = "int_req", "dma_rx", "dma_tx", "dma_rt"; + clocks = <&cpg CPG_MOD R9A07G044_SSI1_PCLK2>, + <&cpg CPG_MOD R9A07G044_SSI1_PCLK_SFR>, + <&audio_clk1>, <&audio_clk2>; + clock-names = "ssi", "ssi_sfr", "audio_clk1", "audio_clk2"; + resets = <&cpg R9A07G044_SSI1_RST_M2_REG>; + dmas = <&dmac 0x2659>, <&dmac 0x265a>; + dma-names = "tx", "rx"; + power-domains = <&cpg>; + #sound-dai-cells = <0>; + status = "disabled"; + }; + + ssi2: ssi@1004a400 { + compatible = "renesas,r9a07g044-ssi", + "renesas,rz-ssi"; + reg = <0 0x1004a400 0 0x400>; + interrupts = , + , + , + ; + interrupt-names = "int_req", "dma_rx", "dma_tx", "dma_rt"; + clocks = <&cpg CPG_MOD R9A07G044_SSI2_PCLK2>, + <&cpg CPG_MOD R9A07G044_SSI2_PCLK_SFR>, + <&audio_clk1>, <&audio_clk2>; + clock-names = "ssi", "ssi_sfr", "audio_clk1", "audio_clk2"; + resets = <&cpg R9A07G044_SSI2_RST_M2_REG>; + dmas = <&dmac 0x265f>; + dma-names = "rt"; + power-domains = <&cpg>; + #sound-dai-cells = <0>; + status = "disabled"; + }; + + ssi3: ssi@1004a800 { + compatible = "renesas,r9a07g044-ssi", + "renesas,rz-ssi"; + reg = <0 0x1004a800 0 0x400>; + interrupts = , + , + , + ; + interrupt-names = "int_req", "dma_rx", "dma_tx", "dma_rt"; + clocks = <&cpg CPG_MOD R9A07G044_SSI3_PCLK2>, + <&cpg CPG_MOD R9A07G044_SSI3_PCLK_SFR>, + <&audio_clk1>, <&audio_clk2>; + clock-names = "ssi", "ssi_sfr", "audio_clk1", "audio_clk2"; + resets = <&cpg R9A07G044_SSI3_RST_M2_REG>; + dmas = <&dmac 0x2661>, <&dmac 0x2662>; + dma-names = "tx", "rx"; + power-domains = <&cpg>; + #sound-dai-cells = <0>; + status = "disabled"; + }; + + spi0: spi@1004ac00 { + compatible = "renesas,r9a07g044-rspi", "renesas,rspi-rz"; + reg = <0 0x1004ac00 0 0x400>; + interrupts = , + , + ; + interrupt-names = "error", "rx", "tx"; + clocks = <&cpg CPG_MOD R9A07G044_RSPI0_CLKB>; + resets = <&cpg R9A07G044_RSPI0_RST>; + power-domains = <&cpg>; + num-cs = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi1: spi@1004b000 { + compatible = "renesas,r9a07g044-rspi", "renesas,rspi-rz"; + reg = <0 0x1004b000 0 0x400>; + interrupts = , + , + ; + interrupt-names = "error", "rx", "tx"; + clocks = <&cpg CPG_MOD R9A07G044_RSPI1_CLKB>; + resets = <&cpg R9A07G044_RSPI1_RST>; + power-domains = <&cpg>; + num-cs = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi2: spi@1004b400 { + compatible = "renesas,r9a07g044-rspi", "renesas,rspi-rz"; + reg = <0 0x1004b400 0 0x400>; + interrupts = , + , + ; + interrupt-names = "error", "rx", "tx"; + clocks = <&cpg CPG_MOD R9A07G044_RSPI2_CLKB>; + resets = <&cpg R9A07G044_RSPI2_RST>; + power-domains = <&cpg>; + num-cs = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + scif0: serial@1004b800 { compatible = "renesas,scif-r9a07g044"; reg = <0 0x1004b800 0 0x400>; @@ -96,6 +318,108 @@ scif0: serial@1004b800 { status = "disabled"; }; + scif1: serial@1004bc00 { + compatible = "renesas,scif-r9a07g044"; + reg = <0 0x1004bc00 0 0x400>; + interrupts = , + , + , + , + , + ; + interrupt-names = "eri", "rxi", "txi", + "bri", "dri", "tei"; + clocks = <&cpg CPG_MOD R9A07G044_SCIF1_CLK_PCK>; + clock-names = "fck"; + power-domains = <&cpg>; + resets = <&cpg R9A07G044_SCIF1_RST_SYSTEM_N>; + status = "disabled"; + }; + + scif2: serial@1004c000 { + compatible = "renesas,scif-r9a07g044"; + reg = <0 0x1004c000 0 0x400>; + interrupts = , + , + , + , + , + ; + interrupt-names = "eri", "rxi", "txi", + "bri", "dri", "tei"; + clocks = <&cpg CPG_MOD R9A07G044_SCIF2_CLK_PCK>; + clock-names = "fck"; + power-domains = <&cpg>; + resets = <&cpg R9A07G044_SCIF2_RST_SYSTEM_N>; + status = "disabled"; + }; + + scif3: serial@1004c400 { + compatible = "renesas,scif-r9a07g044"; + reg = <0 0x1004c400 0 0x400>; + interrupts = , + , + , + , + , + ; + interrupt-names = "eri", "rxi", "txi", + "bri", "dri", "tei"; + clocks = <&cpg CPG_MOD R9A07G044_SCIF3_CLK_PCK>; + clock-names = "fck"; + power-domains = <&cpg>; + resets = <&cpg R9A07G044_SCIF3_RST_SYSTEM_N>; + status = "disabled"; + }; + + scif4: serial@1004c800 { + compatible = "renesas,scif-r9a07g044"; + reg = <0 0x1004c800 0 0x400>; + interrupts = , + , + , + , + , + ; + interrupt-names = "eri", "rxi", "txi", + "bri", "dri", "tei"; + clocks = <&cpg CPG_MOD R9A07G044_SCIF4_CLK_PCK>; + clock-names = "fck"; + power-domains = <&cpg>; + resets = <&cpg R9A07G044_SCIF4_RST_SYSTEM_N>; + status = "disabled"; + }; + + sci0: serial@1004d000 { + compatible = "renesas,r9a07g044-sci", "renesas,sci"; + reg = <0 0x1004d000 0 0x400>; + interrupts = , + , + , + ; + interrupt-names = "eri", "rxi", "txi", "tei"; + clocks = <&cpg CPG_MOD R9A07G044_SCI0_CLKP>; + clock-names = "fck"; + power-domains = <&cpg>; + resets = <&cpg R9A07G044_SCI0_RST>; + status = "disabled"; + }; + + sci1: serial@1004d400 { + compatible = "renesas,r9a07g044-sci", "renesas,sci"; + reg = <0 0x1004d400 0 0x400>; + interrupts = , + , + , + ; + interrupt-names = "eri", "rxi", "txi", "tei"; + clocks = <&cpg CPG_MOD R9A07G044_SCI1_CLKP>; + clock-names = "fck"; + power-domains = <&cpg>; + resets = <&cpg R9A07G044_SCI1_RST>; + status = "disabled"; + }; + canfd: can@10050000 { compatible = "renesas,r9a07g044-canfd", "renesas,rzg2l-canfd"; reg = <0 0x10050000 0 0x8000>; @@ -260,6 +584,33 @@ channel@7 { }; }; + tsu: thermal@10059400 { + compatible = "renesas,r9a07g044-tsu", + "renesas,rzg2l-tsu"; + reg = <0 0x10059400 0 0x400>; + clocks = <&cpg CPG_MOD R9A07G044_TSU_PCLK>; + resets = <&cpg R9A07G044_TSU_PRESETN>; + power-domains = <&cpg>; + #thermal-sensor-cells = <1>; + }; + + sbc: spi@10060000 { + compatible = "renesas,r9a07g044-rpc-if", + "renesas,rzg2l-rpc-if"; + reg = <0 0x10060000 0 0x10000>, + <0 0x20000000 0 0x10000000>, + <0 0x10070000 0 0x10000>; + reg-names = "regs", "dirmap", "wbuf"; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_SPI_CLK2>, + <&cpg CPG_MOD R9A07G044_SPI_CLK>; + resets = <&cpg R9A07G044_SPI_RST>; + power-domains = <&cpg>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + cpg: clock-controller@11010000 { compatible = "renesas,r9a07g044-cpg"; reg = <0 0x11010000 0 0x10000>; @@ -282,7 +633,7 @@ sysc: system-controller@11020000 { status = "disabled"; }; - pinctrl: pin-controller@11030000 { + pinctrl: pinctrl@11030000 { compatible = "renesas,r9a07g044-pinctrl"; reg = <0 0x11030000 0 0x10000>; gpio-controller; @@ -295,6 +646,63 @@ pinctrl: pin-controller@11030000 { <&cpg R9A07G044_GPIO_SPARE_RESETN>; }; + dmac: dma-controller@11820000 { + compatible = "renesas,r9a07g044-dmac", + "renesas,rz-dmac"; + reg = <0 0x11820000 0 0x10000>, + <0 0x11830000 0 0x10000>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "error", + "ch0", "ch1", "ch2", "ch3", + "ch4", "ch5", "ch6", "ch7", + "ch8", "ch9", "ch10", "ch11", + "ch12", "ch13", "ch14", "ch15"; + clocks = <&cpg CPG_MOD R9A07G044_DMAC_ACLK>, + <&cpg CPG_MOD R9A07G044_DMAC_PCLK>; + power-domains = <&cpg>; + resets = <&cpg R9A07G044_DMAC_ARESETN>, + <&cpg R9A07G044_DMAC_RST_ASYNC>; + #dma-cells = <1>; + dma-channels = <16>; + }; + + gpu: gpu@11840000 { + compatible = "renesas,r9a07g044-mali", + "arm,mali-bifrost"; + reg = <0x0 0x11840000 0x0 0x10000>; + interrupts = , + , + , + ; + interrupt-names = "job", "mmu", "gpu", "event"; + clocks = <&cpg CPG_MOD R9A07G044_GPU_CLK>, + <&cpg CPG_MOD R9A07G044_GPU_AXI_CLK>, + <&cpg CPG_MOD R9A07G044_GPU_ACE_CLK>; + clock-names = "gpu", "bus", "bus_ace"; + power-domains = <&cpg>; + resets = <&cpg R9A07G044_GPU_RESETN>, + <&cpg R9A07G044_GPU_AXI_RESETN>, + <&cpg R9A07G044_GPU_ACE_RESETN>; + reset-names = "rst", "axi_rst", "ace_rst"; + operating-points-v2 = <&gpu_opp_table>; + }; + gic: interrupt-controller@11900000 { compatible = "arm,gic-v3"; #interrupt-cells = <3>; @@ -304,6 +712,301 @@ gic: interrupt-controller@11900000 { <0x0 0x11940000 0 0x60000>; interrupts = ; }; + + sdhi0: mmc@11c00000 { + compatible = "renesas,sdhi-r9a07g044", + "renesas,rcar-gen3-sdhi"; + reg = <0x0 0x11c00000 0 0x10000>; + interrupts = , + ; + clocks = <&cpg CPG_MOD R9A07G044_SDHI0_IMCLK>, + <&cpg CPG_MOD R9A07G044_SDHI0_CLK_HS>, + <&cpg CPG_MOD R9A07G044_SDHI0_IMCLK2>, + <&cpg CPG_MOD R9A07G044_SDHI0_ACLK>; + clock-names = "core", "clkh", "cd", "aclk"; + resets = <&cpg R9A07G044_SDHI0_IXRST>; + power-domains = <&cpg>; + status = "disabled"; + }; + + sdhi1: mmc@11c10000 { + compatible = "renesas,sdhi-r9a07g044", + "renesas,rcar-gen3-sdhi"; + reg = <0x0 0x11c10000 0 0x10000>; + interrupts = , + ; + clocks = <&cpg CPG_MOD R9A07G044_SDHI1_IMCLK>, + <&cpg CPG_MOD R9A07G044_SDHI1_CLK_HS>, + <&cpg CPG_MOD R9A07G044_SDHI1_IMCLK2>, + <&cpg CPG_MOD R9A07G044_SDHI1_ACLK>; + clock-names = "core", "clkh", "cd", "aclk"; + resets = <&cpg R9A07G044_SDHI1_IXRST>; + power-domains = <&cpg>; + status = "disabled"; + }; + + eth0: ethernet@11c20000 { + compatible = "renesas,r9a07g044-gbeth", + "renesas,rzg2l-gbeth"; + reg = <0 0x11c20000 0 0x10000>; + interrupts = , + , + ; + interrupt-names = "mux", "fil", "arp_ns"; + phy-mode = "rgmii"; + clocks = <&cpg CPG_MOD R9A07G044_ETH0_CLK_AXI>, + <&cpg CPG_MOD R9A07G044_ETH0_CLK_CHI>, + <&cpg CPG_CORE R9A07G044_CLK_HP>; + clock-names = "axi", "chi", "refclk"; + resets = <&cpg R9A07G044_ETH0_RST_HW_N>; + power-domains = <&cpg>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + eth1: ethernet@11c30000 { + compatible = "renesas,r9a07g044-gbeth", + "renesas,rzg2l-gbeth"; + reg = <0 0x11c30000 0 0x10000>; + interrupts = , + , + ; + interrupt-names = "mux", "fil", "arp_ns"; + phy-mode = "rgmii"; + clocks = <&cpg CPG_MOD R9A07G044_ETH1_CLK_AXI>, + <&cpg CPG_MOD R9A07G044_ETH1_CLK_CHI>, + <&cpg CPG_CORE R9A07G044_CLK_HP>; + clock-names = "axi", "chi", "refclk"; + resets = <&cpg R9A07G044_ETH1_RST_HW_N>; + power-domains = <&cpg>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + phyrst: usbphy-ctrl@11c40000 { + compatible = "renesas,r9a07g044-usbphy-ctrl", + "renesas,rzg2l-usbphy-ctrl"; + reg = <0 0x11c40000 0 0x10000>; + clocks = <&cpg CPG_MOD R9A07G044_USB_PCLK>; + resets = <&cpg R9A07G044_USB_PRESETN>; + power-domains = <&cpg>; + #reset-cells = <1>; + status = "disabled"; + }; + + ohci0: usb@11c50000 { + compatible = "generic-ohci"; + reg = <0 0x11c50000 0 0x100>; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_USB_PCLK>, + <&cpg CPG_MOD R9A07G044_USB_U2H0_HCLK>; + resets = <&phyrst 0>, + <&cpg R9A07G044_USB_U2H0_HRESETN>; + phys = <&usb2_phy0 1>; + phy-names = "usb"; + power-domains = <&cpg>; + status = "disabled"; + }; + + ohci1: usb@11c70000 { + compatible = "generic-ohci"; + reg = <0 0x11c70000 0 0x100>; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_USB_PCLK>, + <&cpg CPG_MOD R9A07G044_USB_U2H1_HCLK>; + resets = <&phyrst 1>, + <&cpg R9A07G044_USB_U2H1_HRESETN>; + phys = <&usb2_phy1 1>; + phy-names = "usb"; + power-domains = <&cpg>; + status = "disabled"; + }; + + ehci0: usb@11c50100 { + compatible = "generic-ehci"; + reg = <0 0x11c50100 0 0x100>; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_USB_PCLK>, + <&cpg CPG_MOD R9A07G044_USB_U2H0_HCLK>; + resets = <&phyrst 0>, + <&cpg R9A07G044_USB_U2H0_HRESETN>; + phys = <&usb2_phy0 2>; + phy-names = "usb"; + companion = <&ohci0>; + power-domains = <&cpg>; + status = "disabled"; + }; + + ehci1: usb@11c70100 { + compatible = "generic-ehci"; + reg = <0 0x11c70100 0 0x100>; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_USB_PCLK>, + <&cpg CPG_MOD R9A07G044_USB_U2H1_HCLK>; + resets = <&phyrst 1>, + <&cpg R9A07G044_USB_U2H1_HRESETN>; + phys = <&usb2_phy1 2>; + phy-names = "usb"; + companion = <&ohci1>; + power-domains = <&cpg>; + status = "disabled"; + }; + + usb2_phy0: usb-phy@11c50200 { + compatible = "renesas,usb2-phy-r9a07g044", + "renesas,rzg2l-usb2-phy"; + reg = <0 0x11c50200 0 0x700>; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_USB_PCLK>, + <&cpg CPG_MOD R9A07G044_USB_U2H0_HCLK>; + resets = <&phyrst 0>; + #phy-cells = <1>; + power-domains = <&cpg>; + status = "disabled"; + }; + + usb2_phy1: usb-phy@11c70200 { + compatible = "renesas,usb2-phy-r9a07g044", + "renesas,rzg2l-usb2-phy"; + reg = <0 0x11c70200 0 0x700>; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_USB_PCLK>, + <&cpg CPG_MOD R9A07G044_USB_U2H1_HCLK>; + resets = <&phyrst 1>; + #phy-cells = <1>; + power-domains = <&cpg>; + status = "disabled"; + }; + + hsusb: usb@11c60000 { + compatible = "renesas,usbhs-r9a07g044", + "renesas,rza2-usbhs"; + reg = <0 0x11c60000 0 0x10000>; + interrupts = , + , + , + ; + clocks = <&cpg CPG_MOD R9A07G044_USB_PCLK>, + <&cpg CPG_MOD R9A07G044_USB_U2P_EXR_CPUCLK>; + resets = <&phyrst 0>, + <&cpg R9A07G044_USB_U2P_EXL_SYSRST>; + renesas,buswait = <7>; + phys = <&usb2_phy0 3>; + phy-names = "usb"; + power-domains = <&cpg>; + status = "disabled"; + }; + + wdt0: watchdog@12800800 { + compatible = "renesas,r9a07g044-wdt", + "renesas,rzg2l-wdt"; + reg = <0 0x12800800 0 0x400>; + clocks = <&cpg CPG_MOD R9A07G044_WDT0_PCLK>, + <&cpg CPG_MOD R9A07G044_WDT0_CLK>; + clock-names = "pclk", "oscclk"; + interrupts = , + ; + interrupt-names = "wdt", "perrout"; + resets = <&cpg R9A07G044_WDT0_PRESETN>; + power-domains = <&cpg>; + status = "disabled"; + }; + + wdt1: watchdog@12800c00 { + compatible = "renesas,r9a07g044-wdt", + "renesas,rzg2l-wdt"; + reg = <0 0x12800C00 0 0x400>; + clocks = <&cpg CPG_MOD R9A07G044_WDT1_PCLK>, + <&cpg CPG_MOD R9A07G044_WDT1_CLK>; + clock-names = "pclk", "oscclk"; + interrupts = , + ; + interrupt-names = "wdt", "perrout"; + resets = <&cpg R9A07G044_WDT1_PRESETN>; + power-domains = <&cpg>; + status = "disabled"; + }; + + wdt2: watchdog@12800400 { + compatible = "renesas,r9a07g044-wdt", + "renesas,rzg2l-wdt"; + reg = <0 0x12800400 0 0x400>; + clocks = <&cpg CPG_MOD R9A07G044_WDT2_PCLK>, + <&cpg CPG_MOD R9A07G044_WDT2_CLK>; + clock-names = "pclk", "oscclk"; + interrupts = , + ; + interrupt-names = "wdt", "perrout"; + resets = <&cpg R9A07G044_WDT2_PRESETN>; + power-domains = <&cpg>; + status = "disabled"; + }; + + ostm0: timer@12801000 { + compatible = "renesas,r9a07g044-ostm", + "renesas,ostm"; + reg = <0x0 0x12801000 0x0 0x400>; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_OSTM0_PCLK>; + resets = <&cpg R9A07G044_OSTM0_PRESETZ>; + power-domains = <&cpg>; + status = "disabled"; + }; + + ostm1: timer@12801400 { + compatible = "renesas,r9a07g044-ostm", + "renesas,ostm"; + reg = <0x0 0x12801400 0x0 0x400>; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_OSTM1_PCLK>; + resets = <&cpg R9A07G044_OSTM1_PRESETZ>; + power-domains = <&cpg>; + status = "disabled"; + }; + + ostm2: timer@12801800 { + compatible = "renesas,r9a07g044-ostm", + "renesas,ostm"; + reg = <0x0 0x12801800 0x0 0x400>; + interrupts = ; + clocks = <&cpg CPG_MOD R9A07G044_OSTM2_PCLK>; + resets = <&cpg R9A07G044_OSTM2_PRESETZ>; + power-domains = <&cpg>; + status = "disabled"; + }; + }; + + thermal-zones { + cpu-thermal { + polling-delay-passive = <250>; + polling-delay = <1000>; + thermal-sensors = <&tsu 0>; + sustainable-power = <717>; + + cooling-maps { + map0 { + trip = <&target>; + cooling-device = <&cpu0 0 2>; + contribution = <1024>; + }; + }; + + trips { + sensor_crit: sensor-crit { + temperature = <125000>; + hysteresis = <1000>; + type = "critical"; + }; + + target: trip-point { + temperature = <100000>; + hysteresis = <1000>; + type = "passive"; + }; + }; + }; }; timer { diff --git a/arch/arm64/boot/dts/renesas/r9a07g044l2-smarc.dts b/arch/arm64/boot/dts/renesas/r9a07g044l2-smarc.dts index d3f72ec62f..247b0b3f1b 100644 --- a/arch/arm64/boot/dts/renesas/r9a07g044l2-smarc.dts +++ b/arch/arm64/boot/dts/renesas/r9a07g044l2-smarc.dts @@ -7,15 +7,10 @@ /dts-v1/; #include "r9a07g044l2.dtsi" +#include "rzg2l-smarc-som.dtsi" #include "rzg2l-smarc.dtsi" / { model = "Renesas SMARC EVK based on r9a07g044l2"; compatible = "renesas,smarc-evk", "renesas,r9a07g044l2", "renesas,r9a07g044"; - - memory@48000000 { - device_type = "memory"; - /* first 128MB is reserved for secure area. */ - reg = <0x0 0x48000000 0x0 0x78000000>; - }; }; diff --git a/arch/arm64/boot/dts/renesas/rzg2l-smarc.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-smarc.dtsi index adcd4f5051..6f2a8bdfa2 100644 --- a/arch/arm64/boot/dts/renesas/rzg2l-smarc.dtsi +++ b/arch/arm64/boot/dts/renesas/rzg2l-smarc.dtsi @@ -6,22 +6,350 @@ */ #include +#include + +/* + * SSI-WM8978 + * + * This command is required when Playback/Capture + * + * amixer cset name='Left Input Mixer L2 Switch' on + * amixer cset name='Right Input Mixer R2 Switch' on + * amixer cset name='Headphone Playback Volume' 100 + * amixer cset name='PCM Volume' 100% + * amixer cset name='Input PGA Volume' 25 + * + */ + +/* comment the #define statement to disable SCIF2 (SER0) on PMOD1 (CN7) */ +#define PMOD1_SER0 1 / { aliases { serial0 = &scif0; + serial1 = &scif2; + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c3 = &i2c3; }; chosen { - bootargs = "ignore_loglevel"; stdout-path = "serial0:115200n8"; }; + + audio_mclock: audio_mclock { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <11289600>; + }; + + snd_rzg2l: sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,bitclock-master = <&cpu_dai>; + simple-audio-card,frame-master = <&cpu_dai>; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,widgets = "Microphone", "Microphone Jack"; + simple-audio-card,routing = + "L2", "Mic Bias", + "R2", "Mic Bias", + "Mic Bias", "Microphone Jack"; + + cpu_dai: simple-audio-card,cpu { + sound-dai = <&ssi0>; + }; + + codec_dai: simple-audio-card,codec { + clocks = <&audio_mclock>; + sound-dai = <&wm8978>; + }; + }; + + usb0_vbus_otg: regulator-usb0-vbus-otg { + compatible = "regulator-fixed"; + + regulator-name = "USB0_VBUS_OTG"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vccq_sdhi1: regulator-vccq-sdhi1 { + compatible = "regulator-gpio"; + regulator-name = "SDHI1 VccQ"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + gpios = <&pinctrl RZG2L_GPIO(39, 1) GPIO_ACTIVE_HIGH>; + gpios-states = <1>; + states = <3300000 1>, <1800000 0>; + }; }; -&extal_clk { - clock-frequency = <24000000>; +&audio_clk1{ + clock-frequency = <11289600>; +}; + +&audio_clk2{ + clock-frequency = <12288000>; +}; + +&canfd { + pinctrl-0 = <&can0_pins &can1_pins>; + pinctrl-names = "default"; + status = "okay"; + + channel0 { + status = "okay"; + }; + + channel1 { + status = "okay"; + }; +}; + +&ehci0 { + dr_mode = "otg"; + status = "okay"; +}; + +&ehci1 { + status = "okay"; +}; + +&hsusb { + dr_mode = "otg"; + status = "okay"; +}; + +&i2c0 { + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pins>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&i2c3 { + pinctrl-0 = <&i2c3_pins>; + pinctrl-names = "default"; + clock-frequency = <400000>; + + status = "okay"; + + wm8978: codec@1a { + compatible = "wlf,wm8978"; + #sound-dai-cells = <0>; + reg = <0x1a>; + }; +}; + +&ohci0 { + dr_mode = "otg"; + status = "okay"; +}; + +&ohci1 { + status = "okay"; +}; + +&phyrst { + status = "okay"; +}; + +&pinctrl { + pinctrl-0 = <&sound_clk_pins>; + pinctrl-names = "default"; + + can0_pins: can0 { + pinmux = , /* TX */ + ; /* RX */ + }; + + /* SW7 should be at position 2->3 so that GPIO8_CAN0_STB line is activated */ + can0-stb { + gpio-hog; + gpios = ; + output-low; + line-name = "can0_stb"; + }; + + can1_pins: can1 { + pinmux = , /* TX */ + ; /* RX */ + }; + + /* SW8 should be at position 2->3 so that GPIO9_CAN1_STB line is activated */ + can1-stb { + gpio-hog; + gpios = ; + output-low; + line-name = "can1_stb"; + }; + + i2c0_pins: i2c0 { + pins = "RIIC0_SDA", "RIIC0_SCL"; + input-enable; + }; + + i2c1_pins: i2c1 { + pins = "RIIC1_SDA", "RIIC1_SCL"; + input-enable; + }; + + i2c3_pins: i2c3 { + pinmux = , /* SDA */ + ; /* SCL */ + }; + + scif0_pins: scif0 { + pinmux = , /* TxD */ + ; /* RxD */ + }; + + scif2_pins: scif2 { + pinmux = , /* TxD */ + , /* RxD */ + , /* CTS# */ + ; /* RTS# */ + }; + + sd1-pwr-en-hog { + gpio-hog; + gpios = ; + output-high; + line-name = "sd1_pwr_en"; + }; + + sdhi1_pins: sd1 { + sd1_data { + pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3"; + power-source = <3300>; + }; + + sd1_ctrl { + pins = "SD1_CLK", "SD1_CMD"; + power-source = <3300>; + }; + + sd1_mux { + pinmux = ; /* SD1_CD */ + }; + }; + + sdhi1_pins_uhs: sd1_uhs { + sd1_data_uhs { + pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3"; + power-source = <1800>; + }; + + sd1_ctrl_uhs { + pins = "SD1_CLK", "SD1_CMD"; + power-source = <1800>; + }; + + sd1_mux_uhs { + pinmux = ; /* SD1_CD */ + }; + }; + + sound_clk_pins: sound_clk { + pins = "AUDIO_CLK1", "AUDIO_CLK2"; + input-enable; + }; + + spi1_pins: spi1 { + pinmux = , /* CK */ + , /* MOSI */ + , /* MISO */ + ; /* SSL */ + }; + + ssi0_pins: ssi0 { + pinmux = , /* BCK */ + , /* RCK */ + , /* TXD */ + ; /* RXD */ + }; + + usb0_pins: usb0 { + pinmux = , /* VBUS */ + , /* OVC */ + ; /* OTG_ID */ + }; + + usb1_pins: usb1 { + pinmux = , /* VBUS */ + ; /* OVC */ + }; }; &scif0 { + pinctrl-0 = <&scif0_pins>; + pinctrl-names = "default"; + status = "okay"; +}; + +/* + * To enable SCIF2 (SER0) on PMOD1 (CN7) + * SW1 should be at position 2->3 so that SER0_CTS# line is activated + * SW2 should be at position 2->3 so that SER0_TX line is activated + * SW3 should be at position 2->3 so that SER0_RX line is activated + * SW4 should be at position 2->3 so that SER0_RTS# line is activated + */ +#if PMOD1_SER0 +&scif2 { + pinctrl-0 = <&scif2_pins>; + pinctrl-names = "default"; + + uart-has-rtscts; + status = "okay"; +}; +#endif + +&sdhi1 { + pinctrl-0 = <&sdhi1_pins>; + pinctrl-1 = <&sdhi1_pins_uhs>; + pinctrl-names = "default", "state_uhs"; + + vmmc-supply = <®_3p3v>; + vqmmc-supply = <&vccq_sdhi1>; + bus-width = <4>; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; + +&spi1 { + pinctrl-0 = <&spi1_pins>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&ssi0 { + pinctrl-0 = <&ssi0_pins>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&usb2_phy0 { + pinctrl-0 = <&usb0_pins>; + pinctrl-names = "default"; + + vbus-supply = <&usb0_vbus_otg>; + status = "okay"; +}; + +&usb2_phy1 { + pinctrl-0 = <&usb1_pins>; + pinctrl-names = "default"; + status = "okay"; }; diff --git a/arch/arm64/boot/dts/renesas/salvator-common.dtsi b/arch/arm64/boot/dts/renesas/salvator-common.dtsi index eb1f3b8230..6092dc4531 100644 --- a/arch/arm64/boot/dts/renesas/salvator-common.dtsi +++ b/arch/arm64/boot/dts/renesas/salvator-common.dtsi @@ -97,6 +97,7 @@ hdmi0-out { port { hdmi0_con: endpoint { + remote-endpoint = <&rcar_dw_hdmi0_out>; }; }; }; @@ -340,6 +341,8 @@ &avb { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id0022.1622", + "ethernet-phy-ieee802.3-c22"; rxc-skew-ps = <1500>; reg = <0>; interrupt-parent = <&gpio2>; @@ -422,10 +425,6 @@ dw_hdmi0_snd_in: endpoint { }; }; -&hdmi0_con { - remote-endpoint = <&rcar_dw_hdmi0_out>; -}; - #ifdef SOC_HAS_HDMI1 &hdmi1 { status = "okay"; @@ -524,47 +523,49 @@ video-receiver@70 { reg-names = "main", "dpll", "cp", "hdmi", "edid", "repeater", "infoframe", "cbus", "cec", "sdp", "txa", "txb" ; - #address-cells = <1>; - #size-cells = <0>; - interrupt-parent = <&gpio6>; interrupt-names = "intrq1", "intrq2"; interrupts = <30 IRQ_TYPE_LEVEL_LOW>, <31 IRQ_TYPE_LEVEL_LOW>; - port@7 { - reg = <7>; + ports { + #address-cells = <1>; + #size-cells = <0>; - adv7482_ain7: endpoint { - remote-endpoint = <&cvbs_con>; + port@7 { + reg = <7>; + + adv7482_ain7: endpoint { + remote-endpoint = <&cvbs_con>; + }; }; - }; - port@8 { - reg = <8>; + port@8 { + reg = <8>; - adv7482_hdmi: endpoint { - remote-endpoint = <&hdmi_in_con>; + adv7482_hdmi: endpoint { + remote-endpoint = <&hdmi_in_con>; + }; }; - }; - port@a { - reg = <10>; + port@a { + reg = <10>; - adv7482_txa: endpoint { - clock-lanes = <0>; - data-lanes = <1 2 3 4>; - remote-endpoint = <&csi40_in>; + adv7482_txa: endpoint { + clock-lanes = <0>; + data-lanes = <1 2 3 4>; + remote-endpoint = <&csi40_in>; + }; }; - }; - port@b { - reg = <11>; + port@b { + reg = <11>; - adv7482_txb: endpoint { - clock-lanes = <0>; - data-lanes = <1>; - remote-endpoint = <&csi20_in>; + adv7482_txb: endpoint { + clock-lanes = <0>; + data-lanes = <1>; + remote-endpoint = <&csi20_in>; + }; }; }; }; diff --git a/arch/arm64/boot/dts/renesas/ulcb.dtsi b/arch/arm64/boot/dts/renesas/ulcb.dtsi index 1f177af3eb..a7e93df4ce 100644 --- a/arch/arm64/boot/dts/renesas/ulcb.dtsi +++ b/arch/arm64/boot/dts/renesas/ulcb.dtsi @@ -48,6 +48,7 @@ hdmi0-out { port { hdmi0_con: endpoint { + remote-endpoint = <&rcar_dw_hdmi0_out>; }; }; }; @@ -154,6 +155,8 @@ &avb { status = "okay"; phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id0022.1622", + "ethernet-phy-ieee802.3-c22"; rxc-skew-ps = <1500>; reg = <0>; interrupt-parent = <&gpio2>; @@ -197,10 +200,6 @@ dw_hdmi0_snd_in: endpoint { }; }; -&hdmi0_con { - remote-endpoint = <&rcar_dw_hdmi0_out>; -}; - &i2c2 { pinctrl-0 = <&i2c2_pins>; pinctrl-names = "default"; diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile index 7fdb41de01..479906f3ad 100644 --- a/arch/arm64/boot/dts/rockchip/Makefile +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -13,6 +13,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3328-nanopi-r2s.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3328-rock64.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3328-rock-pi-e.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3328-roc-cc.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3328-roc-pc.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-evb-act8846.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-geekbox.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-lion-haikou.dtb @@ -24,6 +25,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-ficus.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-firefly.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-gru-bob.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-gru-kevin.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-gru-scarlet-dumo.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-gru-scarlet-inx.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-gru-scarlet-kd.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-hugsun-x99.dtb @@ -42,8 +44,11 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-pinebook-pro.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-puma-haikou.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-roc-pc.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-roc-pc-mezzanine.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-roc-pc-plus.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4a.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4a-plus.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4b.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4b-plus.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4c.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock960.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rockpro64-v2.dtb @@ -51,4 +56,5 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rockpro64.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire-excavator.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399pro-rock-pi-n10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-quartz64-a.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb1-v10.dtb diff --git a/arch/arm64/boot/dts/rockchip/px30-evb.dts b/arch/arm64/boot/dts/rockchip/px30-evb.dts index c1ce9c295e..848bc39cf8 100644 --- a/arch/arm64/boot/dts/rockchip/px30-evb.dts +++ b/arch/arm64/boot/dts/rockchip/px30-evb.dts @@ -114,6 +114,10 @@ &cpu3 { cpu-supply = <&vdd_arm>; }; +&csi_dphy { + status = "okay"; +}; + &display_subsystem { status = "okay"; }; @@ -428,6 +432,36 @@ sensor@4c { }; }; +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + avdd-supply = <&vcc2v8_dvp>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + dvdd-supply = <&vcc1v5_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0>; + reset-gpios = <&gpio2 14 GPIO_ACTIVE_LOW>; + + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1 2>; + }; + }; + }; +}; + &i2s1_2ch { status = "okay"; }; @@ -443,6 +477,24 @@ &io_domains { vccio6-supply = <&vccio_flash>; }; +&isp { + status = "okay"; + + ports { + port@0 { + mipi_in_ucam: endpoint@0 { + reg = <0>; + data-lanes = <1 2>; + remote-endpoint = <&ucam_out>; + }; + }; + }; +}; + +&isp_mmu { + status = "okay"; +}; + &pinctrl { headphone { hp_det: hp-det { diff --git a/arch/arm64/boot/dts/rockchip/px30.dtsi b/arch/arm64/boot/dts/rockchip/px30.dtsi index 5200d0bbd9..56dfbb2e2f 100644 --- a/arch/arm64/boot/dts/rockchip/px30.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30.dtsi @@ -110,7 +110,7 @@ CLUSTER_SLEEP: cluster-sleep { }; }; - cpu0_opp_table: cpu0-opp-table { + cpu0_opp_table: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -864,6 +864,19 @@ dsi_dphy: phy@ff2e0000 { status = "disabled"; }; + csi_dphy: phy@ff2f0000 { + compatible = "rockchip,px30-csi-dphy"; + reg = <0x0 0xff2f0000 0x0 0x4000>; + clocks = <&cru PCLK_MIPICSIPHY>; + clock-names = "pclk"; + #phy-cells = <0>; + power-domains = <&power PX30_PD_VI>; + resets = <&cru SRST_MIPICSIPHY_P>; + reset-names = "apb"; + rockchip,grf = <&grf>; + status = "disabled"; + }; + usb20_otg: usb@ff300000 { compatible = "rockchip,px30-usb", "rockchip,rk3066-usb", "snps,dwc2"; @@ -974,6 +987,18 @@ emmc: mmc@ff390000 { status = "disabled"; }; + sfc: spi@ff3a0000 { + compatible = "rockchip,sfc"; + reg = <0x0 0xff3a0000 0x0 0x4000>; + interrupts = ; + clocks = <&cru SCLK_SFC>, <&cru HCLK_SFC>; + clock-names = "clk_sfc", "hclk_sfc"; + pinctrl-0 = <&sfc_clk &sfc_cs0 &sfc_bus4>; + pinctrl-names = "default"; + power-domains = <&power PX30_PD_MMC_NAND>; + status = "disabled"; + }; + nfc: nand-controller@ff3b0000 { compatible = "rockchip,px30-nfc"; reg = <0x0 0xff3b0000 0x0 0x4000>; @@ -989,7 +1014,7 @@ nfc: nand-controller@ff3b0000 { status = "disabled"; }; - gpu_opp_table: opp-table2 { + gpu_opp_table: opp-table-1 { compatible = "operating-points-v2"; opp-200000000 { @@ -1024,8 +1049,30 @@ gpu: gpu@ff400000 { status = "disabled"; }; + vpu: video-codec@ff442000 { + compatible = "rockchip,px30-vpu"; + reg = <0x0 0xff442000 0x0 0x800>; + interrupts = , + ; + interrupt-names = "vepu", "vdpu"; + clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; + clock-names = "aclk", "hclk"; + iommus = <&vpu_mmu>; + power-domains = <&power PX30_PD_VPU>; + }; + + vpu_mmu: iommu@ff442800 { + compatible = "rockchip,iommu"; + reg = <0x0 0xff442800 0x0 0x100>; + interrupts = ; + clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; + clock-names = "aclk", "iface"; + #iommu-cells = <0>; + power-domains = <&power PX30_PD_VPU>; + }; + dsi: dsi@ff450000 { - compatible = "rockchip,px30-mipi-dsi"; + compatible = "rockchip,px30-mipi-dsi", "snps,dw-mipi-dsi"; reg = <0x0 0xff450000 0x0 0x10000>; interrupts = ; clocks = <&cru PCLK_MIPI_DSI>; @@ -1142,6 +1189,47 @@ vopl_mmu: iommu@ff470f00 { status = "disabled"; }; + isp: isp@ff4a0000 { + compatible = "rockchip,px30-cif-isp"; /*rk3326-rkisp1*/ + reg = <0x0 0xff4a0000 0x0 0x8000>; + interrupts = , + , + ; + interrupt-names = "isp", "mi", "mipi"; + clocks = <&cru SCLK_ISP>, + <&cru ACLK_ISP>, + <&cru HCLK_ISP>, + <&cru PCLK_ISP>; + clock-names = "isp", "aclk", "hclk", "pclk"; + iommus = <&isp_mmu>; + phys = <&csi_dphy>; + phy-names = "dphy"; + power-domains = <&power PX30_PD_VI>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + }; + + isp_mmu: iommu@ff4a8000 { + compatible = "rockchip,iommu"; + reg = <0x0 0xff4a8000 0x0 0x100>; + interrupts = ; + clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>; + clock-names = "aclk", "iface"; + power-domains = <&power PX30_PD_VI>; + rockchip,disable-mmu-reset; + #iommu-cells = <0>; + }; + qos_gmac: qos@ff518000 { compatible = "rockchip,px30-qos", "syscon"; reg = <0x0 0xff518000 0x0 0x20>; @@ -1250,7 +1338,7 @@ pinctrl: pinctrl { #size-cells = <2>; ranges; - gpio0: gpio0@ff040000 { + gpio0: gpio@ff040000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff040000 0x0 0x100>; interrupts = ; @@ -1262,7 +1350,7 @@ gpio0: gpio0@ff040000 { #interrupt-cells = <2>; }; - gpio1: gpio1@ff250000 { + gpio1: gpio@ff250000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff250000 0x0 0x100>; interrupts = ; @@ -1274,7 +1362,7 @@ gpio1: gpio1@ff250000 { #interrupt-cells = <2>; }; - gpio2: gpio2@ff260000 { + gpio2: gpio@ff260000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff260000 0x0 0x100>; interrupts = ; @@ -1286,7 +1374,7 @@ gpio2: gpio2@ff260000 { #interrupt-cells = <2>; }; - gpio3: gpio3@ff270000 { + gpio3: gpio@ff270000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff270000 0x0 0x100>; interrupts = ; @@ -1973,6 +2061,32 @@ flash_bus8: flash-bus8 { }; }; + sfc { + sfc_bus4: sfc-bus4 { + rockchip,pins = + <1 RK_PA0 3 &pcfg_pull_none>, + <1 RK_PA1 3 &pcfg_pull_none>, + <1 RK_PA2 3 &pcfg_pull_none>, + <1 RK_PA3 3 &pcfg_pull_none>; + }; + + sfc_bus2: sfc-bus2 { + rockchip,pins = + <1 RK_PA0 3 &pcfg_pull_none>, + <1 RK_PA1 3 &pcfg_pull_none>; + }; + + sfc_cs0: sfc-cs0 { + rockchip,pins = + <1 RK_PA4 3 &pcfg_pull_none>; + }; + + sfc_clk: sfc-clk { + rockchip,pins = + <1 RK_PB1 3 &pcfg_pull_none>; + }; + }; + lcdc { lcdc_rgb_dclk_pin: lcdc-rgb-dclk-pin { rockchip,pins = diff --git a/arch/arm64/boot/dts/rockchip/rk3308.dtsi b/arch/arm64/boot/dts/rockchip/rk3308.dtsi index a185901aba..1cbe212618 100644 --- a/arch/arm64/boot/dts/rockchip/rk3308.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3308.dtsi @@ -99,7 +99,7 @@ l2: l2-cache { }; }; - cpu0_opp_table: cpu0-opp-table { + cpu0_opp_table: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -731,6 +731,17 @@ gmac: ethernet@ff4e0000 { status = "disabled"; }; + sfc: spi@ff4c0000 { + compatible = "rockchip,sfc"; + reg = <0x0 0xff4c0000 0x0 0x4000>; + interrupts = ; + clocks = <&cru SCLK_SFC>, <&cru HCLK_SFC>; + clock-names = "clk_sfc", "hclk_sfc"; + pinctrl-0 = <&sfc_clk &sfc_cs0 &sfc_bus4>; + pinctrl-names = "default"; + status = "disabled"; + }; + cru: clock-controller@ff500000 { compatible = "rockchip,rk3308-cru"; reg = <0x0 0xff500000 0x0 0x1000>; @@ -779,7 +790,7 @@ pinctrl: pinctrl { #size-cells = <2>; ranges; - gpio0: gpio0@ff220000 { + gpio0: gpio@ff220000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff220000 0x0 0x100>; interrupts = ; @@ -790,7 +801,7 @@ gpio0: gpio0@ff220000 { #interrupt-cells = <2>; }; - gpio1: gpio1@ff230000 { + gpio1: gpio@ff230000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff230000 0x0 0x100>; interrupts = ; @@ -801,7 +812,7 @@ gpio1: gpio1@ff230000 { #interrupt-cells = <2>; }; - gpio2: gpio2@ff240000 { + gpio2: gpio@ff240000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff240000 0x0 0x100>; interrupts = ; @@ -812,7 +823,7 @@ gpio2: gpio2@ff240000 { #interrupt-cells = <2>; }; - gpio3: gpio3@ff250000 { + gpio3: gpio@ff250000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff250000 0x0 0x100>; interrupts = ; @@ -823,7 +834,7 @@ gpio3: gpio3@ff250000 { #interrupt-cells = <2>; }; - gpio4: gpio4@ff260000 { + gpio4: gpio@ff260000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff260000 0x0 0x100>; interrupts = ; @@ -1004,6 +1015,32 @@ flash_bus8: flash-bus8 { }; }; + sfc { + sfc_bus4: sfc-bus4 { + rockchip,pins = + <3 RK_PA0 3 &pcfg_pull_none>, + <3 RK_PA1 3 &pcfg_pull_none>, + <3 RK_PA2 3 &pcfg_pull_none>, + <3 RK_PA3 3 &pcfg_pull_none>; + }; + + sfc_bus2: sfc-bus2 { + rockchip,pins = + <3 RK_PA0 3 &pcfg_pull_none>, + <3 RK_PA1 3 &pcfg_pull_none>; + }; + + sfc_cs0: sfc-cs0 { + rockchip,pins = + <3 RK_PA4 3 &pcfg_pull_none>; + }; + + sfc_clk: sfc-clk { + rockchip,pins = + <3 RK_PA5 3 &pcfg_pull_none>; + }; + }; + gmac { rmii_pins: rmii-pins { rockchip,pins = diff --git a/arch/arm64/boot/dts/rockchip/rk3318-a95x-z2.dts b/arch/arm64/boot/dts/rockchip/rk3318-a95x-z2.dts index 763cf9b462..43c928ac98 100644 --- a/arch/arm64/boot/dts/rockchip/rk3318-a95x-z2.dts +++ b/arch/arm64/boot/dts/rockchip/rk3318-a95x-z2.dts @@ -185,7 +185,6 @@ &gmac2phy { assigned-clock-parents = <&cru SCLK_MAC2PHY_SRC>; assigned-clock-rate = <50000000>; assigned-clocks = <&cru SCLK_MAC2PHY>; - clock_in_out = "output"; status = "okay"; }; @@ -194,8 +193,6 @@ &gpu { }; &hdmi { - ddc-i2c-scl-high-time-ns = <9625>; - ddc-i2c-scl-low-time-ns = <10000>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts b/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts index 7fc674a99a..ea0695b51e 100644 --- a/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts +++ b/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts @@ -207,7 +207,8 @@ vcc_host: vcc_host { gpio = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; enable-active-high; regulator-always-on; - vin-supply = <&vccsys>; + regulator-boot-on; + vin-supply = <&usb_midu>; }; }; @@ -311,6 +312,7 @@ rk817: pmic@20 { vcc5-supply = <&vccsys>; vcc6-supply = <&vccsys>; vcc7-supply = <&vccsys>; + vcc8-supply = <&vccsys>; regulators { vdd_logic: DCDC_REG1 { @@ -460,6 +462,14 @@ regulator-state-mem { regulator-suspend-microvolt = <3000000>; }; }; + + usb_midu: BOOST { + regulator-name = "usb_midu"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5400000>; + regulator-always-on; + regulator-boot-on; + }; }; rk817_codec: codec { @@ -517,6 +527,22 @@ &sdmmc { status = "okay"; }; +&sfc { + pinctrl-0 = <&sfc_clk &sfc_cs0 &sfc_bus2>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <108000000>; + spi-rx-bus-width = <2>; + spi-tx-bus-width = <1>; + }; +}; + &tsadc { status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts b/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts index 1b0f7e4551..f69a38f42d 100644 --- a/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts +++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts @@ -345,7 +345,7 @@ &spdif { &spi0 { status = "okay"; - spiflash@0 { + flash@0 { compatible = "jedec,spi-nor"; reg = <0>; diff --git a/arch/arm64/boot/dts/rockchip/rk3328.dtsi b/arch/arm64/boot/dts/rockchip/rk3328.dtsi index 3cbe83e6fb..b822533dc7 100644 --- a/arch/arm64/boot/dts/rockchip/rk3328.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3328.dtsi @@ -105,7 +105,7 @@ l2: l2-cache0 { }; }; - cpu0_opp_table: opp_table0 { + cpu0_opp_table: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -623,7 +623,6 @@ h265e_mmu: iommu@ff330200 { compatible = "rockchip,iommu"; reg = <0x0 0xff330200 0 0x100>; interrupts = ; - interrupt-names = "h265e_mmu"; clocks = <&cru ACLK_H265>, <&cru PCLK_H265>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -634,7 +633,6 @@ vepu_mmu: iommu@ff340800 { compatible = "rockchip,iommu"; reg = <0x0 0xff340800 0x0 0x40>; interrupts = ; - interrupt-names = "vepu_mmu"; clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -656,7 +654,6 @@ vpu_mmu: iommu@ff350800 { compatible = "rockchip,iommu"; reg = <0x0 0xff350800 0x0 0x40>; interrupts = ; - interrupt-names = "vpu_mmu"; clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -667,7 +664,6 @@ rkvdec_mmu: iommu@ff360480 { compatible = "rockchip,iommu"; reg = <0x0 0xff360480 0x0 0x40>, <0x0 0xff3604c0 0x0 0x40>; interrupts = ; - interrupt-names = "rkvdec_mmu"; clocks = <&cru ACLK_RKVDEC>, <&cru HCLK_RKVDEC>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -700,7 +696,6 @@ vop_mmu: iommu@ff373f00 { compatible = "rockchip,iommu"; reg = <0x0 0xff373f00 0x0 0x100>; interrupts = ; - interrupt-names = "vop_mmu"; clocks = <&cru ACLK_VOP>, <&cru HCLK_VOP>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -1019,7 +1014,7 @@ pinctrl: pinctrl { #size-cells = <2>; ranges; - gpio0: gpio0@ff210000 { + gpio0: gpio@ff210000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff210000 0x0 0x100>; interrupts = ; @@ -1032,7 +1027,7 @@ gpio0: gpio0@ff210000 { #interrupt-cells = <2>; }; - gpio1: gpio1@ff220000 { + gpio1: gpio@ff220000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff220000 0x0 0x100>; interrupts = ; @@ -1045,7 +1040,7 @@ gpio1: gpio1@ff220000 { #interrupt-cells = <2>; }; - gpio2: gpio2@ff230000 { + gpio2: gpio@ff230000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff230000 0x0 0x100>; interrupts = ; @@ -1058,7 +1053,7 @@ gpio2: gpio2@ff230000 { #interrupt-cells = <2>; }; - gpio3: gpio3@ff240000 { + gpio3: gpio@ff240000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff240000 0x0 0x100>; interrupts = ; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-lion.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-lion.dtsi index bcd7977fb0..5753e57fd7 100644 --- a/arch/arm64/boot/dts/rockchip/rk3368-lion.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3368-lion.dtsi @@ -29,15 +29,15 @@ i2cmux1 { i2c-parent = <&i2c1>; mux-gpios = <&gpio1 RK_PA7 GPIO_ACTIVE_HIGH>; - /* Q7_GPO_I2C */ - i2c@0 { + /* Q7_GP0_I2C */ + i2c_gp0: i2c@0 { reg = <0>; #address-cells = <1>; #size-cells = <0>; }; /* Q7_SMB */ - i2c@1 { + i2c_smb: i2c@1 { reg = <1>; #address-cells = <1>; #size-cells = <0>; @@ -52,7 +52,7 @@ i2cmux2 { mux-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; /* Q7_LVDS_BLC_I2C */ - i2c@0 { + i2c_lvds_blc: i2c@0 { reg = <0>; #address-cells = <1>; #size-cells = <0>; @@ -69,8 +69,8 @@ rtc_twi: rtc@6f { }; }; - /* Q7_GP2_I2C */ - i2c@1 { + /* Q7_GP2_I2C = LVDS_DID_CLK/DAT */ + i2c_gp2: i2c@1 { reg = <1>; #address-cells = <1>; #size-cells = <0>; @@ -144,7 +144,7 @@ &emmc { mmc-hs200-1_8v; non-removable; vmmc-supply = <&vcc33_io>; - vqmmc-supply = <&vcc18_io>; + vqmmc-supply = <&vcc_18>; pinctrl-names = "default"; pinctrl-0 = <&emmc_clk>, <&emmc_cmd>, <&emmc_bus8>; status = "okay"; @@ -238,13 +238,6 @@ vdd10_pll: LDO_REG3 { regulator-boot-on; }; - vcc18_io: LDO_REG4 { - regulator-name = "vcc18_io"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-boot-on; - }; - vdd10_video: LDO_REG6 { regulator-name = "vdd10_video"; regulator-min-microvolt = <1000000>; @@ -253,6 +246,14 @@ vdd10_video: LDO_REG6 { regulator-boot-on; }; + vcc_18: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_18"; + }; + vcc18_video: LDO_REG8 { regulator-name = "vcc18_video"; regulator-min-microvolt = <1800000>; @@ -272,6 +273,18 @@ &i2c2 { status = "okay"; }; +/* The RK3368-uQ7 "Lion" has most IO voltages hardwired to 3.3V. */ +&io_domains { + audio-supply = <&vcc33_io>; + dvp-supply = <&vcc33_io>; + flash0-supply = <&vcc_18>; + gpio30-supply = <&vcc33_io>; + gpio1830-supply = <&vcc33_io>; + sdcard-supply = <&vcc33_io>; + wifi-supply = <&vcc33_io>; + status = "okay"; +}; + &pinctrl { leds { module_led_pins: module-led-pins { @@ -291,6 +304,12 @@ pmic_sleep: pmic-sleep { }; }; +&pmu_io_domains { + pmu-supply = <&vcc33_io>; + vop-supply = <&vcc33_io>; + status = "okay"; +}; + &spi1 { status = "okay"; diff --git a/arch/arm64/boot/dts/rockchip/rk3368.dtsi b/arch/arm64/boot/dts/rockchip/rk3368.dtsi index 4c64fbefb4..c99da90328 100644 --- a/arch/arm64/boot/dts/rockchip/rk3368.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3368.dtsi @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -615,6 +616,115 @@ mbox: mbox@ff6b0000 { status = "disabled"; }; + pmu: power-management@ff730000 { + compatible = "rockchip,rk3368-pmu", "syscon", "simple-mfd"; + reg = <0x0 0xff730000 0x0 0x1000>; + + power: power-controller { + compatible = "rockchip,rk3368-power-controller"; + #power-domain-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + + /* + * Note: Although SCLK_* are the working clocks + * of device without including on the NOC, needed for + * synchronous reset. + * + * The clocks on the which NOC: + * ACLK_IEP/ACLK_VIP/ACLK_VOP0 are on ACLK_VIO0_NIU. + * ACLK_ISP/ACLK_VOP1 are on ACLK_VIO1_NIU. + * ACLK_RGA is on ACLK_RGA_NIU. + * The others (HCLK_*,PLCK_*) are on HCLK_VIO_NIU. + * + * Which clock are device clocks: + * clocks devices + * *_IEP IEP:Image Enhancement Processor + * *_ISP ISP:Image Signal Processing + * *_VIP VIP:Video Input Processor + * *_VOP* VOP:Visual Output Processor + * *_RGA RGA + * *_EDP* EDP + * *_DPHY* LVDS + * *_HDMI HDMI + * *_MIPI_* MIPI + */ + power-domain@RK3368_PD_VIO { + reg = ; + clocks = <&cru ACLK_IEP>, + <&cru ACLK_ISP>, + <&cru ACLK_VIP>, + <&cru ACLK_RGA>, + <&cru ACLK_VOP>, + <&cru ACLK_VOP_IEP>, + <&cru DCLK_VOP>, + <&cru HCLK_IEP>, + <&cru HCLK_ISP>, + <&cru HCLK_RGA>, + <&cru HCLK_VIP>, + <&cru HCLK_VOP>, + <&cru HCLK_VIO_HDCPMMU>, + <&cru PCLK_EDP_CTRL>, + <&cru PCLK_HDMI_CTRL>, + <&cru PCLK_HDCP>, + <&cru PCLK_ISP>, + <&cru PCLK_VIP>, + <&cru PCLK_DPHYRX>, + <&cru PCLK_DPHYTX0>, + <&cru PCLK_MIPI_CSI>, + <&cru PCLK_MIPI_DSI0>, + <&cru SCLK_VOP0_PWM>, + <&cru SCLK_EDP_24M>, + <&cru SCLK_EDP>, + <&cru SCLK_HDCP>, + <&cru SCLK_ISP>, + <&cru SCLK_RGA>, + <&cru SCLK_HDMI_CEC>, + <&cru SCLK_HDMI_HDCP>; + pm_qos = <&qos_iep>, + <&qos_isp_r0>, + <&qos_isp_r1>, + <&qos_isp_w0>, + <&qos_isp_w1>, + <&qos_vip>, + <&qos_vop>, + <&qos_rga_r>, + <&qos_rga_w>; + #power-domain-cells = <0>; + }; + + /* + * Note: ACLK_VCODEC/HCLK_VCODEC are VCODEC + * (video endecoder & decoder) clocks that on the + * ACLK_VCODEC_NIU and HCLK_VCODEC_NIU (NOC). + */ + power-domain@RK3368_PD_VIDEO { + reg = ; + clocks = <&cru ACLK_VIDEO>, + <&cru HCLK_VIDEO>, + <&cru SCLK_HEVC_CABAC>, + <&cru SCLK_HEVC_CORE>; + pm_qos = <&qos_hevc_r>, + <&qos_vpu_r>, + <&qos_vpu_w>; + #power-domain-cells = <0>; + }; + + /* + * Note: ACLK_GPU is the GPU clock, + * and on the ACLK_GPU_NIU (NOC). + */ + power-domain@RK3368_PD_GPU_1 { + reg = ; + clocks = <&cru ACLK_GPU_CFG>, + <&cru ACLK_GPU_MEM>, + <&cru SCLK_GPU_CORE>; + pm_qos = <&qos_gpu>; + #power-domain-cells = <0>; + }; + }; + }; + pmugrf: syscon@ff738000 { compatible = "rockchip,rk3368-pmugrf", "syscon", "simple-mfd"; reg = <0x0 0xff738000 0x0 0x1000>; @@ -709,9 +819,9 @@ iep_mmu: iommu@ff900800 { compatible = "rockchip,iommu"; reg = <0x0 0xff900800 0x0 0x100>; interrupts = ; - interrupt-names = "iep_mmu"; clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>; clock-names = "aclk", "iface"; + power-domains = <&power RK3368_PD_VIO>; #iommu-cells = <0>; status = "disabled"; }; @@ -721,10 +831,10 @@ isp_mmu: iommu@ff914000 { reg = <0x0 0xff914000 0x0 0x100>, <0x0 0xff915000 0x0 0x100>; interrupts = ; - interrupt-names = "isp_mmu"; clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>; clock-names = "aclk", "iface"; #iommu-cells = <0>; + power-domains = <&power RK3368_PD_VIO>; rockchip,disable-mmu-reset; status = "disabled"; }; @@ -733,9 +843,9 @@ vop_mmu: iommu@ff930300 { compatible = "rockchip,iommu"; reg = <0x0 0xff930300 0x0 0x100>; interrupts = ; - interrupt-names = "vop_mmu"; clocks = <&cru ACLK_VOP>, <&cru HCLK_VOP>; clock-names = "aclk", "iface"; + power-domains = <&power RK3368_PD_VIO>; #iommu-cells = <0>; status = "disabled"; }; @@ -745,7 +855,6 @@ hevc_mmu: iommu@ff9a0440 { reg = <0x0 0xff9a0440 0x0 0x40>, <0x0 0xff9a0480 0x0 0x40>; interrupts = ; - interrupt-names = "hevc_mmu"; clocks = <&cru ACLK_VIDEO>, <&cru HCLK_VIDEO>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -757,13 +866,77 @@ vpu_mmu: iommu@ff9a0800 { reg = <0x0 0xff9a0800 0x0 0x100>; interrupts = , ; - interrupt-names = "vepu_mmu", "vdpu_mmu"; clocks = <&cru ACLK_VIDEO>, <&cru HCLK_VIDEO>; clock-names = "aclk", "iface"; #iommu-cells = <0>; status = "disabled"; }; + qos_iep: qos@ffad0000 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffad0000 0x0 0x20>; + }; + + qos_isp_r0: qos@ffad0080 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffad0080 0x0 0x20>; + }; + + qos_isp_r1: qos@ffad0100 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffad0100 0x0 0x20>; + }; + + qos_isp_w0: qos@ffad0180 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffad0180 0x0 0x20>; + }; + + qos_isp_w1: qos@ffad0200 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffad0200 0x0 0x20>; + }; + + qos_vip: qos@ffad0280 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffad0280 0x0 0x20>; + }; + + qos_vop: qos@ffad0300 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffad0300 0x0 0x20>; + }; + + qos_rga_r: qos@ffad0380 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffad0380 0x0 0x20>; + }; + + qos_rga_w: qos@ffad0400 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffad0400 0x0 0x20>; + }; + + qos_hevc_r: qos@ffae0000 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffae0000 0x0 0x20>; + }; + + qos_vpu_r: qos@ffae0100 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffae0100 0x0 0x20>; + }; + + qos_vpu_w: qos@ffae0180 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffae0180 0x0 0x20>; + }; + + qos_gpu: qos@ffaf0000 { + compatible = "rockchip,rk3368-qos", "syscon"; + reg = <0x0 0xffaf0000 0x0 0x20>; + }; + efuse256: efuse@ffb00000 { compatible = "rockchip,rk3368-efuse"; reg = <0x0 0xffb00000 0x0 0x20>; @@ -802,7 +975,7 @@ pinctrl: pinctrl { #size-cells = <0x2>; ranges; - gpio0: gpio0@ff750000 { + gpio0: gpio@ff750000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff750000 0x0 0x100>; clocks = <&cru PCLK_GPIO0>; @@ -815,7 +988,7 @@ gpio0: gpio0@ff750000 { #interrupt-cells = <0x2>; }; - gpio1: gpio1@ff780000 { + gpio1: gpio@ff780000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff780000 0x0 0x100>; clocks = <&cru PCLK_GPIO1>; @@ -828,7 +1001,7 @@ gpio1: gpio1@ff780000 { #interrupt-cells = <0x2>; }; - gpio2: gpio2@ff790000 { + gpio2: gpio@ff790000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff790000 0x0 0x100>; clocks = <&cru PCLK_GPIO2>; @@ -841,7 +1014,7 @@ gpio2: gpio2@ff790000 { #interrupt-cells = <0x2>; }; - gpio3: gpio3@ff7a0000 { + gpio3: gpio@ff7a0000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff7a0000 0x0 0x100>; clocks = <&cru PCLK_GPIO3>; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-bob.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-bob.dts index e6c1c94c8d..31ebb4e5fd 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-bob.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-bob.dts @@ -16,6 +16,7 @@ / { "google,bob-rev7", "google,bob-rev6", "google,bob-rev5", "google,bob-rev4", "google,bob", "google,gru", "rockchip,rk3399"; + chassis-type = "convertible"; edp_panel: edp-panel { compatible = "boe,nv101wxmn51"; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi index 1384dabbdf..9b2c679f5e 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi @@ -251,6 +251,182 @@ edp_out_panel: endpoint@0 { }; }; +&gpio0 { + gpio-line-names = /* GPIO0 A 0-7 */ + "AP_RTC_CLK_IN", + "EC_AP_INT_L", + "PP1800_AUDIO_EN", + "BT_HOST_WAKE_L", + "WLAN_MODULE_PD_L", + "H1_INT_OD_L", + "CENTERLOGIC_DVS_PWM", + "", + + /* GPIO0 B 0-4 */ + "WIFI_HOST_WAKE_L", + "PMUIO2_33_18_L", + "PP1500_EN", + "AP_EC_WARM_RESET_REQ", + "PP3000_EN"; +}; + +&gpio1 { + gpio-line-names = /* GPIO1 A 0-7 */ + "", + "", + "SPK_PA_EN", + "", + "TRACKPAD_INT_L", + "AP_EC_S3_S0_L", + "AP_EC_OVERTEMP", + "AP_SPI_FLASH_MISO", + + /* GPIO1 B 0-7 */ + "AP_SPI_FLASH_MOSI_R", + "AP_SPI_FLASH_CLK_R", + "AP_SPI_FLASH_CS_L_R", + "WLAN_MODULE_RESET_L", + "WIFI_DISABLE_L", + "MIC_INT", + "", + "AP_I2C_DVS_SDA", + + /* GPIO1 C 0-7 */ + "AP_I2C_DVS_SCL", + "AP_BL_EN", + /* + * AP_FLASH_WP is crossystem ABI. Schematics call it + * AP_FW_WP or CPU1_FW_WP, depending on the variant. + */ + "AP_FLASH_WP", + "LITCPU_DVS_PWM", + "AP_I2C_AUDIO_SDA", + "AP_I2C_AUDIO_SCL", + "", + "HEADSET_INT_L"; +}; + +&gpio2 { + gpio-line-names = /* GPIO2 A 0-7 */ + "", + "", + "SD_IO_PWR_EN", + "", + "", + "", + "", + "", + + /* GPIO2 B 0-7 */ + "", + "", + "", + "", + "", + "", + "", + "", + + /* GPIO2 C 0-7 */ + "", + "", + "", + "", + "AP_SPI_EC_MISO", + "AP_SPI_EC_MOSI", + "AP_SPI_EC_CLK", + "AP_SPI_EC_CS_L", + + /* GPIO2 D 0-4 */ + "BT_DEV_WAKE_L", + "", + "WIFI_PCIE_CLKREQ_L", + "WIFI_PERST_L", + "SD_PWR_3000_1800_L"; +}; + +&gpio3 { + gpio-line-names = /* GPIO3 A 0-7 */ + "", + "", + "", + "", + "AP_SPI_TPM_MISO", + "AP_SPI_TPM_MOSI_R", + "AP_SPI_TPM_CLK_R", + "AP_SPI_TPM_CS_L_R", + + /* GPIO3 B 0-7 */ + "EC_IN_RW", + "", + "AP_I2C_TP_SDA", + "AP_I2C_TP_SCL", + "AP_I2C_TP_PU_EN", + "TOUCH_INT_L", + "", + "", + + /* GPIO3 C 0-7 */ + "", + "", + "", + "", + "", + "", + "", + "", + + /* GPIO3 D 0-7 */ + "I2S0_SCLK", + "I2S0_LRCK_RX", + "I2S0_LRCK_TX", + "I2S0_SDI_0", + "I2S0_SDI_1", + "", + "I2S0_SDO_1", + "I2S0_SDO_0"; +}; + +&gpio4 { + gpio-line-names = /* GPIO4 A 0-7 */ + "I2S_MCLK", + "AP_I2C_MIC_SDA", + "AP_I2C_MIC_SCL", + "", + "", + "", + "", + "", + + /* GPIO4 B 0-7 */ + "", + "", + "", + "", + "", + "", + "", + "", + + /* GPIO4 C 0-7 */ + "AP_I2C_TS_SDA", + "AP_I2C_TS_SCL", + "GPU_DVS_PWM", + "UART_DBG_TX_AP_RX", + "UART_AP_TX_DBG_RX", + "", + "BIGCPU_DVS_PWM", + "EDP_HPD_3V0", + + /* GPIO4 D 0-5 */ + "SD_CARD_DET_L", + "USB_DP_HPD", + "TOUCH_RESET_L", + "PP3300_DISP_EN", + "", + "SD_SLOT_PWR_EN"; +}; + ap_i2c_mic: &i2c1 { status = "okay"; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts index 2bbef9fcbe..6863689df0 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts @@ -24,6 +24,7 @@ / { "google,kevin-rev9", "google,kevin-rev8", "google,kevin-rev7", "google,kevin-rev6", "google,kevin", "google,gru", "rockchip,rk3399"; + chassis-type = "convertible"; /* Power tree */ diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi index 5d7a9d96d1..a9817b3d7e 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi @@ -8,6 +8,8 @@ #include "rk3399-gru.dtsi" /{ + chassis-type = "tablet"; + /* Power tree */ /* ppvar_sys children, sorted by name */ @@ -389,6 +391,186 @@ &cru { <400000000>; }; +&gpio0 { + gpio-line-names = /* GPIO0 A 0-7 */ + "CLK_32K_AP", + "EC_IN_RW_OD", + "SPK_PA_EN", + "WLAN_PERST_1V8_L", + "WLAN_PD_1V8_L", + "WLAN_RF_KILL_1V8_L", + "BIGCPU_DVS_PWM", + "SD_CD_L_JTAG_EN", + + /* GPIO0 B 0-5 */ + "BT_EN_BT_RF_KILL_1V8_L", + "PMUIO2_33_18_L_PP3300_S0_EN", + "TOUCH_RESET_L", + "AP_EC_WARM_RESET_REQ", + "PEN_RESET_L", + /* + * AP_FLASH_WP_L is crossystem ABI. Schematics call + * it AP_FLASH_WP_R_ODL. + */ + "AP_FLASH_WP_L"; +}; + +&gpio1 { + gpio-line-names = /* GPIO1 A 0-7 */ + "PEN_INT_ODL", + "PEN_EJECT_ODL", + "BT_HOST_WAKE_1V8_L", + "WLAN_HOST_WAKE_1V8_L", + "TOUCH_INT_ODL", + "AP_EC_S3_S0_L", + "AP_EC_OVERTEMP", + "AP_SPI_FLASH_MISO", + + /* GPIO1 B 0-7 */ + "AP_SPI_FLASH_MOSI_R", + "AP_SPI_FLASH_CLK_R", + "AP_SPI_FLASH_CS_L_R", + "SD_CARD_DET_ODL", + "", + "AP_EXPANSION_IO1", + "AP_EXPANSION_IO2", + "AP_I2C_DISP_SDA", + + /* GPIO1 C 0-7 */ + "AP_I2C_DISP_SCL", + "H1_INT_ODL", + "EC_AP_INT_ODL", + "LITCPU_DVS_PWM", + "AP_I2C_AUDIO_SDA", + "AP_I2C_AUDIO_SCL", + "AP_EXPANSION_IO3", + "HEADSET_INT_ODL", + + /* GPIO1 D0 */ + "AP_EXPANSION_IO4"; +}; + +&gpio2 { + gpio-line-names = /* GPIO2 A 0-7 */ + "AP_I2C_PEN_SDA", + "AP_I2C_PEN_SCL", + "SD_IO_PWR_EN", + "UCAM_RST_L", + "PP1250_CAM_EN", + "WCAM_RST_L", + "AP_EXPANSION_IO5", + "AP_I2C_CAM_SDA", + + /* GPIO2 B 0-7 */ + "AP_I2C_CAM_SCL", + "AP_H1_SPI_MISO", + "AP_H1_SPI_MOSI", + "AP_H1_SPI_CLK", + "AP_H1_SPI_CS_L", + "", + "", + "", + + /* GPIO2 C 0-7 */ + "UART_EXPANSION_TX_AP_RX", + "UART_AP_TX_EXPANSION_RX", + "UART_EXPANSION_RTS_AP_CTS", + "UART_AP_RTS_EXPANSION_CTS", + "AP_SPI_EC_MISO", + "AP_SPI_EC_MOSI", + "AP_SPI_EC_CLK", + "AP_SPI_EC_CS_L", + + /* GPIO2 D 0-4 */ + "PP2800_CAM_EN", + "CLK_24M_CAM", + "WLAN_PCIE_CLKREQ_1V8_L", + "", + "SD_PWR_3000_1800_L"; +}; + +&gpio3 { + gpio-line-names = /* GPIO3 A 0-7 */ + "", + "", + "", + "", + "", + "", + "", + "", + + /* GPIO3 B 0-7 */ + "", + "", + "", + "", + "", + "", + "", + "", + + /* GPIO3 C 0-7 */ + "", + "", + "", + "", + "", + "", + "", + "", + + /* GPIO3 D 0-7 */ + "I2S0_SCLK", + "I2S0_LRCK_RX", + "I2S0_LRCK_TX", + "I2S0_SDI_0", + "STRAP_LCDBIAS_L", + "STRAP_FEATURE_1", + "STRAP_FEATURE_2", + "I2S0_SDO_0"; +}; + +&gpio4 { + gpio-line-names = /* GPIO4 A 0-7 */ + "I2S_MCLK", + "AP_I2C_EXPANSION_SDA", + "AP_I2C_EXPANSION_SCL", + "DMIC_EN", + "", + "", + "", + "", + + /* GPIO4 B 0-7 */ + "", + "", + "", + "", + "", + "", + "", + "", + + /* GPIO4 C 0-7 */ + "AP_I2C_TS_SDA", + "AP_I2C_TS_SCL", + "GPU_DVS_PWM", + "UART_DBG_TX_AP_RX", + "UART_AP_TX_DBG_RX", + "BL_EN", + "BL_PWM", + "", + + /* GPIO4 D 0-5 */ + "", + "DISPLAY_RST_L", + "", + "PPVARP_LCD_EN", + "PPVARN_LCD_EN", + "SD_SLOT_PWR_EN"; +}; + &i2c_tunnel { google,remote-bus = <0>; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi index 2f8e117109..162f08bca0 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi @@ -457,7 +457,7 @@ &pcie0 { vpcie0v9-supply = <&pp900_pcie>; pci_rootport: pcie@0,0 { - reg = <0x83000000 0x0 0x00000000 0x0 0x00000000>; + reg = <0x0000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; ranges; @@ -550,7 +550,7 @@ &spi1 { pinctrl-names = "default", "sleep"; pinctrl-1 = <&spi1_sleep>; - spiflash@0 { + flash@0 { compatible = "jedec,spi-nor"; reg = <0>; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts b/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts index 354f54767b..1eb287a3f8 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-kobol-helios64.dts @@ -21,6 +21,9 @@ / { aliases { mmc0 = &sdmmc; mmc1 = &sdhci; + spi1 = &spi1; + spi2 = &spi2; + spi5 = &spi5; }; avdd_0v9_s0: avdd-0v9-s0 { @@ -43,6 +46,10 @@ avdd_1v8_s0: avdd-1v8-s0 { vin-supply = <&vcc3v3_sys_s3>; }; + chosen { + stdout-path = "serial2:1500000n8"; + }; + clkin_gmac: external-gmac-clock { compatible = "fixed-clock"; clock-frequency = <125000000>; @@ -82,6 +89,30 @@ led-1 { }; }; + hdd_a_power: hdd-a-power { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio1 RK_PA0 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&hdd_a_power_en>; + pinctrl-names = "default"; + regulator-always-on; + regulator-boot-on; + regulator-name = "hdd_a_power"; + startup-delay-us = <2000000>; + }; + + hdd_b_power: hdd-b-power { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio1 RK_PA1 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&hdd_b_power_en>; + pinctrl-names = "default"; + regulator-always-on; + regulator-boot-on; + regulator-name = "hdd_b_power"; + startup-delay-us = <2000000>; + }; + pcie_power: pcie-power { compatible = "regulator-fixed"; enable-active-high; @@ -94,6 +125,18 @@ pcie_power: pcie-power { vin-supply = <&vcc5v0_perdev>; }; + usblan_power: usblan-power { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio1 RK_PC7 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb_lan_en>; + regulator-name = "usblan_power"; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc5v0_usb>; + }; + vcc1v8_sys_s0: vcc1v8-sys-s0 { compatible = "regulator-fixed"; regulator-name = "vcc1v8_sys_s0"; @@ -383,7 +426,10 @@ &pcie_phy { }; &pcie0 { + ep-gpios = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; + max-link-speed = <2>; num-lanes = <2>; + pinctrl-names = "default"; status = "okay"; vpcie12v-supply = <&vcc12v_dcin>; @@ -423,9 +469,21 @@ pmic_int_l: pmic-int-l { }; power { + hdd_a_power_en: hdd-a-power-en { + rockchip,pins = <1 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + hdd_b_power_en: hdd-b-power-en { + rockchip,pins = <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + vcc5v0_usb_en: vcc5v0-usb-en { rockchip,pins = <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; }; + + usb_lan_en: usb-lan-en { + rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; + }; }; vcc3v0-sd { @@ -470,11 +528,40 @@ &sdmmc { status = "okay"; }; +&spi1 { + status = "okay"; + + spiflash: flash@0 { + compatible = "jedec,spi-nor"; + reg = <0x0>; + spi-max-frequency = <25000000>; + status = "okay"; + m25p,fast-read; + }; +}; + +/* UEXT connector */ +&spi2 { + status = "okay"; +}; + +&spi5 { + status = "okay"; +}; + &tcphy1 { /* phy for &usbdrd_dwc3_1 */ status = "okay"; }; +&tsadc { + /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-mode = <1>; + /* tshut polarity 0:LOW 1:HIGH */ + rockchip,hw-tshut-polarity = <1>; + status = "okay"; +}; + &u2phy1 { status = "okay"; @@ -495,5 +582,41 @@ &usbdrd3_1 { usb@fe900000 { dr_mode = "host"; status = "okay"; + #address-cells = <1>; + #size-cells = <0>; + + hub@1 { + compatible = "usb2109,0815"; + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + #trigger-source-cells = <0>; + }; + + port@2 { + reg = <2>; + #trigger-source-cells = <0>; + }; + + port@3 { + reg = <3>; + #trigger-source-cells = <0>; + }; + + device@4 { + compatible = "usbbda,8156"; + reg = <4>; + #address-cells = <2>; + #size-cells = <0>; + + interface@0 { /* interface 0 of configuration 1 */ + compatible = "usbbda,8156.config1.0"; + reg = <0 1>; + }; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-op1-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-op1-opp.dtsi index 69cc9b05ba..2180e0f750 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-op1-opp.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-op1-opp.dtsi @@ -4,7 +4,7 @@ */ / { - cluster0_opp: opp-table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -39,7 +39,7 @@ opp06 { }; }; - cluster1_opp: opp-table1 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; @@ -82,7 +82,7 @@ opp08 { }; }; - gpu_opp_table: opp-table2 { + gpu_opp_table: opp-table-2 { compatible = "operating-points-v2"; opp00 { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi index da41cd81eb..fee5e71112 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi @@ -4,7 +4,7 @@ */ / { - cluster0_opp: opp-table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -35,7 +35,7 @@ opp05 { }; }; - cluster1_opp: opp-table1 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; @@ -74,7 +74,7 @@ opp07 { }; }; - gpu_opp_table: opp-table2 { + gpu_opp_table: opp-table-2 { compatible = "operating-points-v2"; opp00 { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts index 9e5d07f571..c2f021a1a1 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts @@ -17,6 +17,7 @@ / { model = "Pine64 Pinebook Pro"; compatible = "pine64,pinebook-pro", "rockchip,rk3399"; + chassis-type = "laptop"; aliases { mmc0 = &sdio0; @@ -707,7 +708,7 @@ fusb0: fusb30x@22 { connector { compatible = "usb-c-connector"; - data-role = "host"; + data-role = "dual"; label = "USB-C"; op-sink-microwatt = <1000000>; power-role = "dual"; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi index 100a769165..92acf6ea29 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi @@ -36,6 +36,29 @@ sdio_pwrseq: sdio-pwrseq { reset-gpios = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>; }; + sound: sound { + compatible = "audio-graph-card"; + label = "Analog"; + dais = <&i2s0_p0>; + }; + + sound-dit { + compatible = "audio-graph-card"; + label = "SPDIF"; + dais = <&spdif_p0>; + }; + + spdif-dit { + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + + port { + dit_p0_0: endpoint { + remote-endpoint = <&spdif_p0_0>; + }; + }; + }; + vcc12v_dcin: dc-12v { compatible = "regulator-fixed"; regulator-name = "vcc12v_dcin"; @@ -422,6 +445,20 @@ &i2c1 { i2c-scl-rising-time-ns = <300>; i2c-scl-falling-time-ns = <15>; status = "okay"; + + es8316: codec@11 { + compatible = "everest,es8316"; + reg = <0x11>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + #sound-dai-cells = <0>; + + port { + es8316_p0_0: endpoint { + remote-endpoint = <&i2s0_p0_0>; + }; + }; + }; }; &i2c3 { @@ -441,6 +478,14 @@ &i2s0 { rockchip,capture-channels = <2>; rockchip,playback-channels = <2>; status = "okay"; + + i2s0_p0: port { + i2s0_p0_0: endpoint { + dai-format = "i2s"; + mclk-fs = <256>; + remote-endpoint = <&es8316_p0_0>; + }; + }; }; &i2s1 { @@ -498,6 +543,16 @@ bt_wake_l: bt-wake-l { }; }; + es8316 { + hp_detect: hp-detect { + rockchip,pins = <1 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + hp_int: hp-int { + rockchip,pins = <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + pcie { pcie_pwr_en: pcie-pwr-en { rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; @@ -603,6 +658,15 @@ &sdhci { status = "okay"; }; +&spdif { + + spdif_p0: port { + spdif_p0_0: endpoint { + remote-endpoint = <&dit_p0_0>; + }; + }; +}; + &tcphy0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4b.dts b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4b.dts index 6c63e61706..cf48746a3a 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4b.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4b.dts @@ -34,13 +34,16 @@ &uart0 { status = "okay"; bluetooth { - compatible = "brcm,bcm43438-bt"; + compatible = "brcm,bcm4345c5"; clocks = <&rk808 1>; - clock-names = "ext_clock"; + clock-names = "lpo"; device-wakeup-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>; host-wakeup-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; shutdown-gpios = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>; + max-speed = <1500000>; pinctrl-names = "default"; pinctrl-0 = <&bt_host_wake_l &bt_wake_l &bt_enable_h>; + vbat-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc_1v8>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4c.dts b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4c.dts index 99169bcd51..793d848339 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4c.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4c.dts @@ -17,6 +17,13 @@ aliases { }; }; +&es8316 { + pinctrl-0 = <&hp_detect &hp_int>; + pinctrl-names = "default"; + interrupt-parent = <&gpio1>; + interrupts = ; +}; + &sdio0 { status = "okay"; @@ -31,18 +38,25 @@ brcmf: wifi@1 { }; }; +&sound { + hp-det-gpio = <&gpio1 RK_PA0 GPIO_ACTIVE_HIGH>; +}; + &uart0 { status = "okay"; bluetooth { - compatible = "brcm,bcm43438-bt"; + compatible = "brcm,bcm4345c5"; clocks = <&rk808 1>; - clock-names = "ext_clock"; + clock-names = "lpo"; device-wakeup-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>; host-wakeup-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; shutdown-gpios = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>; + max-speed = <1500000>; pinctrl-names = "default"; pinctrl-0 = <&bt_host_wake_l &bt_wake_l &bt_enable_h>; + vbat-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc_1v8>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rockpro64.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-rockpro64.dtsi index 6bff8db7d3..83db4ca673 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-rockpro64.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-rockpro64.dtsi @@ -69,6 +69,7 @@ diy_led: led-1 { fan: pwm-fan { compatible = "pwm-fan"; + cooling-levels = <0 100 150 200 255>; #cooling-cells = <2>; fan-supply = <&vcc12v_dcin>; pwms = <&pwm1 0 50000 0>; @@ -245,6 +246,34 @@ &cpu_b1 { cpu-supply = <&vdd_cpu_b>; }; +&cpu_thermal { + trips { + cpu_warm: cpu_warm { + temperature = <55000>; + hysteresis = <2000>; + type = "active"; + }; + + cpu_hot: cpu_hot { + temperature = <65000>; + hysteresis = <2000>; + type = "active"; + }; + }; + + cooling-maps { + map2 { + trip = <&cpu_warm>; + cooling-device = <&fan THERMAL_NO_LIMIT 1>; + }; + + map3 { + trip = <&cpu_hot>; + cooling-device = <&fan 2 THERMAL_NO_LIMIT>; + }; + }; +}; + &emmc_phy { status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi index 3871c7fd83..080457a68e 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi @@ -124,6 +124,12 @@ cpu_b0: cpu@100 { #cooling-cells = <2>; /* min followed by max */ dynamic-power-coefficient = <436>; cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + + thermal-idle { + #cooling-cells = <2>; + duration-us = <10000>; + exit-latency-us = <500>; + }; }; cpu_b1: cpu@101 { @@ -136,6 +142,12 @@ cpu_b1: cpu@101 { #cooling-cells = <2>; /* min followed by max */ dynamic-power-coefficient = <436>; cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + + thermal-idle { + #cooling-cells = <2>; + duration-us = <10000>; + exit-latency-us = <500>; + }; }; idle-states { @@ -361,6 +373,54 @@ usb_host1_ohci: usb@fe3e0000 { status = "disabled"; }; + debug@fe430000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0 0xfe430000 0 0x1000>; + clocks = <&cru PCLK_COREDBG_L>; + clock-names = "apb_pclk"; + cpu = <&cpu_l0>; + }; + + debug@fe432000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0 0xfe432000 0 0x1000>; + clocks = <&cru PCLK_COREDBG_L>; + clock-names = "apb_pclk"; + cpu = <&cpu_l1>; + }; + + debug@fe434000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0 0xfe434000 0 0x1000>; + clocks = <&cru PCLK_COREDBG_L>; + clock-names = "apb_pclk"; + cpu = <&cpu_l2>; + }; + + debug@fe436000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0 0xfe436000 0 0x1000>; + clocks = <&cru PCLK_COREDBG_L>; + clock-names = "apb_pclk"; + cpu = <&cpu_l3>; + }; + + debug@fe610000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0 0xfe610000 0 0x1000>; + clocks = <&cru PCLK_COREDBG_B>; + clock-names = "apb_pclk"; + cpu = <&cpu_b0>; + }; + + debug@fe710000 { + compatible = "arm,coresight-cpu-debug", "arm,primecell"; + reg = <0 0xfe710000 0 0x1000>; + clocks = <&cru PCLK_COREDBG_B>; + clock-names = "apb_pclk"; + cpu = <&cpu_b1>; + }; + usbdrd3_0: usb@fe800000 { compatible = "rockchip,rk3399-dwc3"; #address-cells = <2>; @@ -1251,7 +1311,6 @@ vpu_mmu: iommu@ff650800 { compatible = "rockchip,iommu"; reg = <0x0 0xff650800 0x0 0x40>; interrupts = ; - interrupt-names = "vpu_mmu"; clocks = <&cru ACLK_VCODEC>, <&cru HCLK_VCODEC>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -1273,7 +1332,6 @@ vdec_mmu: iommu@ff660480 { compatible = "rockchip,iommu"; reg = <0x0 0xff660480 0x0 0x40>, <0x0 0xff6604c0 0x0 0x40>; interrupts = ; - interrupt-names = "vdec_mmu"; clocks = <&cru ACLK_VDU>, <&cru HCLK_VDU>; clock-names = "aclk", "iface"; power-domains = <&power RK3399_PD_VDU>; @@ -1284,7 +1342,6 @@ iep_mmu: iommu@ff670800 { compatible = "rockchip,iommu"; reg = <0x0 0xff670800 0x0 0x40>; interrupts = ; - interrupt-names = "iep_mmu"; clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -1666,7 +1723,6 @@ vopl_mmu: iommu@ff8f3f00 { compatible = "rockchip,iommu"; reg = <0x0 0xff8f3f00 0x0 0x100>; interrupts = ; - interrupt-names = "vopl_mmu"; clocks = <&cru ACLK_VOP1>, <&cru HCLK_VOP1>; clock-names = "aclk", "iface"; power-domains = <&power RK3399_PD_VOPL>; @@ -1723,7 +1779,6 @@ vopb_mmu: iommu@ff903f00 { compatible = "rockchip,iommu"; reg = <0x0 0xff903f00 0x0 0x100>; interrupts = ; - interrupt-names = "vopb_mmu"; clocks = <&cru ACLK_VOP0>, <&cru HCLK_VOP0>; clock-names = "aclk", "iface"; power-domains = <&power RK3399_PD_VOPB>; @@ -1761,7 +1816,6 @@ isp0_mmu: iommu@ff914000 { compatible = "rockchip,iommu"; reg = <0x0 0xff914000 0x0 0x100>, <0x0 0xff915000 0x0 0x100>; interrupts = ; - interrupt-names = "isp0_mmu"; clocks = <&cru ACLK_ISP0_WRAPPER>, <&cru HCLK_ISP0_WRAPPER>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -1769,11 +1823,36 @@ isp0_mmu: iommu@ff914000 { rockchip,disable-mmu-reset; }; + isp1: isp1@ff920000 { + compatible = "rockchip,rk3399-cif-isp"; + reg = <0x0 0xff920000 0x0 0x4000>; + interrupts = ; + clocks = <&cru SCLK_ISP1>, + <&cru ACLK_ISP1_WRAPPER>, + <&cru HCLK_ISP1_WRAPPER>; + clock-names = "isp", "aclk", "hclk"; + iommus = <&isp1_mmu>; + phys = <&mipi_dsi1>; + phy-names = "dphy"; + power-domains = <&power RK3399_PD_ISP1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + }; + isp1_mmu: iommu@ff924000 { compatible = "rockchip,iommu"; reg = <0x0 0xff924000 0x0 0x100>, <0x0 0xff925000 0x0 0x100>; interrupts = ; - interrupt-names = "isp1_mmu"; clocks = <&cru ACLK_ISP1_WRAPPER>, <&cru HCLK_ISP1_WRAPPER>; clock-names = "aclk", "iface"; #iommu-cells = <0>; @@ -1802,10 +1881,10 @@ hdmi: hdmi@ff940000 { interrupts = ; clocks = <&cru PCLK_HDMI_CTRL>, <&cru SCLK_HDMI_SFR>, - <&cru PLL_VPLL>, + <&cru SCLK_HDMI_CEC>, <&cru PCLK_VIO_GRF>, - <&cru SCLK_HDMI_CEC>; - clock-names = "iahb", "isfr", "vpll", "grf", "cec"; + <&cru PLL_VPLL>; + clock-names = "iahb", "isfr", "cec", "grf", "vpll"; power-domains = <&power RK3399_PD_HDCP>; reg-io-width = <4>; rockchip,grf = <&grf>; @@ -1878,6 +1957,7 @@ mipi_dsi1: mipi@ff968000 { rockchip,grf = <&grf>; #address-cells = <1>; #size-cells = <0>; + #phy-cells = <0>; status = "disabled"; ports { @@ -1958,7 +2038,7 @@ pinctrl: pinctrl { #size-cells = <2>; ranges; - gpio0: gpio0@ff720000 { + gpio0: gpio@ff720000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff720000 0x0 0x100>; clocks = <&pmucru PCLK_GPIO0_PMU>; @@ -1971,7 +2051,7 @@ gpio0: gpio0@ff720000 { #interrupt-cells = <0x2>; }; - gpio1: gpio1@ff730000 { + gpio1: gpio@ff730000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff730000 0x0 0x100>; clocks = <&pmucru PCLK_GPIO1_PMU>; @@ -1984,7 +2064,7 @@ gpio1: gpio1@ff730000 { #interrupt-cells = <0x2>; }; - gpio2: gpio2@ff780000 { + gpio2: gpio@ff780000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff780000 0x0 0x100>; clocks = <&cru PCLK_GPIO2>; @@ -1997,7 +2077,7 @@ gpio2: gpio2@ff780000 { #interrupt-cells = <0x2>; }; - gpio3: gpio3@ff788000 { + gpio3: gpio@ff788000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff788000 0x0 0x100>; clocks = <&cru PCLK_GPIO3>; @@ -2010,7 +2090,7 @@ gpio3: gpio3@ff788000 { #interrupt-cells = <0x2>; }; - gpio4: gpio4@ff790000 { + gpio4: gpio@ff790000 { compatible = "rockchip,gpio-bank"; reg = <0x0 0xff790000 0x0 0x100>; clocks = <&cru PCLK_GPIO4>; @@ -2114,6 +2194,18 @@ clk_32k: clk-32k { }; }; + cif { + cif_clkin: cif-clkin { + rockchip,pins = + <2 RK_PB2 3 &pcfg_pull_none>; + }; + + cif_clkouta: cif-clkouta { + rockchip,pins = + <2 RK_PB3 3 &pcfg_pull_none>; + }; + }; + edp { edp_hpd: edp-hpd { rockchip,pins = diff --git a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts index 166399b7f1..d9eb92d590 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts @@ -285,8 +285,6 @@ regulator-state-mem { vcc_ddr: DCDC_REG3 { regulator-always-on; regulator-boot-on; - regulator-min-microvolt = <1100000>; - regulator-max-microvolt = <1100000>; regulator-initial-mode = <0x2>; regulator-name = "vcc_ddr"; regulator-state-mem { diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb1-v10.dts index 6978655709..184e2aa241 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-evb1-v10.dts +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-v10.dts @@ -13,6 +13,13 @@ / { model = "Rockchip RK3568 EVB1 DDR4 V10 Board"; compatible = "rockchip,rk3568-evb1-v10", "rockchip,rk3568"; + aliases { + ethernet0 = &gmac0; + ethernet1 = &gmac1; + mmc0 = &sdmmc0; + mmc1 = &sdhci; + }; + chosen: chosen { stdout-path = "serial2:1500000n8"; }; @@ -67,10 +74,316 @@ regulator-state-mem { }; }; +&gmac0 { + assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>; + assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>; + assigned-clock-rates = <0>, <125000000>; + clock_in_out = "output"; + phy-handle = <&rgmii_phy0>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_miim + &gmac0_tx_bus2 + &gmac0_rx_bus2 + &gmac0_rgmii_clk + &gmac0_rgmii_bus>; + status = "okay"; +}; + +&gmac1 { + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>; + assigned-clock-rates = <0>, <125000000>; + clock_in_out = "output"; + phy-handle = <&rgmii_phy1>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m1_miim + &gmac1m1_tx_bus2 + &gmac1m1_rx_bus2 + &gmac1m1_rgmii_clk + &gmac1m1_rgmii_bus>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = ; + #clock-cells = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int>; + rockchip,system-power-controller; + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + wakeup-source; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-name = "vdd_logic"; + regulator-always-on; + regulator-boot-on; + regulator-init-microvolt = <900000>; + regulator-initial-mode = <0x2>; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-name = "vdd_gpu"; + regulator-init-microvolt = <900000>; + regulator-initial-mode = <0x2>; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vdd_npu: DCDC_REG4 { + regulator-name = "vdd_npu"; + regulator-init-microvolt = <900000>; + regulator-initial-mode = <0x2>; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG5 { + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_image: LDO_REG1 { + regulator-name = "vdda0v9_image"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-name = "vdda_0v9"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-name = "vdda0v9_pmu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-name = "vccio_acodec"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-name = "vccio_sd"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-name = "vcc3v3_pmu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca_1v8: LDO_REG7 { + regulator-name = "vcca_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca1v8_pmu: LDO_REG8 { + regulator-name = "vcca1v8_pmu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca1v8_image: LDO_REG9 { + regulator-name = "vcca1v8_image"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v3: SWITCH_REG1 { + regulator-name = "vcc_3v3"; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_sd: SWITCH_REG2 { + regulator-name = "vcc3v3_sd"; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&mdio0 { + rgmii_phy0: ethernet-phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + reset-assert-us = <20000>; + reset-deassert-us = <100000>; + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + }; +}; + +&mdio1 { + rgmii_phy1: ethernet-phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + reset-assert-us = <20000>; + reset-deassert-us = <100000>; + reset-gpios = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>; + }; +}; + +&pinctrl { + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pmu_io_domains { + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio2-supply = <&vcc_1v8>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcc_1v8>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_1v8>; + vccio7-supply = <&vcc_3v3>; + status = "okay"; +}; + +&saradc { + vref-supply = <&vcca_1v8>; + status = "okay"; +}; + &sdhci { bus-width = <8>; max-frequency = <200000000>; non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_bus8 &emmc_clk &emmc_cmd &emmc_datastrobe>; + status = "okay"; +}; + +&sdmmc0 { + bus-width = <4>; + cap-sd-highspeed; + cd-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; + disable-wp; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; + sd-uhs-sdr104; + vmmc-supply = <&vcc3v3_sd>; + vqmmc-supply = <&vccio_sd>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi index a588ca95ac..8f90c66dd9 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi @@ -3108,4 +3108,13 @@ gmac1m1_rgmii_clk_level2: gmac1m1-rgmii-clk-level2 { <4 RK_PA0 3 &pcfg_pull_none_drv_level_2>; }; }; + + tsadc { + /omit-if-no-ref/ + tsadc_pin: tsadc-pin { + rockchip,pins = + /* tsadc_pin */ + <0 RK_PA1 0 &pcfg_pull_none>; + }; + }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3568.dtsi b/arch/arm64/boot/dts/rockchip/rk3568.dtsi index d225e6a45d..d91df1cde7 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568.dtsi @@ -3,591 +3,93 @@ * Copyright (c) 2021 Rockchip Electronics Co., Ltd. */ -#include -#include -#include -#include -#include -#include -#include +#include "rk356x.dtsi" / { compatible = "rockchip,rk3568"; - interrupt-parent = <&gic>; - #address-cells = <2>; - #size-cells = <2>; - - aliases { - gpio0 = &gpio0; - gpio1 = &gpio1; - gpio2 = &gpio2; - gpio3 = &gpio3; - gpio4 = &gpio4; - i2c0 = &i2c0; - i2c1 = &i2c1; - i2c2 = &i2c2; - i2c3 = &i2c3; - i2c4 = &i2c4; - i2c5 = &i2c5; - serial0 = &uart0; - serial1 = &uart1; - serial2 = &uart2; - serial3 = &uart3; - serial4 = &uart4; - serial5 = &uart5; - serial6 = &uart6; - serial7 = &uart7; - serial8 = &uart8; - serial9 = &uart9; + qos_pcie3x1: qos@fe190080 { + compatible = "rockchip,rk3568-qos", "syscon"; + reg = <0x0 0xfe190080 0x0 0x20>; }; - cpus { - #address-cells = <2>; - #size-cells = <0>; - - cpu0: cpu@0 { - device_type = "cpu"; - compatible = "arm,cortex-a55"; - reg = <0x0 0x0>; - clocks = <&scmi_clk 0>; - enable-method = "psci"; - operating-points-v2 = <&cpu0_opp_table>; - }; - - cpu1: cpu@100 { - device_type = "cpu"; - compatible = "arm,cortex-a55"; - reg = <0x0 0x100>; - enable-method = "psci"; - operating-points-v2 = <&cpu0_opp_table>; - }; - - cpu2: cpu@200 { - device_type = "cpu"; - compatible = "arm,cortex-a55"; - reg = <0x0 0x200>; - enable-method = "psci"; - operating-points-v2 = <&cpu0_opp_table>; - }; - - cpu3: cpu@300 { - device_type = "cpu"; - compatible = "arm,cortex-a55"; - reg = <0x0 0x300>; - enable-method = "psci"; - operating-points-v2 = <&cpu0_opp_table>; - }; + qos_pcie3x2: qos@fe190100 { + compatible = "rockchip,rk3568-qos", "syscon"; + reg = <0x0 0xfe190100 0x0 0x20>; }; - cpu0_opp_table: cpu0-opp-table { - compatible = "operating-points-v2"; - opp-shared; - - opp-408000000 { - opp-hz = /bits/ 64 <408000000>; - opp-microvolt = <900000 900000 1150000>; - clock-latency-ns = <40000>; - }; - - opp-600000000 { - opp-hz = /bits/ 64 <600000000>; - opp-microvolt = <900000 900000 1150000>; - }; - - opp-816000000 { - opp-hz = /bits/ 64 <816000000>; - opp-microvolt = <900000 900000 1150000>; - opp-suspend; - }; - - opp-1104000000 { - opp-hz = /bits/ 64 <1104000000>; - opp-microvolt = <900000 900000 1150000>; - }; - - opp-1416000000 { - opp-hz = /bits/ 64 <1416000000>; - opp-microvolt = <900000 900000 1150000>; - }; - - opp-1608000000 { - opp-hz = /bits/ 64 <1608000000>; - opp-microvolt = <975000 975000 1150000>; - }; - - opp-1800000000 { - opp-hz = /bits/ 64 <1800000000>; - opp-microvolt = <1050000 1050000 1150000>; - }; - - opp-1992000000 { - opp-hz = /bits/ 64 <1992000000>; - opp-microvolt = <1150000 1150000 1150000>; - }; + qos_sata0: qos@fe190200 { + compatible = "rockchip,rk3568-qos", "syscon"; + reg = <0x0 0xfe190200 0x0 0x20>; }; - firmware { - scmi: scmi { - compatible = "arm,scmi-smc"; - arm,smc-id = <0x82000010>; - shmem = <&scmi_shmem>; - #address-cells = <1>; - #size-cells = <0>; - - scmi_clk: protocol@14 { - reg = <0x14>; - #clock-cells = <1>; - }; - }; - }; - - pmu { - compatible = "arm,cortex-a55-pmu"; - interrupts = , - , - , - ; - interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>; - }; - - psci { - compatible = "arm,psci-1.0"; - method = "smc"; - }; - - timer { - compatible = "arm,armv8-timer"; - interrupts = , - , - , - ; - arm,no-tick-in-suspend; - }; - - xin24m: xin24m { - compatible = "fixed-clock"; - clock-frequency = <24000000>; - clock-output-names = "xin24m"; - #clock-cells = <0>; - }; - - xin32k: xin32k { - compatible = "fixed-clock"; - clock-frequency = <32768>; - clock-output-names = "xin32k"; - pinctrl-0 = <&clk32k_out0>; - pinctrl-names = "default"; - #clock-cells = <0>; - }; - - sram@10f000 { - compatible = "mmio-sram"; - reg = <0x0 0x0010f000 0x0 0x100>; - #address-cells = <1>; - #size-cells = <1>; - ranges = <0 0x0 0x0010f000 0x100>; - - scmi_shmem: sram@0 { - compatible = "arm,scmi-shmem"; - reg = <0x0 0x100>; - }; - }; - - gic: interrupt-controller@fd400000 { - compatible = "arm,gic-v3"; - reg = <0x0 0xfd400000 0 0x10000>, /* GICD */ - <0x0 0xfd460000 0 0x80000>; /* GICR */ - interrupts = ; - interrupt-controller; - #interrupt-cells = <3>; - mbi-alias = <0x0 0xfd100000>; - mbi-ranges = <296 24>; - msi-controller; - }; - - pmugrf: syscon@fdc20000 { - compatible = "rockchip,rk3568-pmugrf", "syscon", "simple-mfd"; - reg = <0x0 0xfdc20000 0x0 0x10000>; - }; - - grf: syscon@fdc60000 { - compatible = "rockchip,rk3568-grf", "syscon", "simple-mfd"; - reg = <0x0 0xfdc60000 0x0 0x10000>; - }; - - pmucru: clock-controller@fdd00000 { - compatible = "rockchip,rk3568-pmucru"; - reg = <0x0 0xfdd00000 0x0 0x1000>; - #clock-cells = <1>; - #reset-cells = <1>; - }; - - cru: clock-controller@fdd20000 { - compatible = "rockchip,rk3568-cru"; - reg = <0x0 0xfdd20000 0x0 0x1000>; - #clock-cells = <1>; - #reset-cells = <1>; - }; - - i2c0: i2c@fdd40000 { - compatible = "rockchip,rk3568-i2c", "rockchip,rk3399-i2c"; - reg = <0x0 0xfdd40000 0x0 0x1000>; - interrupts = ; - clocks = <&pmucru CLK_I2C0>, <&pmucru PCLK_I2C0>; - clock-names = "i2c", "pclk"; - pinctrl-0 = <&i2c0_xfer>; - pinctrl-names = "default"; - #address-cells = <1>; - #size-cells = <0>; - status = "disabled"; - }; - - uart0: serial@fdd50000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfdd50000 0x0 0x100>; - interrupts = ; - clocks = <&pmucru SCLK_UART0>, <&pmucru PCLK_UART0>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 0>, <&dmac0 1>; - pinctrl-0 = <&uart0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - sdmmc2: mmc@fe000000 { - compatible = "rockchip,rk3568-dw-mshc", "rockchip,rk3288-dw-mshc"; - reg = <0x0 0xfe000000 0x0 0x4000>; - interrupts = ; - clocks = <&cru HCLK_SDMMC2>, <&cru CLK_SDMMC2>, - <&cru SCLK_SDMMC2_DRV>, <&cru SCLK_SDMMC2_SAMPLE>; - clock-names = "biu", "ciu", "ciu-drive", "ciu-sample"; - fifo-depth = <0x100>; - max-frequency = <150000000>; - resets = <&cru SRST_SDMMC2>; - reset-names = "reset"; - status = "disabled"; - }; - - sdmmc0: mmc@fe2b0000 { - compatible = "rockchip,rk3568-dw-mshc", "rockchip,rk3288-dw-mshc"; - reg = <0x0 0xfe2b0000 0x0 0x4000>; - interrupts = ; - clocks = <&cru HCLK_SDMMC0>, <&cru CLK_SDMMC0>, - <&cru SCLK_SDMMC0_DRV>, <&cru SCLK_SDMMC0_SAMPLE>; - clock-names = "biu", "ciu", "ciu-drive", "ciu-sample"; - fifo-depth = <0x100>; - max-frequency = <150000000>; - resets = <&cru SRST_SDMMC0>; - reset-names = "reset"; - status = "disabled"; - }; - - sdmmc1: mmc@fe2c0000 { - compatible = "rockchip,rk3568-dw-mshc", "rockchip,rk3288-dw-mshc"; - reg = <0x0 0xfe2c0000 0x0 0x4000>; - interrupts = ; - clocks = <&cru HCLK_SDMMC1>, <&cru CLK_SDMMC1>, - <&cru SCLK_SDMMC1_DRV>, <&cru SCLK_SDMMC1_SAMPLE>; - clock-names = "biu", "ciu", "ciu-drive", "ciu-sample"; - fifo-depth = <0x100>; - max-frequency = <150000000>; - resets = <&cru SRST_SDMMC1>; - reset-names = "reset"; - status = "disabled"; - }; - - sdhci: mmc@fe310000 { - compatible = "rockchip,rk3568-dwcmshc"; - reg = <0x0 0xfe310000 0x0 0x10000>; - interrupts = ; - assigned-clocks = <&cru BCLK_EMMC>, <&cru TCLK_EMMC>; - assigned-clock-rates = <200000000>, <24000000>; - clocks = <&cru CCLK_EMMC>, <&cru HCLK_EMMC>, - <&cru ACLK_EMMC>, <&cru BCLK_EMMC>, - <&cru TCLK_EMMC>; - clock-names = "core", "bus", "axi", "block", "timer"; - status = "disabled"; - }; - - dmac0: dmac@fe530000 { - compatible = "arm,pl330", "arm,primecell"; - reg = <0x0 0xfe530000 0x0 0x4000>; - interrupts = , - ; - arm,pl330-periph-burst; - clocks = <&cru ACLK_BUS>; - clock-names = "apb_pclk"; - #dma-cells = <1>; - }; - - dmac1: dmac@fe550000 { - compatible = "arm,pl330", "arm,primecell"; - reg = <0x0 0xfe550000 0x0 0x4000>; - interrupts = , - ; - arm,pl330-periph-burst; - clocks = <&cru ACLK_BUS>; - clock-names = "apb_pclk"; - #dma-cells = <1>; - }; - - i2c1: i2c@fe5a0000 { - compatible = "rockchip,rk3568-i2c", "rockchip,rk3399-i2c"; - reg = <0x0 0xfe5a0000 0x0 0x1000>; - interrupts = ; - clocks = <&cru CLK_I2C1>, <&cru PCLK_I2C1>; - clock-names = "i2c", "pclk"; - pinctrl-0 = <&i2c1_xfer>; - pinctrl-names = "default"; - #address-cells = <1>; - #size-cells = <0>; - status = "disabled"; - }; - - i2c2: i2c@fe5b0000 { - compatible = "rockchip,rk3568-i2c", "rockchip,rk3399-i2c"; - reg = <0x0 0xfe5b0000 0x0 0x1000>; - interrupts = ; - clocks = <&cru CLK_I2C2>, <&cru PCLK_I2C2>; - clock-names = "i2c", "pclk"; - pinctrl-0 = <&i2c2m0_xfer>; - pinctrl-names = "default"; - #address-cells = <1>; - #size-cells = <0>; - status = "disabled"; - }; - - i2c3: i2c@fe5c0000 { - compatible = "rockchip,rk3568-i2c", "rockchip,rk3399-i2c"; - reg = <0x0 0xfe5c0000 0x0 0x1000>; - interrupts = ; - clocks = <&cru CLK_I2C3>, <&cru PCLK_I2C3>; - clock-names = "i2c", "pclk"; - pinctrl-0 = <&i2c3m0_xfer>; - pinctrl-names = "default"; - #address-cells = <1>; - #size-cells = <0>; - status = "disabled"; - }; - - i2c4: i2c@fe5d0000 { - compatible = "rockchip,rk3568-i2c", "rockchip,rk3399-i2c"; - reg = <0x0 0xfe5d0000 0x0 0x1000>; - interrupts = ; - clocks = <&cru CLK_I2C4>, <&cru PCLK_I2C4>; - clock-names = "i2c", "pclk"; - pinctrl-0 = <&i2c4m0_xfer>; - pinctrl-names = "default"; - #address-cells = <1>; - #size-cells = <0>; - status = "disabled"; - }; - - i2c5: i2c@fe5e0000 { - compatible = "rockchip,rk3568-i2c", "rockchip,rk3399-i2c"; - reg = <0x0 0xfe5e0000 0x0 0x1000>; - interrupts = ; - clocks = <&cru CLK_I2C5>, <&cru PCLK_I2C5>; - clock-names = "i2c", "pclk"; - pinctrl-0 = <&i2c5m0_xfer>; - pinctrl-names = "default"; - #address-cells = <1>; - #size-cells = <0>; - status = "disabled"; - }; - - uart1: serial@fe650000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfe650000 0x0 0x100>; - interrupts = ; - clocks = <&cru SCLK_UART1>, <&cru PCLK_UART1>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 2>, <&dmac0 3>; - pinctrl-0 = <&uart1m0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - uart2: serial@fe660000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfe660000 0x0 0x100>; - interrupts = ; - clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 4>, <&dmac0 5>; - pinctrl-0 = <&uart2m0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - uart3: serial@fe670000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfe670000 0x0 0x100>; - interrupts = ; - clocks = <&cru SCLK_UART3>, <&cru PCLK_UART3>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 6>, <&dmac0 7>; - pinctrl-0 = <&uart3m0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - uart4: serial@fe680000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfe680000 0x0 0x100>; - interrupts = ; - clocks = <&cru SCLK_UART4>, <&cru PCLK_UART4>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 8>, <&dmac0 9>; - pinctrl-0 = <&uart4m0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - uart5: serial@fe690000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfe690000 0x0 0x100>; - interrupts = ; - clocks = <&cru SCLK_UART5>, <&cru PCLK_UART5>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 10>, <&dmac0 11>; - pinctrl-0 = <&uart5m0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - uart6: serial@fe6a0000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfe6a0000 0x0 0x100>; - interrupts = ; - clocks = <&cru SCLK_UART6>, <&cru PCLK_UART6>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 12>, <&dmac0 13>; - pinctrl-0 = <&uart6m0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - uart7: serial@fe6b0000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfe6b0000 0x0 0x100>; - interrupts = ; - clocks = <&cru SCLK_UART7>, <&cru PCLK_UART7>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 14>, <&dmac0 15>; - pinctrl-0 = <&uart7m0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - uart8: serial@fe6c0000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfe6c0000 0x0 0x100>; - interrupts = ; - clocks = <&cru SCLK_UART8>, <&cru PCLK_UART8>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 16>, <&dmac0 17>; - pinctrl-0 = <&uart8m0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - uart9: serial@fe6d0000 { - compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; - reg = <0x0 0xfe6d0000 0x0 0x100>; - interrupts = ; - clocks = <&cru SCLK_UART9>, <&cru PCLK_UART9>; - clock-names = "baudclk", "apb_pclk"; - dmas = <&dmac0 18>, <&dmac0 19>; - pinctrl-0 = <&uart9m0_xfer>; - pinctrl-names = "default"; - reg-io-width = <4>; - reg-shift = <2>; - status = "disabled"; - }; - - pinctrl: pinctrl { - compatible = "rockchip,rk3568-pinctrl"; + gmac0: ethernet@fe2a0000 { + compatible = "rockchip,rk3568-gmac", "snps,dwmac-4.20a"; + reg = <0x0 0xfe2a0000 0x0 0x10000>; + interrupts = , + ; + interrupt-names = "macirq", "eth_wake_irq"; + clocks = <&cru SCLK_GMAC0>, <&cru SCLK_GMAC0_RX_TX>, + <&cru SCLK_GMAC0_RX_TX>, <&cru CLK_MAC0_REFOUT>, + <&cru ACLK_GMAC0>, <&cru PCLK_GMAC0>, + <&cru SCLK_GMAC0_RX_TX>, <&cru CLK_GMAC0_PTP_REF>; + clock-names = "stmmaceth", "mac_clk_rx", + "mac_clk_tx", "clk_mac_refout", + "aclk_mac", "pclk_mac", + "clk_mac_speed", "ptp_ref"; + resets = <&cru SRST_A_GMAC0>; + reset-names = "stmmaceth"; rockchip,grf = <&grf>; - rockchip,pmu = <&pmugrf>; - #address-cells = <2>; - #size-cells = <2>; - ranges; + snps,axi-config = <&gmac0_stmmac_axi_setup>; + snps,mixed-burst; + snps,mtl-rx-config = <&gmac0_mtl_rx_setup>; + snps,mtl-tx-config = <&gmac0_mtl_tx_setup>; + snps,tso; + status = "disabled"; - gpio0: gpio@fdd60000 { - compatible = "rockchip,gpio-bank"; - reg = <0x0 0xfdd60000 0x0 0x100>; - interrupts = ; - clocks = <&pmucru PCLK_GPIO0>; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; + mdio0: mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <0x1>; + #size-cells = <0x0>; }; - gpio1: gpio@fe740000 { - compatible = "rockchip,gpio-bank"; - reg = <0x0 0xfe740000 0x0 0x100>; - interrupts = ; - clocks = <&cru PCLK_GPIO1>; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; + gmac0_stmmac_axi_setup: stmmac-axi-config { + snps,blen = <0 0 0 0 16 8 4>; + snps,rd_osr_lmt = <8>; + snps,wr_osr_lmt = <4>; }; - gpio2: gpio@fe750000 { - compatible = "rockchip,gpio-bank"; - reg = <0x0 0xfe750000 0x0 0x100>; - interrupts = ; - clocks = <&cru PCLK_GPIO2>; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; + gmac0_mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <1>; + queue0 {}; }; - gpio3: gpio@fe760000 { - compatible = "rockchip,gpio-bank"; - reg = <0x0 0xfe760000 0x0 0x100>; - interrupts = ; - clocks = <&cru PCLK_GPIO3>; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - }; - - gpio4: gpio@fe770000 { - compatible = "rockchip,gpio-bank"; - reg = <0x0 0xfe770000 0x0 0x100>; - interrupts = ; - clocks = <&cru PCLK_GPIO4>; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; + gmac0_mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <1>; + queue0 {}; }; }; }; -#include "rk3568-pinctrl.dtsi" +&cpu0_opp_table { + opp-1992000000 { + opp-hz = /bits/ 64 <1992000000>; + opp-microvolt = <1150000 1150000 1150000>; + }; +}; + +&power { + power-domain@RK3568_PD_PIPE { + reg = ; + clocks = <&cru PCLK_PIPE>; + pm_qos = <&qos_pcie2x1>, + <&qos_pcie3x1>, + <&qos_pcie3x2>, + <&qos_sata0>, + <&qos_sata1>, + <&qos_sata2>, + <&qos_usb3_0>, + <&qos_usb3_1>; + #power-domain-cells = <0>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk356x.dtsi b/arch/arm64/boot/dts/rockchip/rk356x.dtsi index a68033a239..8ccce54ee8 100644 --- a/arch/arm64/boot/dts/rockchip/rk356x.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk356x.dtsi @@ -651,7 +651,7 @@ &i2s1m0_sdo0 &i2s1m0_sdo1 status = "disabled"; }; - dmac0: dmac@fe530000 { + dmac0: dma-controller@fe530000 { compatible = "arm,pl330", "arm,primecell"; reg = <0x0 0xfe530000 0x0 0x4000>; interrupts = , @@ -662,7 +662,7 @@ dmac0: dmac@fe530000 { #dma-cells = <1>; }; - dmac1: dmac@fe550000 { + dmac1: dma-controller@fe550000 { compatible = "arm,pl330", "arm,primecell"; reg = <0x0 0xfe550000 0x0 0x4000>; interrupts = , diff --git a/arch/arm64/boot/dts/ti/Makefile b/arch/arm64/boot/dts/ti/Makefile index d56c742f5a..90be511bc4 100644 --- a/arch/arm64/boot/dts/ti/Makefile +++ b/arch/arm64/boot/dts/ti/Makefile @@ -8,12 +8,16 @@ dtb-$(CONFIG_ARCH_K3) += k3-am654-base-board.dtb dtb-$(CONFIG_ARCH_K3) += k3-am6528-iot2050-basic.dtb +dtb-$(CONFIG_ARCH_K3) += k3-am6528-iot2050-basic-pg2.dtb dtb-$(CONFIG_ARCH_K3) += k3-am6548-iot2050-advanced.dtb +dtb-$(CONFIG_ARCH_K3) += k3-am6548-iot2050-advanced-pg2.dtb dtb-$(CONFIG_ARCH_K3) += k3-j721e-common-proc-board.dtb +dtb-$(CONFIG_ARCH_K3) += k3-j721e-sk.dtb dtb-$(CONFIG_ARCH_K3) += k3-j7200-common-proc-board.dtb -dtb-$(CONFIG_ARCH_K3) += k3-am642-evm.dtb +dtb-$(CONFIG_ARCH_K3) += k3-j721s2-common-proc-board.dtb +dtb-$(CONFIG_ARCH_K3) += k3-am642-evm.dtb dtb-$(CONFIG_ARCH_K3) += k3-am642-sk.dtb diff --git a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi index 42d1d219a3..012011dc61 100644 --- a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi @@ -564,6 +564,14 @@ cpts@39000000 { ti,cpts-ext-ts-inputs = <8>; }; + timesync_router: pinctrl@a40000 { + compatible = "pinctrl-single"; + reg = <0x0 0xa40000 0x0 0x800>; + #pinctrl-cells = <1>; + pinctrl-single,register-width = <32>; + pinctrl-single,function-mask = <0x000107ff>; + }; + usbss0: cdns-usb@f900000{ compatible = "ti,am64-usb"; reg = <0x00 0xf900000 0x00 0x100>; @@ -973,4 +981,312 @@ ecap2: pwm@23120000 { clocks = <&k3_clks 53 0>; clock-names = "fck"; }; + + icssg0: icssg@30000000 { + compatible = "ti,am642-icssg"; + reg = <0x00 0x30000000 0x00 0x80000>; + power-domains = <&k3_pds 81 TI_SCI_PD_EXCLUSIVE>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x00 0x30000000 0x80000>; + + icssg0_mem: memories@0 { + reg = <0x0 0x2000>, + <0x2000 0x2000>, + <0x10000 0x10000>; + reg-names = "dram0", "dram1", "shrdram2"; + }; + + icssg0_cfg: cfg@26000 { + compatible = "ti,pruss-cfg", "syscon"; + reg = <0x26000 0x200>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x26000 0x2000>; + + clocks { + #address-cells = <1>; + #size-cells = <0>; + + icssg0_coreclk_mux: coreclk-mux@3c { + reg = <0x3c>; + #clock-cells = <0>; + clocks = <&k3_clks 81 0>, /* icssg0_core_clk */ + <&k3_clks 81 20>; /* icssg0_iclk */ + assigned-clocks = <&icssg0_coreclk_mux>; + assigned-clock-parents = <&k3_clks 81 20>; + }; + + icssg0_iepclk_mux: iepclk-mux@30 { + reg = <0x30>; + #clock-cells = <0>; + clocks = <&k3_clks 81 3>, /* icssg0_iep_clk */ + <&icssg0_coreclk_mux>; /* icssg0_coreclk_mux */ + assigned-clocks = <&icssg0_iepclk_mux>; + assigned-clock-parents = <&icssg0_coreclk_mux>; + }; + }; + }; + + icssg0_mii_rt: mii-rt@32000 { + compatible = "ti,pruss-mii", "syscon"; + reg = <0x32000 0x100>; + }; + + icssg0_mii_g_rt: mii-g-rt@33000 { + compatible = "ti,pruss-mii-g", "syscon"; + reg = <0x33000 0x1000>; + }; + + icssg0_intc: interrupt-controller@20000 { + compatible = "ti,icssg-intc"; + reg = <0x20000 0x2000>; + interrupt-controller; + #interrupt-cells = <3>; + interrupts = , + , + , + , + , + , + , + ; + interrupt-names = "host_intr0", "host_intr1", + "host_intr2", "host_intr3", + "host_intr4", "host_intr5", + "host_intr6", "host_intr7"; + }; + + pru0_0: pru@34000 { + compatible = "ti,am642-pru"; + reg = <0x34000 0x3000>, + <0x22000 0x100>, + <0x22400 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-pru0_0-fw"; + }; + + rtu0_0: rtu@4000 { + compatible = "ti,am642-rtu"; + reg = <0x4000 0x2000>, + <0x23000 0x100>, + <0x23400 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-rtu0_0-fw"; + }; + + tx_pru0_0: txpru@a000 { + compatible = "ti,am642-tx-pru"; + reg = <0xa000 0x1800>, + <0x25000 0x100>, + <0x25400 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-txpru0_0-fw"; + }; + + pru0_1: pru@38000 { + compatible = "ti,am642-pru"; + reg = <0x38000 0x3000>, + <0x24000 0x100>, + <0x24400 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-pru0_1-fw"; + }; + + rtu0_1: rtu@6000 { + compatible = "ti,am642-rtu"; + reg = <0x6000 0x2000>, + <0x23800 0x100>, + <0x23c00 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-rtu0_1-fw"; + }; + + tx_pru0_1: txpru@c000 { + compatible = "ti,am642-tx-pru"; + reg = <0xc000 0x1800>, + <0x25800 0x100>, + <0x25c00 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-txpru0_1-fw"; + }; + + icssg0_mdio: mdio@32400 { + compatible = "ti,davinci_mdio"; + reg = <0x32400 0x100>; + clocks = <&k3_clks 62 3>; + clock-names = "fck"; + #address-cells = <1>; + #size-cells = <0>; + bus_freq = <1000000>; + }; + }; + + icssg1: icssg@30080000 { + compatible = "ti,am642-icssg"; + reg = <0x00 0x30080000 0x00 0x80000>; + power-domains = <&k3_pds 82 TI_SCI_PD_EXCLUSIVE>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x00 0x30080000 0x80000>; + + icssg1_mem: memories@0 { + reg = <0x0 0x2000>, + <0x2000 0x2000>, + <0x10000 0x10000>; + reg-names = "dram0", "dram1", "shrdram2"; + }; + + icssg1_cfg: cfg@26000 { + compatible = "ti,pruss-cfg", "syscon"; + reg = <0x26000 0x200>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x26000 0x2000>; + + clocks { + #address-cells = <1>; + #size-cells = <0>; + + icssg1_coreclk_mux: coreclk-mux@3c { + reg = <0x3c>; + #clock-cells = <0>; + clocks = <&k3_clks 82 0>, /* icssg1_core_clk */ + <&k3_clks 82 20>; /* icssg1_iclk */ + assigned-clocks = <&icssg1_coreclk_mux>; + assigned-clock-parents = <&k3_clks 82 20>; + }; + + icssg1_iepclk_mux: iepclk-mux@30 { + reg = <0x30>; + #clock-cells = <0>; + clocks = <&k3_clks 82 3>, /* icssg1_iep_clk */ + <&icssg1_coreclk_mux>; /* icssg1_coreclk_mux */ + assigned-clocks = <&icssg1_iepclk_mux>; + assigned-clock-parents = <&icssg1_coreclk_mux>; + }; + }; + }; + + icssg1_mii_rt: mii-rt@32000 { + compatible = "ti,pruss-mii", "syscon"; + reg = <0x32000 0x100>; + }; + + icssg1_mii_g_rt: mii-g-rt@33000 { + compatible = "ti,pruss-mii-g", "syscon"; + reg = <0x33000 0x1000>; + }; + + icssg1_intc: interrupt-controller@20000 { + compatible = "ti,icssg-intc"; + reg = <0x20000 0x2000>; + interrupt-controller; + #interrupt-cells = <3>; + interrupts = , + , + , + , + , + , + , + ; + interrupt-names = "host_intr0", "host_intr1", + "host_intr2", "host_intr3", + "host_intr4", "host_intr5", + "host_intr6", "host_intr7"; + }; + + pru1_0: pru@34000 { + compatible = "ti,am642-pru"; + reg = <0x34000 0x4000>, + <0x22000 0x100>, + <0x22400 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-pru1_0-fw"; + }; + + rtu1_0: rtu@4000 { + compatible = "ti,am642-rtu"; + reg = <0x4000 0x2000>, + <0x23000 0x100>, + <0x23400 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-rtu1_0-fw"; + }; + + tx_pru1_0: txpru@a000 { + compatible = "ti,am642-tx-pru"; + reg = <0xa000 0x1800>, + <0x25000 0x100>, + <0x25400 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-txpru1_0-fw"; + }; + + pru1_1: pru@38000 { + compatible = "ti,am642-pru"; + reg = <0x38000 0x4000>, + <0x24000 0x100>, + <0x24400 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-pru1_1-fw"; + }; + + rtu1_1: rtu@6000 { + compatible = "ti,am642-rtu"; + reg = <0x6000 0x2000>, + <0x23800 0x100>, + <0x23c00 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-rtu1_1-fw"; + }; + + tx_pru1_1: txpru@c000 { + compatible = "ti,am642-tx-pru"; + reg = <0xc000 0x1800>, + <0x25800 0x100>, + <0x25c00 0x100>; + reg-names = "iram", "control", "debug"; + firmware-name = "am64x-txpru1_1-fw"; + }; + + icssg1_mdio: mdio@32400 { + compatible = "ti,davinci_mdio"; + reg = <0x32400 0x100>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&k3_clks 82 0>; + clock-names = "fck"; + bus_freq = <1000000>; + }; + }; + + main_mcan0: can@20701000 { + compatible = "bosch,m_can"; + reg = <0x00 0x20701000 0x00 0x200>, + <0x00 0x20708000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 98 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 98 5>, <&k3_clks 98 0>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan1: can@20711000 { + compatible = "bosch,m_can"; + reg = <0x00 0x20711000 0x00 0x200>, + <0x00 0x20718000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 99 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 99 5>, <&k3_clks 99 0>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; }; diff --git a/arch/arm64/boot/dts/ti/k3-am64-mcu.dtsi b/arch/arm64/boot/dts/ti/k3-am64-mcu.dtsi index 59cc58f7d0..2bb5c9ff17 100644 --- a/arch/arm64/boot/dts/ti/k3-am64-mcu.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am64-mcu.dtsi @@ -97,4 +97,12 @@ mcu_gpio0: gpio@4201000 { clocks = <&k3_clks 79 0>; clock-names = "gpio"; }; + + mcu_pmx0: pinctrl@4084000 { + compatible = "pinctrl-single"; + reg = <0x00 0x4084000 0x00 0x84>; + #pinctrl-cells = <1>; + pinctrl-single,register-width = <32>; + pinctrl-single,function-mask = <0xffffffff>; + }; }; diff --git a/arch/arm64/boot/dts/ti/k3-am64.dtsi b/arch/arm64/boot/dts/ti/k3-am64.dtsi index de6805b0c7..120974726b 100644 --- a/arch/arm64/boot/dts/ti/k3-am64.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am64.dtsi @@ -30,6 +30,8 @@ aliases { serial8 = &main_uart6; ethernet0 = &cpsw_port1; ethernet1 = &cpsw_port2; + mmc0 = &sdhci0; + mmc1 = &sdhci1; }; chosen { }; diff --git a/arch/arm64/boot/dts/ti/k3-am642-evm.dts b/arch/arm64/boot/dts/ti/k3-am642-evm.dts index 24ce494261..e94ae178b1 100644 --- a/arch/arm64/boot/dts/ti/k3-am642-evm.dts +++ b/arch/arm64/boot/dts/ti/k3-am642-evm.dts @@ -184,6 +184,20 @@ cpsw3g_phy3: ethernet-phy@3 { }; }; }; + + transceiver1: can-phy0 { + compatible = "ti,tcan1042"; + #phy-cells = <0>; + max-bitrate = <5000000>; + standby-gpios = <&exp1 8 GPIO_ACTIVE_HIGH>; + }; + + transceiver2: can-phy1 { + compatible = "ti,tcan1042"; + #phy-cells = <0>; + max-bitrate = <5000000>; + standby-gpios = <&exp1 9 GPIO_ACTIVE_HIGH>; + }; }; &main_pmx0 { @@ -294,6 +308,20 @@ main_ecap0_pins_default: main-ecap0-pins-default { AM64X_IOPAD(0x0270, PIN_INPUT, 0) /* (D18) ECAP0_IN_APWM_OUT */ >; }; + + main_mcan0_pins_default: main-mcan0-pins-default { + pinctrl-single,pins = < + AM64X_IOPAD(0x0254, PIN_INPUT, 0) /* (B17) MCAN0_RX */ + AM64X_IOPAD(0x0250, PIN_OUTPUT, 0) /* (A17) MCAN0_TX */ + >; + }; + + main_mcan1_pins_default: main-mcan1-pins-default { + pinctrl-single,pins = < + AM64X_IOPAD(0x025c, PIN_INPUT, 0) /* (D17) MCAN1_RX */ + AM64X_IOPAD(0x0258, PIN_OUTPUT, 0) /* (C17) MCAN1_TX */ + >; + }; }; &main_uart0 { @@ -630,3 +658,23 @@ &epwm7 { &epwm8 { status = "disabled"; }; + +&icssg0_mdio { + status = "disabled"; +}; + +&icssg1_mdio { + status = "disabled"; +}; + +&main_mcan0 { + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan0_pins_default>; + phys = <&transceiver1>; +}; + +&main_mcan1 { + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan1_pins_default>; + phys = <&transceiver2>; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am642-sk.dts b/arch/arm64/boot/dts/ti/k3-am642-sk.dts index 6b45cdeeee..a9785bec12 100644 --- a/arch/arm64/boot/dts/ti/k3-am642-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-am642-sk.dts @@ -517,3 +517,19 @@ &epwm7 { &epwm8 { status = "disabled"; }; + +&icssg0_mdio { + status = "disabled"; +}; + +&icssg1_mdio { + status = "disabled"; +}; + +&main_mcan0 { + status = "disabled"; +}; + +&main_mcan1 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am65-iot2050-common.dtsi b/arch/arm64/boot/dts/ti/k3-am65-iot2050-common.dtsi index 1008e9162b..3079eaee01 100644 --- a/arch/arm64/boot/dts/ti/k3-am65-iot2050-common.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am65-iot2050-common.dtsi @@ -4,19 +4,19 @@ * * Authors: * Le Jin - * Jan Kiszka + * Jan Kiszka * - * Common bits of the IOT2050 Basic and Advanced variants + * Common bits of the IOT2050 Basic and Advanced variants, PG1 and PG2 */ -/dts-v1/; - #include "k3-am654.dtsi" #include / { aliases { spi0 = &mcu_spi0; + mmc0 = &sdhci1; + mmc1 = &sdhci0; }; chosen { @@ -555,7 +555,6 @@ &sdhci1 { pinctrl-0 = <&main_mmc1_pins_default>; ti,driver-strength-ohm = <50>; disable-wp; - no-1-8-v; }; &usb0 { @@ -629,10 +628,6 @@ dpi_out: endpoint { }; }; -&serdes0 { - status = "disabled"; -}; - &pcie0_rc { status = "disabled"; }; @@ -651,16 +646,34 @@ &pcie1_rc { reset-gpios = <&wkup_gpio0 27 GPIO_ACTIVE_HIGH>; }; +&m_can0 { + status = "disabled"; +}; + +&m_can1 { + status = "disabled"; +}; + &pcie1_ep { status = "disabled"; }; &mailbox0_cluster0 { - status = "disabled"; + interrupts = <436>; + + mbox_mcu_r5fss0_core0: mbox-mcu-r5fss0-core0 { + ti,mbox-tx = <1 0 0>; + ti,mbox-rx = <0 0 0>; + }; }; &mailbox0_cluster1 { - status = "disabled"; + interrupts = <432>; + + mbox_mcu_r5fss0_core1: mbox-mcu-r5fss0-core1 { + ti,mbox-tx = <1 0 0>; + ti,mbox-rx = <0 0 0>; + }; }; &mailbox0_cluster2 { @@ -703,6 +716,18 @@ &mailbox0_cluster11 { status = "disabled"; }; +&mcu_r5fss0_core0 { + memory-region = <&mcu_r5fss0_core0_dma_memory_region>, + <&mcu_r5fss0_core0_memory_region>; + mboxes = <&mailbox0_cluster0 &mbox_mcu_r5fss0_core0>; +}; + +&mcu_r5fss0_core1 { + memory-region = <&mcu_r5fss0_core1_dma_memory_region>, + <&mcu_r5fss0_core1_memory_region>; + mboxes = <&mailbox0_cluster1 &mbox_mcu_r5fss0_core1>; +}; + &icssg0_mdio { status = "disabled"; }; @@ -714,3 +739,15 @@ &icssg1_mdio { &icssg2_mdio { status = "disabled"; }; + +&mcasp0 { + status = "disabled"; +}; + +&mcasp1 { + status = "disabled"; +}; + +&mcasp2 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi index ba4e5d3e1e..ce8bb4a610 100644 --- a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi @@ -692,8 +692,8 @@ pcie0_rc: pcie@5500000 { power-domains = <&k3_pds 120 TI_SCI_PD_EXCLUSIVE>; #address-cells = <3>; #size-cells = <2>; - ranges = <0x81000000 0 0 0x0 0x10020000 0 0x00010000 - 0x82000000 0 0x10030000 0x0 0x10030000 0 0x07FD0000>; + ranges = <0x81000000 0 0 0x0 0x10020000 0 0x00010000>, + <0x82000000 0 0x10030000 0x0 0x10030000 0 0x07FD0000>; ti,syscon-pcie-id = <&pcie_devid>; ti,syscon-pcie-mode = <&pcie0_mode>; bus-range = <0x0 0xff>; @@ -725,8 +725,8 @@ pcie1_rc: pcie@5600000 { power-domains = <&k3_pds 121 TI_SCI_PD_EXCLUSIVE>; #address-cells = <3>; #size-cells = <2>; - ranges = <0x81000000 0 0 0x0 0x18020000 0 0x00010000 - 0x82000000 0 0x18030000 0x0 0x18030000 0 0x07FD0000>; + ranges = <0x81000000 0 0 0x0 0x18020000 0 0x00010000>, + <0x82000000 0 0x18030000 0x0 0x18030000 0 0x07FD0000>; ti,syscon-pcie-id = <&pcie_devid>; ti,syscon-pcie-mode = <&pcie1_mode>; bus-range = <0x0 0xff>; diff --git a/arch/arm64/boot/dts/ti/k3-am65-mcu.dtsi b/arch/arm64/boot/dts/ti/k3-am65-mcu.dtsi index c93ff1520a..8d592bf41d 100644 --- a/arch/arm64/boot/dts/ti/k3-am65-mcu.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am65-mcu.dtsi @@ -159,6 +159,36 @@ mcu_udmap: dma-controller@285c0000 { }; }; + m_can0: mcan@40528000 { + compatible = "bosch,m_can"; + reg = <0x0 0x40528000 0x0 0x400>, + <0x0 0x40500000 0x0 0x4400>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 102 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 102 5>, <&k3_clks 102 0>; + clock-names = "hclk", "cclk"; + interrupt-parent = <&gic500>; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + m_can1: mcan@40568000 { + compatible = "bosch,m_can"; + reg = <0x0 0x40568000 0x0 0x400>, + <0x0 0x40540000 0x0 0x4400>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 103 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 103 5>, <&k3_clks 103 0>; + clock-names = "hclk", "cclk"; + interrupt-parent = <&gic500>; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + fss: fss@47000000 { compatible = "simple-bus"; #address-cells = <2>; diff --git a/arch/arm64/boot/dts/ti/k3-am65-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-am65-wakeup.dtsi index 9d21cdf6fc..9c69d0917f 100644 --- a/arch/arm64/boot/dts/ti/k3-am65-wakeup.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am65-wakeup.dtsi @@ -100,8 +100,4 @@ wkup_vtm0: temperature-sensor@42050000 { power-domains = <&k3_pds 80 TI_SCI_PD_EXCLUSIVE>; #thermal-sensor-cells = <1>; }; - - thermal_zones: thermal-zones { - #include "k3-am654-industrial-thermal.dtsi" - }; }; diff --git a/arch/arm64/boot/dts/ti/k3-am65.dtsi b/arch/arm64/boot/dts/ti/k3-am65.dtsi index a9fc1af03f..a58a39fa42 100644 --- a/arch/arm64/boot/dts/ti/k3-am65.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am65.dtsi @@ -31,6 +31,8 @@ aliases { i2c4 = &main_i2c2; i2c5 = &main_i2c3; ethernet0 = &cpsw_port1; + mmc0 = &sdhci0; + mmc1 = &sdhci1; }; chosen { }; diff --git a/arch/arm64/boot/dts/ti/k3-am6528-iot2050-basic.dts b/arch/arm64/boot/dts/ti/k3-am6528-iot2050-basic.dts index 94bb5dd391..87928ff282 100644 --- a/arch/arm64/boot/dts/ti/k3-am6528-iot2050-basic.dts +++ b/arch/arm64/boot/dts/ti/k3-am6528-iot2050-basic.dts @@ -4,63 +4,21 @@ * * Authors: * Le Jin - * Jan Kiszka + * Jan Kiszka * - * AM6528-based (dual-core) IOT2050 Basic variant + * AM6528-based (dual-core) IOT2050 Basic variant, Product Generation 1 * 1 GB RAM, no eMMC, main_uart0 on connector X30 + * + * Product homepage: + * https://new.siemens.com/global/en/products/automation/pc-based/iot-gateways/simatic-iot2050.html */ /dts-v1/; -#include "k3-am65-iot2050-common.dtsi" +#include "k3-am6528-iot2050-basic-common.dtsi" +#include "k3-am65-iot2050-common-pg1.dtsi" / { compatible = "siemens,iot2050-basic", "ti,am654"; model = "SIMATIC IOT2050 Basic"; - - memory@80000000 { - device_type = "memory"; - /* 1G RAM */ - reg = <0x00000000 0x80000000 0x00000000 0x40000000>; - }; - - cpus { - cpu-map { - /delete-node/ cluster1; - }; - /delete-node/ cpu@100; - /delete-node/ cpu@101; - }; - - /delete-node/ l2-cache1; -}; - -/* eMMC */ -&sdhci0 { - status = "disabled"; -}; - -&main_pmx0 { - main_uart0_pins_default: main-uart0-pins-default { - pinctrl-single,pins = < - AM65X_IOPAD(0x01e4, PIN_INPUT, 0) /* (AF11) UART0_RXD */ - AM65X_IOPAD(0x01e8, PIN_OUTPUT, 0) /* (AE11) UART0_TXD */ - AM65X_IOPAD(0x01ec, PIN_INPUT, 0) /* (AG11) UART0_CTSn */ - AM65X_IOPAD(0x01f0, PIN_OUTPUT, 0) /* (AD11) UART0_RTSn */ - AM65X_IOPAD(0x0188, PIN_INPUT, 1) /* (D25) UART0_DCDn */ - AM65X_IOPAD(0x018c, PIN_INPUT, 1) /* (B26) UART0_DSRn */ - AM65X_IOPAD(0x0190, PIN_OUTPUT, 1) /* (A24) UART0_DTRn */ - AM65X_IOPAD(0x0194, PIN_INPUT, 1) /* (E24) UART0_RIN */ - >; - }; -}; - -&main_uart0 { - pinctrl-names = "default"; - pinctrl-0 = <&main_uart0_pins_default>; -}; - -&mcu_r5fss0 { - /* lock-step mode not supported on this board */ - ti,cluster-mode = <0>; }; diff --git a/arch/arm64/boot/dts/ti/k3-am654-base-board.dts b/arch/arm64/boot/dts/ti/k3-am654-base-board.dts index cfbcebfa37..9043f91c9b 100644 --- a/arch/arm64/boot/dts/ti/k3-am654-base-board.dts +++ b/arch/arm64/boot/dts/ti/k3-am654-base-board.dts @@ -416,6 +416,14 @@ &pcie1_ep { status = "disabled"; }; +&m_can0 { + status = "disabled"; +}; + +&m_can1 { + status = "disabled"; +}; + &mailbox0_cluster0 { interrupts = <436>; diff --git a/arch/arm64/boot/dts/ti/k3-am654.dtsi b/arch/arm64/boot/dts/ti/k3-am654.dtsi index f0a6541b80..a892579000 100644 --- a/arch/arm64/boot/dts/ti/k3-am654.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am654.dtsi @@ -112,4 +112,8 @@ msmc_l3: l3-cache0 { compatible = "cache"; cache-level = <3>; }; + + thermal_zones: thermal-zones { + #include "k3-am654-industrial-thermal.dtsi" + }; }; diff --git a/arch/arm64/boot/dts/ti/k3-am6548-iot2050-advanced.dts b/arch/arm64/boot/dts/ti/k3-am6548-iot2050-advanced.dts index ec9617c13c..077f165bdc 100644 --- a/arch/arm64/boot/dts/ti/k3-am6548-iot2050-advanced.dts +++ b/arch/arm64/boot/dts/ti/k3-am6548-iot2050-advanced.dts @@ -4,57 +4,21 @@ * * Authors: * Le Jin - * Jan Kiszka + * Jan Kiszka * - * AM6548-based (quad-core) IOT2050 Advanced variant + * AM6548-based (quad-core) IOT2050 Advanced variant, Product Generation 1 * 2 GB RAM, 16 GB eMMC, USB-serial converter on connector X30 + * + * Product homepage: + * https://new.siemens.com/global/en/products/automation/pc-based/iot-gateways/simatic-iot2050.html */ /dts-v1/; -#include "k3-am65-iot2050-common.dtsi" +#include "k3-am6548-iot2050-advanced-common.dtsi" +#include "k3-am65-iot2050-common-pg1.dtsi" / { compatible = "siemens,iot2050-advanced", "ti,am654"; model = "SIMATIC IOT2050 Advanced"; - - memory@80000000 { - device_type = "memory"; - /* 2G RAM */ - reg = <0x00000000 0x80000000 0x00000000 0x80000000>; - }; -}; - -&main_pmx0 { - main_mmc0_pins_default: main-mmc0-pins-default { - pinctrl-single,pins = < - AM65X_IOPAD(0x01a8, PIN_INPUT_PULLDOWN, 0) /* (B25) MMC0_CLK */ - AM65X_IOPAD(0x01ac, PIN_INPUT_PULLUP, 0) /* (B27) MMC0_CMD */ - AM65X_IOPAD(0x01a4, PIN_INPUT_PULLUP, 0) /* (A26) MMC0_DAT0 */ - AM65X_IOPAD(0x01a0, PIN_INPUT_PULLUP, 0) /* (E25) MMC0_DAT1 */ - AM65X_IOPAD(0x019c, PIN_INPUT_PULLUP, 0) /* (C26) MMC0_DAT2 */ - AM65X_IOPAD(0x0198, PIN_INPUT_PULLUP, 0) /* (A25) MMC0_DAT3 */ - AM65X_IOPAD(0x0194, PIN_INPUT_PULLUP, 0) /* (E24) MMC0_DAT4 */ - AM65X_IOPAD(0x0190, PIN_INPUT_PULLUP, 0) /* (A24) MMC0_DAT5 */ - AM65X_IOPAD(0x018c, PIN_INPUT_PULLUP, 0) /* (B26) MMC0_DAT6 */ - AM65X_IOPAD(0x0188, PIN_INPUT_PULLUP, 0) /* (D25) MMC0_DAT7 */ - AM65X_IOPAD(0x01b8, PIN_OUTPUT_PULLUP, 7) /* (B23) MMC0_SDWP */ - AM65X_IOPAD(0x01b4, PIN_INPUT_PULLUP, 0) /* (A23) MMC0_SDCD */ - AM65X_IOPAD(0x01b0, PIN_INPUT, 0) /* (C25) MMC0_DS */ - >; - }; -}; - -/* eMMC */ -&sdhci0 { - pinctrl-names = "default"; - pinctrl-0 = <&main_mmc0_pins_default>; - bus-width = <8>; - non-removable; - ti,driver-strength-ohm = <50>; - disable-wp; -}; - -&main_uart0 { - status = "disabled"; }; diff --git a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts index d14f3c18b6..121975dc82 100644 --- a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts +++ b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts @@ -12,6 +12,9 @@ #include / { + compatible = "ti,j7200-evm", "ti,j7200"; + model = "Texas Instruments J7200 EVM"; + chosen { stdout-path = "serial2:115200n8"; bootargs = "console=ttyS2,115200n8 earlycon=ns16550a,mmio32,0x02800000"; diff --git a/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi b/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi index 7daa280220..05a627ad6c 100644 --- a/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi @@ -633,6 +633,7 @@ pcie1_ep: pcie-ep@2910000 { clocks = <&k3_clks 240 6>; clock-names = "fck"; max-functions = /bits/ 8 <6>; + max-virtual-functions = /bits/ 8 <4 4 4 4 0 0>; dma-coherent; }; diff --git a/arch/arm64/boot/dts/ti/k3-j7200.dtsi b/arch/arm64/boot/dts/ti/k3-j7200.dtsi index 7586b5aea4..64fef4e67d 100644 --- a/arch/arm64/boot/dts/ti/k3-j7200.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j7200.dtsi @@ -30,6 +30,8 @@ aliases { serial9 = &main_uart7; serial10 = &main_uart8; serial11 = &main_uart9; + mmc0 = &main_sdhci0; + mmc1 = &main_sdhci1; }; chosen { }; diff --git a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts index 8bd02d9e28..2d7596911b 100644 --- a/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts +++ b/arch/arm64/boot/dts/ti/k3-j721e-common-proc-board.dts @@ -12,6 +12,9 @@ #include / { + compatible = "ti,j721e-evm", "ti,j721e"; + model = "Texas Instruments J721e EVM"; + chosen { stdout-path = "serial2:115200n8"; bootargs = "console=ttyS2,115200n8 earlycon=ns16550a,mmio32,0x02800000"; @@ -109,6 +112,42 @@ sound0: sound@0 { "cpb-codec-scki", "cpb-codec-scki-48000", "cpb-codec-scki-44100"; }; + + transceiver1: can-phy0 { + compatible = "ti,tcan1043"; + #phy-cells = <0>; + max-bitrate = <5000000>; + pinctrl-names = "default"; + pinctrl-0 = <&mcu_mcan0_gpio_pins_default>; + standby-gpios = <&wkup_gpio0 54 GPIO_ACTIVE_LOW>; + enable-gpios = <&wkup_gpio0 0 GPIO_ACTIVE_HIGH>; + }; + + transceiver2: can-phy1 { + compatible = "ti,tcan1042"; + #phy-cells = <0>; + max-bitrate = <5000000>; + pinctrl-names = "default"; + pinctrl-0 = <&mcu_mcan1_gpio_pins_default>; + standby-gpios = <&wkup_gpio0 2 GPIO_ACTIVE_HIGH>; + }; + + transceiver3: can-phy2 { + compatible = "ti,tcan1043"; + #phy-cells = <0>; + max-bitrate = <5000000>; + standby-gpios = <&exp2 7 GPIO_ACTIVE_LOW>; + enable-gpios = <&exp2 6 GPIO_ACTIVE_HIGH>; + }; + + transceiver4: can-phy3 { + compatible = "ti,tcan1042"; + #phy-cells = <0>; + max-bitrate = <5000000>; + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan2_gpio_pins_default>; + standby-gpios = <&main_gpio0 127 GPIO_ACTIVE_HIGH>; + }; }; &main_pmx0 { @@ -204,6 +243,26 @@ audi_ext_refclk2_pins_default: audi-ext-refclk2-pins-default { J721E_IOPAD(0x1a4, PIN_OUTPUT, 3) /* (W26) RGMII6_RXC.AUDIO_EXT_REFCLK2 */ >; }; + + main_mcan0_pins_default: main-mcan0-pins-default { + pinctrl-single,pins = < + J721E_IOPAD(0x208, PIN_INPUT, 0) /* (W5) MCAN0_RX */ + J721E_IOPAD(0x20c, PIN_OUTPUT, 0) /* (W6) MCAN0_TX */ + >; + }; + + main_mcan2_pins_default: main-mcan2-pins-default { + pinctrl-single,pins = < + J721E_IOPAD(0x01f0, PIN_INPUT, 3) /* (AC2) MCAN2_RX.GPIO0_123 */ + J721E_IOPAD(0x01f4, PIN_OUTPUT, 3) /* (AB1) MCAN2_TX.GPIO0_124 */ + >; + }; + + main_mcan2_gpio_pins_default: main-mcan2-gpio-pins-default { + pinctrl-single,pins = < + J721E_IOPAD(0x200, PIN_INPUT, 7) /* (AC4) UART1_CTSn.GPIO0_127 */ + >; + }; }; &wkup_pmx0 { @@ -249,6 +308,33 @@ J721E_WKUP_IOPAD(0x008c, PIN_OUTPUT, 0) /* MCU_MDIO0_MDC */ J721E_WKUP_IOPAD(0x0088, PIN_INPUT, 0) /* MCU_MDIO0_MDIO */ >; }; + + mcu_mcan0_pins_default: mcu-mcan0-pins-default { + pinctrl-single,pins = < + J721E_WKUP_IOPAD(0xac, PIN_INPUT, 0) /* (C29) MCU_MCAN0_RX */ + J721E_WKUP_IOPAD(0xa8, PIN_OUTPUT, 0) /* (D29) MCU_MCAN0_TX */ + >; + }; + + mcu_mcan0_gpio_pins_default: mcu-mcan0-gpio-pins-default { + pinctrl-single,pins = < + J721E_WKUP_IOPAD(0xb0, PIN_INPUT, 7) /* (F26) WKUP_GPIO0_0 */ + J721E_WKUP_IOPAD(0x98, PIN_INPUT, 7) /* (E28) MCU_SPI0_D1.WKUP_GPIO0_54 */ + >; + }; + + mcu_mcan1_pins_default: mcu-mcan1-pins-default { + pinctrl-single,pins = < + J721E_WKUP_IOPAD(0xc4, PIN_INPUT, 0) /* (G24) WKUP_GPIO0_5.MCU_MCAN1_RX */ + J721E_WKUP_IOPAD(0xc0, PIN_OUTPUT, 0) /* (G25) WKUP_GPIO0_4.MCU_MCAN1_TX */ + >; + }; + + mcu_mcan1_gpio_pins_default: mcu-mcan1-gpio-pins-default { + pinctrl-single,pins = < + J721E_WKUP_IOPAD(0xb8, PIN_INPUT, 7) /* (F28) WKUP_GPIO0_2 */ + >; + }; }; &wkup_uart0 { @@ -770,3 +856,75 @@ &icssg0_mdio { &icssg1_mdio { status = "disabled"; }; + +&mcu_mcan0 { + pinctrl-names = "default"; + pinctrl-0 = <&mcu_mcan0_pins_default>; + phys = <&transceiver1>; +}; + +&mcu_mcan1 { + pinctrl-names = "default"; + pinctrl-0 = <&mcu_mcan1_pins_default>; + phys = <&transceiver2>; +}; + +&main_mcan0 { + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan0_pins_default>; + phys = <&transceiver3>; +}; + +&main_mcan1 { + status = "disabled"; +}; + +&main_mcan2 { + pinctrl-names = "default"; + pinctrl-0 = <&main_mcan2_pins_default>; + phys = <&transceiver4>; +}; + +&main_mcan3 { + status = "disabled"; +}; + +&main_mcan4 { + status = "disabled"; +}; + +&main_mcan5 { + status = "disabled"; +}; + +&main_mcan6 { + status = "disabled"; +}; + +&main_mcan7 { + status = "disabled"; +}; + +&main_mcan8 { + status = "disabled"; +}; + +&main_mcan9 { + status = "disabled"; +}; + +&main_mcan10 { + status = "disabled"; +}; + +&main_mcan11 { + status = "disabled"; +}; + +&main_mcan12 { + status = "disabled"; +}; + +&main_mcan13 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi index e85c89eebf..599861259a 100644 --- a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi @@ -1940,4 +1940,200 @@ icssg1_mdio: mdio@32400 { bus_freq = <1000000>; }; }; + + main_mcan0: can@2701000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02701000 0x00 0x200>, + <0x00 0x02708000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 156 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 156 0>, <&k3_clks 156 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan1: can@2711000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02711000 0x00 0x200>, + <0x00 0x02718000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 158 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 158 0>, <&k3_clks 158 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan2: can@2721000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02721000 0x00 0x200>, + <0x00 0x02728000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 160 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 160 0>, <&k3_clks 160 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan3: can@2731000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02731000 0x00 0x200>, + <0x00 0x02738000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 161 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 161 0>, <&k3_clks 161 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan4: can@2741000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02741000 0x00 0x200>, + <0x00 0x02748000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 162 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 162 0>, <&k3_clks 162 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan5: can@2751000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02751000 0x00 0x200>, + <0x00 0x02758000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 163 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 163 0>, <&k3_clks 163 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan6: can@2761000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02761000 0x00 0x200>, + <0x00 0x02768000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 164 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 164 0>, <&k3_clks 164 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan7: can@2771000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02771000 0x00 0x200>, + <0x00 0x02778000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 165 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 165 0>, <&k3_clks 165 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan8: can@2781000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02781000 0x00 0x200>, + <0x00 0x02788000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 166 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 166 0>, <&k3_clks 166 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan9: can@2791000 { + compatible = "bosch,m_can"; + reg = <0x00 0x02791000 0x00 0x200>, + <0x00 0x02798000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 167 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 167 0>, <&k3_clks 167 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan10: can@27a1000 { + compatible = "bosch,m_can"; + reg = <0x00 0x027a1000 0x00 0x200>, + <0x00 0x027a8000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 168 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 168 0>, <&k3_clks 168 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan11: can@27b1000 { + compatible = "bosch,m_can"; + reg = <0x00 0x027b1000 0x00 0x200>, + <0x00 0x027b8000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 169 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 169 0>, <&k3_clks 169 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan12: can@27c1000 { + compatible = "bosch,m_can"; + reg = <0x00 0x027c1000 0x00 0x200>, + <0x00 0x027c8000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 170 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 170 0>, <&k3_clks 170 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + main_mcan13: can@27d1000 { + compatible = "bosch,m_can"; + reg = <0x00 0x027d1000 0x00 0x200>, + <0x00 0x027d8000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 171 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 171 0>, <&k3_clks 171 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; }; diff --git a/arch/arm64/boot/dts/ti/k3-j721e-mcu-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-mcu-wakeup.dtsi index d2dceda72f..b4972dfb7d 100644 --- a/arch/arm64/boot/dts/ti/k3-j721e-mcu-wakeup.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j721e-mcu-wakeup.dtsi @@ -390,4 +390,32 @@ mcu_r5fss0_core1: r5f@41400000 { ti,loczrama = <1>; }; }; + + mcu_mcan0: can@40528000 { + compatible = "bosch,m_can"; + reg = <0x00 0x40528000 0x00 0x200>, + <0x00 0x40500000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 172 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 172 0>, <&k3_clks 172 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; + + mcu_mcan1: can@40568000 { + compatible = "bosch,m_can"; + reg = <0x00 0x40568000 0x00 0x200>, + <0x00 0x40540000 0x00 0x8000>; + reg-names = "m_can", "message_ram"; + power-domains = <&k3_pds 173 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 173 0>, <&k3_clks 173 1>; + clock-names = "hclk", "cclk"; + interrupts = , + ; + interrupt-names = "int0", "int1"; + bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; + }; }; diff --git a/arch/arm64/boot/dts/ti/k3-j721e.dtsi b/arch/arm64/boot/dts/ti/k3-j721e.dtsi index 69ce048a21..4a3872fce5 100644 --- a/arch/arm64/boot/dts/ti/k3-j721e.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j721e.dtsi @@ -31,6 +31,9 @@ aliases { serial10 = &main_uart8; serial11 = &main_uart9; ethernet0 = &cpsw_port1; + mmc0 = &main_sdhci0; + mmc1 = &main_sdhci1; + mmc2 = &main_sdhci2; }; chosen { }; diff --git a/arch/arm64/boot/dts/toshiba/Makefile b/arch/arm64/boot/dts/toshiba/Makefile index 8cd460d5b6..7ccb4664bb 100644 --- a/arch/arm64/boot/dts/toshiba/Makefile +++ b/arch/arm64/boot/dts/toshiba/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_VISCONTI) += tmpv7708-rm-mbrc.dtb +dtb-$(CONFIG_ARCH_VISCONTI) += tmpv7708-visrobo-vrb.dtb diff --git a/arch/arm64/boot/dts/toshiba/tmpv7708-rm-mbrc.dts b/arch/arm64/boot/dts/toshiba/tmpv7708-rm-mbrc.dts index 29a4d9fc1e..9375b0faee 100644 --- a/arch/arm64/boot/dts/toshiba/tmpv7708-rm-mbrc.dts +++ b/arch/arm64/boot/dts/toshiba/tmpv7708-rm-mbrc.dts @@ -76,3 +76,9 @@ &pwm_mux { &pwm { status = "okay"; }; + +&pcie { + status = "okay"; + clocks = <&extclk100mhz>, <&clk600mhz>, <&clk25mhz>; + clock-names = "ref", "core", "aux"; +}; diff --git a/arch/arm64/boot/dts/toshiba/tmpv7708.dtsi b/arch/arm64/boot/dts/toshiba/tmpv7708.dtsi index 4b4231ff43..01d7ee61ad 100644 --- a/arch/arm64/boot/dts/toshiba/tmpv7708.dtsi +++ b/arch/arm64/boot/dts/toshiba/tmpv7708.dtsi @@ -134,6 +134,13 @@ uart_clk: uart-clk { #clock-cells = <0>; }; + clk25mhz: clk25mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <25000000>; + clock-output-names = "clk25mhz"; + }; + clk125mhz: clk125mhz { compatible = "fixed-clock"; clock-frequency = <125000000>; @@ -141,6 +148,13 @@ clk125mhz: clk125mhz { clock-output-names = "clk125mhz"; }; + clk150mhz: clk150mhz { + compatible = "fixed-clock"; + clock-frequency = <150000000>; + #clock-cells = <0>; + clock-output-names = "clk150mhz"; + }; + clk300mhz: clk300mhz { compatible = "fixed-clock"; clock-frequency = <300000000>; @@ -148,6 +162,20 @@ clk300mhz: clk300mhz { clock-output-names = "clk300mhz"; }; + clk600mhz: clk600mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <600000000>; + clock-output-names = "clk600mhz"; + }; + + extclk100mhz: extclk100mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; + clock-output-names = "extclk100mhz"; + }; + wdt_clk: wdt-clk { compatible = "fixed-clock"; clock-frequency = <150000000>; @@ -441,6 +469,37 @@ pwm: pwm@241c0000 { #pwm-cells = <2>; status = "disabled"; }; + + pcie: pcie@28400000 { + compatible = "toshiba,visconti-pcie"; + reg = <0x0 0x28400000 0x0 0x00400000>, + <0x0 0x70000000 0x0 0x10000000>, + <0x0 0x28050000 0x0 0x00010000>, + <0x0 0x24200000 0x0 0x00002000>, + <0x0 0x24162000 0x0 0x00001000>; + reg-names = "dbi", "config", "ulreg", "smu", "mpu"; + device_type = "pci"; + bus-range = <0x00 0xff>; + num-lanes = <2>; + num-viewport = <8>; + + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + ranges = <0x81000000 0 0x40000000 0 0x40000000 0 0x00010000 + 0x82000000 0 0x50000000 0 0x50000000 0 0x20000000>; + interrupts = , + ; + interrupt-names = "msi", "intr"; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = + <0 0 0 1 &gic GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH + 0 0 0 2 &gic GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH + 0 0 0 3 &gic GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH + 0 0 0 4 &gic GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH>; + max-link-speed = <2>; + status = "disabled"; + }; }; }; diff --git a/arch/arm64/boot/dts/xilinx/Makefile b/arch/arm64/boot/dts/xilinx/Makefile index 11fb4fd3eb..4e159540d0 100644 --- a/arch/arm64/boot/dts/xilinx/Makefile +++ b/arch/arm64/boot/dts/xilinx/Makefile @@ -12,7 +12,21 @@ dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-zcu100-revC.dtb dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-zcu102-revA.dtb dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-zcu102-revB.dtb dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-zcu102-rev1.0.dtb +dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-zcu102-rev1.1.dtb dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-zcu104-revA.dtb dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-zcu104-revC.dtb dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-zcu106-revA.dtb dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-zcu111-revA.dtb + +dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-sm-k26-revA.dtb +dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-smk-k26-revA.dtb + +sm-k26-revA-sck-kv-g-revA-dtbs := zynqmp-sm-k26-revA.dtb zynqmp-sck-kv-g-revA.dtbo +sm-k26-revA-sck-kv-g-revB-dtbs := zynqmp-sm-k26-revA.dtb zynqmp-sck-kv-g-revB.dtbo +smk-k26-revA-sm-k26-revA-sck-kv-g-revA-dtbs := zynqmp-smk-k26-revA.dtb zynqmp-sck-kv-g-revA.dtbo +smk-k26-revA-sm-k26-revA-sck-kv-g-revB-dtbs := zynqmp-smk-k26-revA.dtb zynqmp-sck-kv-g-revB.dtbo + +dtb-$(CONFIG_ARCH_ZYNQMP) += sm-k26-revA-sck-kv-g-revA.dtb +dtb-$(CONFIG_ARCH_ZYNQMP) += sm-k26-revA-sck-kv-g-revB.dtb +dtb-$(CONFIG_ARCH_ZYNQMP) += smk-k26-revA-sm-k26-revA-sck-kv-g-revA.dtb +dtb-$(CONFIG_ARCH_ZYNQMP) += smk-k26-revA-sm-k26-revA-sck-kv-g-revB.dtb diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi index cf52952247..1e0b1bca7c 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi +++ b/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi @@ -2,7 +2,7 @@ /* * Clock specification for Xilinx ZynqMP * - * (C) Copyright 2017 - 2019, Xilinx, Inc. + * (C) Copyright 2017 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -40,6 +40,17 @@ aux_ref_clk: aux_ref_clk { }; }; +&zynqmp_firmware { + zynqmp_clk: clock-controller { + #clock-cells = <1>; + compatible = "xlnx,zynqmp-clk"; + clocks = <&pss_ref_clk>, <&video_clk>, <&pss_alt_ref_clk>, + <&aux_ref_clk>, <>_crx_ref_clk>; + clock-names = "pss_ref_clk", "video_clk", "pss_alt_ref_clk", + "aux_ref_clk", "gt_crx_ref_clk"; + }; +}; + &can0 { clocks = <&zynqmp_clk CAN0_REF>, <&zynqmp_clk LPD_LSBUS>; }; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1232-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1232-revA.dts index 2e05fa4169..f1598527e5 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1232-revA.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1232-revA.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZC1232 * - * (C) Copyright 2017 - 2019, Xilinx, Inc. + * (C) Copyright 2017 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -19,6 +19,7 @@ / { aliases { serial0 = &uart0; serial1 = &dcc; + spi0 = &qspi; }; chosen { @@ -36,6 +37,19 @@ &dcc { status = "okay"; }; +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; /* 32MB */ + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; + spi-max-frequency = <108000000>; /* Based on DC1 spec */ + }; +}; + &sata { status = "okay"; /* SATA OOB timing settings */ diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1254-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1254-revA.dts index 3d0aaa02f1..04efa1683e 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1254-revA.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1254-revA.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZC1254 * - * (C) Copyright 2015 - 2019, Xilinx, Inc. + * (C) Copyright 2015 - 2021, Xilinx, Inc. * * Michal Simek * Siva Durga Prasad Paladugu @@ -20,6 +20,7 @@ / { aliases { serial0 = &uart0; serial1 = &dcc; + spi0 = &qspi; }; chosen { @@ -37,6 +38,19 @@ &dcc { status = "okay"; }; +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; /* 32MB */ + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; /* FIXME also DUAL configuration possible */ + spi-max-frequency = <108000000>; /* Based on DC1 spec */ + }; +}; + &uart0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1275-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1275-revA.dts index 66a90483b0..e971ba8c14 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1275-revA.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1275-revA.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZC1275 * - * (C) Copyright 2017 - 2019, Xilinx, Inc. + * (C) Copyright 2017 - 2021, Xilinx, Inc. * * Michal Simek * Siva Durga Prasad Paladugu @@ -20,6 +20,7 @@ / { aliases { serial0 = &uart0; serial1 = &dcc; + spi0 = &qspi; }; chosen { @@ -37,6 +38,21 @@ &dcc { status = "okay"; }; +&gpio { + status = "okay"; +}; + +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; + spi-max-frequency = <108000000>; + }; +}; + &uart0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts index 69f6e46107..b05be25528 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP zc1751-xm015-dc1 * - * (C) Copyright 2015 - 2019, Xilinx, Inc. + * (C) Copyright 2015 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -11,7 +11,9 @@ #include "zynqmp.dtsi" #include "zynqmp-clk-ccf.dtsi" +#include #include +#include / { model = "ZynqMP zc1751-xm015-dc1 RevA"; @@ -24,6 +26,8 @@ aliases { mmc1 = &sdhci1; rtc0 = &rtc; serial0 = &uart0; + spi0 = &qspi; + usb0 = &usb0; }; chosen { @@ -35,6 +39,24 @@ memory@0 { device_type = "memory"; reg = <0x0 0x0 0x0 0x80000000>, <0x8 0x00000000 0x0 0x80000000>; }; + + clock_si5338_0: clk27 { /* u55 SI5338-GM */ + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <27000000>; + }; + + clock_si5338_2: clk26 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <26000000>; + }; + + clock_si5338_3: clk150 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <150000000>; + }; }; &fpd_dma_chan1 { @@ -73,6 +95,8 @@ &gem3 { status = "okay"; phy-handle = <&phy0>; phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gem3_default>; phy0: ethernet-phy@0 { reg = <0>; }; @@ -80,12 +104,19 @@ phy0: ethernet-phy@0 { &gpio { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gpio_default>; }; &i2c1 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + scl-gpios = <&gpio 36 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 37 GPIO_ACTIVE_HIGH>; eeprom: eeprom@55 { compatible = "atmel,24c64"; /* 24AA64 */ @@ -93,6 +124,236 @@ eeprom: eeprom@55 { }; }; +&pinctrl0 { + status = "okay"; + pinctrl_i2c1_default: i2c1-default { + mux { + groups = "i2c1_9_grp"; + function = "i2c1"; + }; + + conf { + groups = "i2c1_9_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_gpio: i2c1-gpio { + mux { + groups = "gpio0_36_grp", "gpio0_37_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_36_grp", "gpio0_37_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_uart0_default: uart0-default { + mux { + groups = "uart0_8_grp"; + function = "uart0"; + }; + + conf { + groups = "uart0_8_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO34"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO35"; + bias-disable; + }; + }; + + pinctrl_usb0_default: usb0-default { + mux { + groups = "usb0_0_grp"; + function = "usb0"; + }; + + conf { + groups = "usb0_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO52", "MIO53", "MIO55"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO54", "MIO56", "MIO57", "MIO58", "MIO59", + "MIO60", "MIO61", "MIO62", "MIO63"; + bias-disable; + }; + }; + + pinctrl_gem3_default: gem3-default { + mux { + function = "ethernet3"; + groups = "ethernet3_0_grp"; + }; + + conf { + groups = "ethernet3_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO70", "MIO71", "MIO72", "MIO73", "MIO74", + "MIO75"; + bias-high-impedance; + low-power-disable; + }; + + conf-tx { + pins = "MIO64", "MIO65", "MIO66", "MIO67", "MIO68", + "MIO69"; + bias-disable; + low-power-enable; + }; + + mux-mdio { + function = "mdio3"; + groups = "mdio3_0_grp"; + }; + + conf-mdio { + groups = "mdio3_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + }; + + pinctrl_sdhci0_default: sdhci0-default { + mux { + groups = "sdio0_0_grp"; + function = "sdio0"; + }; + + conf { + groups = "sdio0_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + + mux-cd { + groups = "sdio0_cd_0_grp"; + function = "sdio0_cd"; + }; + + conf-cd { + groups = "sdio0_cd_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + + mux-wp { + groups = "sdio0_wp_0_grp"; + function = "sdio0_wp"; + }; + + conf-wp { + groups = "sdio0_wp_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_sdhci1_default: sdhci1-default { + mux { + groups = "sdio1_0_grp"; + function = "sdio1"; + }; + + conf { + groups = "sdio1_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + + mux-cd { + groups = "sdio1_cd_0_grp"; + function = "sdio1_cd"; + }; + + conf-cd { + groups = "sdio1_cd_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + + mux-wp { + groups = "sdio1_wp_0_grp"; + function = "sdio1_wp"; + }; + + conf-wp { + groups = "sdio1_wp_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_gpio_default: gpio-default { + mux { + function = "gpio0"; + groups = "gpio0_38_grp"; + }; + + conf { + groups = "gpio0_38_grp"; + bias-disable; + slew-rate = ; + power-source = ; + }; + }; +}; + +&psgtr { + status = "okay"; + /* dp, usb3, sata */ + clocks = <&clock_si5338_0>, <&clock_si5338_2>, <&clock_si5338_3>; + clock-names = "ref1", "ref2", "ref3"; +}; + +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; /* Micron MT25QU512ABB8ESF */ + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; + spi-max-frequency = <108000000>; /* Based on DC1 spec */ + }; +}; + &rtc { status = "okay"; }; @@ -108,25 +369,60 @@ &sata { ceva,p1-comwake-params = /bits/ 8 <0x06 0x19 0x08 0x0E>; ceva,p1-burst-params = /bits/ 8 <0x13 0x08 0x4A 0x06>; ceva,p1-retry-params = /bits/ 16 <0x96A4 0x3FFC>; + phy-names = "sata-phy"; + phys = <&psgtr 3 PHY_TYPE_SATA 1 3>; }; /* eMMC */ &sdhci0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci0_default>; bus-width = <8>; + xlnx,mio-bank = <0>; }; /* SD1 with level shifter */ &sdhci1 { status = "okay"; + /* + * This property should be removed for supporting UHS mode + */ + no-1-8-v; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci1_default>; + xlnx,mio-bank = <1>; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; }; /* ULPI SMSC USB3320 */ &usb0 { status = "okay"; - dr_mode = "host"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0_default>; + phy-names = "usb3-phy"; + phys = <&psgtr 2 PHY_TYPE_USB3 0 2>; +}; + +&dwc3_0 { + status = "okay"; + dr_mode = "host"; + snps,usb3_lpm_capable; + maximum-speed = "super-speed"; +}; + +&zynqmp_dpdma { + status = "okay"; +}; + +&zynqmp_dpsub { + status = "okay"; + phy-names = "dp-phy0", "dp-phy1"; + phys = <&psgtr 1 PHY_TYPE_DP 0 0>, + <&psgtr 0 PHY_TYPE_DP 1 1>; }; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts index f7124e15f0..938b76bd05 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP zc1751-xm016-dc2 * - * (C) Copyright 2015 - 2019, Xilinx, Inc. + * (C) Copyright 2015 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -12,14 +12,13 @@ #include "zynqmp.dtsi" #include "zynqmp-clk-ccf.dtsi" #include +#include / { model = "ZynqMP zc1751-xm016-dc2 RevA"; compatible = "xlnx,zynqmp-zc1751", "xlnx,zynqmp"; aliases { - can0 = &can0; - can1 = &can1; ethernet0 = &gem2; i2c0 = &i2c0; rtc0 = &rtc; @@ -27,6 +26,7 @@ aliases { serial1 = &uart1; spi0 = &spi0; spi1 = &spi1; + usb0 = &usb1; }; chosen { @@ -42,10 +42,14 @@ memory@0 { &can0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can0_default>; }; &can1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can1_default>; }; &fpd_dma_chan1 { @@ -84,6 +88,8 @@ &gem2 { status = "okay"; phy-handle = <&phy0>; phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gem2_default>; phy0: ethernet-phy@5 { reg = <5>; ti,rx-internal-delay = <0x8>; @@ -100,6 +106,11 @@ &gpio { &i2c0 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c0_default>; + pinctrl-1 = <&pinctrl_i2c0_gpio>; + scl-gpios = <&gpio 6 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 7 GPIO_ACTIVE_HIGH>; tca6416_u26: gpio@20 { compatible = "ti,tca6416"; @@ -115,6 +126,311 @@ rtc@68 { }; }; +&nand0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_nand0_default>; + arasan,has-mdma; + + nand@0 { + reg = <0x0>; + #address-cells = <0x2>; + #size-cells = <0x1>; + nand-ecc-mode = "soft"; + nand-ecc-algo = "bch"; + nand-rb = <0>; + label = "main-storage-0"; + }; + nand@1 { + reg = <0x1>; + #address-cells = <0x2>; + #size-cells = <0x1>; + nand-ecc-mode = "soft"; + nand-ecc-algo = "bch"; + nand-rb = <0>; + label = "main-storage-1"; + }; +}; + +&pinctrl0 { + status = "okay"; + pinctrl_can0_default: can0-default { + mux { + function = "can0"; + groups = "can0_9_grp"; + }; + + conf { + groups = "can0_9_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO38"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO39"; + bias-disable; + }; + }; + + pinctrl_can1_default: can1-default { + mux { + function = "can1"; + groups = "can1_8_grp"; + }; + + conf { + groups = "can1_8_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO33"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO32"; + bias-disable; + }; + }; + + pinctrl_i2c0_default: i2c0-default { + mux { + groups = "i2c0_1_grp"; + function = "i2c0"; + }; + + conf { + groups = "i2c0_1_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c0_gpio: i2c0-gpio { + mux { + groups = "gpio0_6_grp", "gpio0_7_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_6_grp", "gpio0_7_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_uart0_default: uart0-default { + mux { + groups = "uart0_10_grp"; + function = "uart0"; + }; + + conf { + groups = "uart0_10_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO42"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO43"; + bias-disable; + }; + }; + + pinctrl_uart1_default: uart1-default { + mux { + groups = "uart1_10_grp"; + function = "uart1"; + }; + + conf { + groups = "uart1_10_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO41"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO40"; + bias-disable; + }; + }; + + pinctrl_usb1_default: usb1-default { + mux { + groups = "usb1_0_grp"; + function = "usb1"; + }; + + conf { + groups = "usb1_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO64", "MIO65", "MIO67"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO66", "MIO68", "MIO69", "MIO70", "MIO71", + "MIO72", "MIO73", "MIO74", "MIO75"; + bias-disable; + }; + }; + + pinctrl_gem2_default: gem2-default { + mux { + function = "ethernet2"; + groups = "ethernet2_0_grp"; + }; + + conf { + groups = "ethernet2_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO58", "MIO59", "MIO60", "MIO61", "MIO62", + "MIO63"; + bias-high-impedance; + low-power-disable; + }; + + conf-tx { + pins = "MIO52", "MIO53", "MIO54", "MIO55", "MIO56", + "MIO57"; + bias-disable; + low-power-enable; + }; + + mux-mdio { + function = "mdio2"; + groups = "mdio2_0_grp"; + }; + + conf-mdio { + groups = "mdio2_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + }; + + pinctrl_nand0_default: nand0-default { + mux { + groups = "nand0_0_grp"; + function = "nand0"; + }; + + conf { + groups = "nand0_0_grp"; + bias-pull-up; + }; + + mux-ce { + groups = "nand0_ce_0_grp"; + function = "nand0_ce"; + }; + + conf-ce { + groups = "nand0_ce_0_grp"; + bias-pull-up; + }; + + mux-rb { + groups = "nand0_rb_0_grp"; + function = "nand0_rb"; + }; + + conf-rb { + groups = "nand0_rb_0_grp"; + bias-pull-up; + }; + + mux-dqs { + groups = "nand0_dqs_0_grp"; + function = "nand0_dqs"; + }; + + conf-dqs { + groups = "nand0_dqs_0_grp"; + bias-pull-up; + }; + }; + + pinctrl_spi0_default: spi0-default { + mux { + groups = "spi0_0_grp"; + function = "spi0"; + }; + + conf { + groups = "spi0_0_grp"; + bias-disable; + slew-rate = ; + power-source = ; + }; + + mux-cs { + groups = "spi0_ss_0_grp", "spi0_ss_1_grp", + "spi0_ss_2_grp"; + function = "spi0_ss"; + }; + + conf-cs { + groups = "spi0_ss_0_grp", "spi0_ss_1_grp", + "spi0_ss_2_grp"; + bias-disable; + }; + }; + + pinctrl_spi1_default: spi1-default { + mux { + groups = "spi1_3_grp"; + function = "spi1"; + }; + + conf { + groups = "spi1_3_grp"; + bias-disable; + slew-rate = ; + power-source = ; + }; + + mux-cs { + groups = "spi1_ss_9_grp", "spi1_ss_10_grp", + "spi1_ss_11_grp"; + function = "spi1_ss"; + }; + + conf-cs { + groups = "spi1_ss_9_grp", "spi1_ss_10_grp", + "spi1_ss_11_grp"; + bias-disable; + }; + }; +}; + &rtc { status = "okay"; }; @@ -122,6 +438,8 @@ &rtc { &spi0 { status = "okay"; num-cs = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_spi0_default>; spi0_flash0: flash@0 { #address-cells = <1>; @@ -140,6 +458,8 @@ partition@0 { &spi1 { status = "okay"; num-cs = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_spi1_default>; spi1_flash0: flash@0 { #address-cells = <1>; @@ -157,14 +477,26 @@ partition@0 { /* ULPI SMSC USB3320 */ &usb1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb1_default>; +}; + +&dwc3_1 { status = "okay"; dr_mode = "host"; + snps,usb3_lpm_capable; + maximum-speed = "super-speed"; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; }; &uart1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1_default>; }; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm017-dc3.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm017-dc3.dts index 4ea6ef5a7f..381cc682ce 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm017-dc3.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm017-dc3.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP zc1751-xm017-dc3 * - * (C) Copyright 2016 - 2019, Xilinx, Inc. + * (C) Copyright 2016 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -11,6 +11,7 @@ #include "zynqmp.dtsi" #include "zynqmp-clk-ccf.dtsi" +#include / { model = "ZynqMP zc1751-xm017-dc3 RevA"; @@ -24,6 +25,8 @@ aliases { rtc0 = &rtc; serial0 = &uart0; serial1 = &uart1; + usb0 = &usb0; + usb1 = &usb1; }; chosen { @@ -35,6 +38,18 @@ memory@0 { device_type = "memory"; reg = <0x0 0x0 0x0 0x80000000>, <0x8 0x00000000 0x0 0x80000000>; }; + + clock_si5338_2: clk26 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <26000000>; + }; + + clock_si5338_3: clk125 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + }; }; &fpd_dma_chan1 { @@ -107,6 +122,20 @@ &i2c1 { clock-frequency = <400000>; }; +/* MT29F64G08AECDBJ4-6 */ +&nand0 { + status = "okay"; + arasan,has-mdma; + num-cs = <2>; +}; + +&psgtr { + status = "okay"; + /* usb3, sata */ + clocks = <&clock_si5338_2>, <&clock_si5338_3>; + clock-names = "ref2", "ref3"; +}; + &rtc { status = "okay"; }; @@ -122,6 +151,8 @@ &sata { ceva,p1-comwake-params = /bits/ 8 <0x06 0x19 0x08 0x0E>; ceva,p1-burst-params = /bits/ 8 <0x13 0x08 0x4A 0x06>; ceva,p1-retry-params = /bits/ 16 <0x96A4 0x3FFC>; + phy-names = "sata-phy"; + phys = <&psgtr 2 PHY_TYPE_SATA 0 3>; }; &sdhci1 { /* emmc with some settings */ @@ -139,12 +170,28 @@ &uart1 { }; &usb0 { + status = "okay"; + phy-names = "usb3-phy"; + phys = <&psgtr 0 PHY_TYPE_USB3 0 2>; +}; + +&dwc3_0 { status = "okay"; dr_mode = "host"; + snps,usb3_lpm_capable; + maximum-speed = "super-speed"; }; /* ULPI SMSC USB3320 */ &usb1 { status = "okay"; - dr_mode = "host"; + phy-names = "usb3-phy"; + phys = <&psgtr 3 PHY_TYPE_USB3 1 2>; +}; + +&dwc3_1 { + status = "okay"; + dr_mode = "host"; + snps,usb3_lpm_capable; + maximum-speed = "super-speed"; }; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm018-dc4.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm018-dc4.dts index 2366cd9f09..05a2b79738 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm018-dc4.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm018-dc4.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP zc1751-xm018-dc4 * - * (C) Copyright 2015 - 2019, Xilinx, Inc. + * (C) Copyright 2015 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -26,6 +26,7 @@ aliases { rtc0 = &rtc; serial0 = &uart0; serial1 = &uart1; + spi0 = &qspi; }; chosen { @@ -161,6 +162,19 @@ &i2c1 { status = "okay"; }; +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; /* 32MB */ + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; /* also DUAL configuration possible */ + spi-max-frequency = <108000000>; /* Based on DC1 spec */ + }; +}; + &rtc { status = "okay"; }; @@ -176,3 +190,11 @@ &uart1 { &watchdog0 { status = "okay"; }; + +&zynqmp_dpdma { + status = "okay"; +}; + +&zynqmp_dpsub { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts index 41934e3525..ae2d03d983 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP zc1751-xm019-dc5 * - * (C) Copyright 2015 - 2019, Xilinx, Inc. + * (C) Copyright 2015 - 2021, Xilinx, Inc. * * Siva Durga Prasad * Michal Simek @@ -13,6 +13,7 @@ #include "zynqmp.dtsi" #include "zynqmp-clk-ccf.dtsi" #include +#include / { model = "ZynqMP zc1751-xm019-dc5 RevA"; @@ -74,6 +75,8 @@ &gem1 { status = "okay"; phy-handle = <&phy0>; phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gem1_default>; phy0: ethernet-phy@0 { reg = <0>; }; @@ -85,41 +88,366 @@ &gpio { &i2c0 { status = "okay"; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c0_default>; + pinctrl-1 = <&pinctrl_i2c0_gpio>; + scl-gpios = <&gpio 74 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 75 GPIO_ACTIVE_HIGH>; }; &i2c1 { status = "okay"; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + scl-gpios = <&gpio 76 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 77 GPIO_ACTIVE_HIGH>; + +}; + +&pinctrl0 { + status = "okay"; + pinctrl_i2c0_default: i2c0-default { + mux { + groups = "i2c0_18_grp"; + function = "i2c0"; + }; + + conf { + groups = "i2c0_18_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c0_gpio: i2c0-gpio { + mux { + groups = "gpio0_74_grp", "gpio0_75_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_74_grp", "gpio0_75_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_default: i2c1-default { + mux { + groups = "i2c1_19_grp"; + function = "i2c1"; + }; + + conf { + groups = "i2c1_19_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_gpio: i2c1-gpio { + mux { + groups = "gpio0_76_grp", "gpio0_77_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_76_grp", "gpio0_77_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_uart0_default: uart0-default { + mux { + groups = "uart0_17_grp"; + function = "uart0"; + }; + + conf { + groups = "uart0_17_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO70"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO71"; + bias-disable; + }; + }; + + pinctrl_uart1_default: uart1-default { + mux { + groups = "uart1_18_grp"; + function = "uart1"; + }; + + conf { + groups = "uart1_18_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO73"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO72"; + bias-disable; + }; + }; + + pinctrl_gem1_default: gem1-default { + mux { + function = "ethernet1"; + groups = "ethernet1_0_grp"; + }; + + conf { + groups = "ethernet1_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO44", "MIO45", "MIO46", "MIO47", "MIO48", + "MIO49"; + bias-high-impedance; + low-power-disable; + }; + + conf-tx { + pins = "MIO38", "MIO39", "MIO40", "MIO41", "MIO42", + "MIO43"; + bias-disable; + low-power-enable; + }; + + mux-mdio { + function = "mdio1"; + groups = "mdio1_0_grp"; + }; + + conf-mdio { + groups = "mdio1_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + }; + + pinctrl_sdhci0_default: sdhci0-default { + mux { + groups = "sdio0_0_grp"; + function = "sdio0"; + }; + + conf { + groups = "sdio0_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + + mux-cd { + groups = "sdio0_cd_0_grp"; + function = "sdio0_cd"; + }; + + conf-cd { + groups = "sdio0_cd_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + + mux-wp { + groups = "sdio0_wp_0_grp"; + function = "sdio0_wp"; + }; + + conf-wp { + groups = "sdio0_wp_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_watchdog0_default: watchdog0-default { + mux-clk { + groups = "swdt0_clk_1_grp"; + function = "swdt0_clk"; + }; + + conf-clk { + groups = "swdt0_clk_1_grp"; + bias-pull-up; + }; + + mux-rst { + groups = "swdt0_rst_1_grp"; + function = "swdt0_rst"; + }; + + conf-rst { + groups = "swdt0_rst_1_grp"; + bias-disable; + slew-rate = ; + }; + }; + + pinctrl_ttc0_default: ttc0-default { + mux-clk { + groups = "ttc0_clk_0_grp"; + function = "ttc0_clk"; + }; + + conf-clk { + groups = "ttc0_clk_0_grp"; + bias-pull-up; + }; + + mux-wav { + groups = "ttc0_wav_0_grp"; + function = "ttc0_wav"; + }; + + conf-wav { + groups = "ttc0_wav_0_grp"; + bias-disable; + slew-rate = ; + }; + }; + + pinctrl_ttc1_default: ttc1-default { + mux-clk { + groups = "ttc1_clk_0_grp"; + function = "ttc1_clk"; + }; + + conf-clk { + groups = "ttc1_clk_0_grp"; + bias-pull-up; + }; + + mux-wav { + groups = "ttc1_wav_0_grp"; + function = "ttc1_wav"; + }; + + conf-wav { + groups = "ttc1_wav_0_grp"; + bias-disable; + slew-rate = ; + }; + }; + + pinctrl_ttc2_default: ttc2-default { + mux-clk { + groups = "ttc2_clk_0_grp"; + function = "ttc2_clk"; + }; + + conf-clk { + groups = "ttc2_clk_0_grp"; + bias-pull-up; + }; + + mux-wav { + groups = "ttc2_wav_0_grp"; + function = "ttc2_wav"; + }; + + conf-wav { + groups = "ttc2_wav_0_grp"; + bias-disable; + slew-rate = ; + }; + }; + + pinctrl_ttc3_default: ttc3-default { + mux-clk { + groups = "ttc3_clk_0_grp"; + function = "ttc3_clk"; + }; + + conf-clk { + groups = "ttc3_clk_0_grp"; + bias-pull-up; + }; + + mux-wav { + groups = "ttc3_wav_0_grp"; + function = "ttc3_wav"; + }; + + conf-wav { + groups = "ttc3_wav_0_grp"; + bias-disable; + slew-rate = ; + }; + }; }; &sdhci0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci0_default>; no-1-8-v; + xlnx,mio-bank = <0>; }; &ttc0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ttc0_default>; }; &ttc1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ttc1_default>; }; &ttc2 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ttc2_default>; }; &ttc3 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ttc3_default>; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; }; &uart1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1_default>; }; &watchdog0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_watchdog0_default>; }; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts index a53598c362..f6aad4159c 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZCU100 revC * - * (C) Copyright 2016 - 2019, Xilinx, Inc. + * (C) Copyright 2016 - 2021, Xilinx, Inc. * * Michal Simek * Nathalie Chan King Choy @@ -15,6 +15,7 @@ #include #include #include +#include #include / { @@ -29,6 +30,8 @@ aliases { serial2 = &dcc; spi0 = &spi0; spi1 = &spi1; + usb0 = &usb0; + usb1 = &usb1; mmc0 = &sdhci0; mmc1 = &sdhci1; }; @@ -110,13 +113,13 @@ ina226 { io-channels = <&u35 0>, <&u35 1>, <&u35 2>, <&u35 3>; }; - si5335a_0: clk26 { + si5335_0: si5335_0 { /* clk0_usb - u23 */ compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <26000000>; }; - si5335a_1: clk27 { + si5335_1: si5335_1 { /* clk1_dp - u23 */ compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <27000000>; @@ -160,6 +163,11 @@ &gpio { &i2c1 { status = "okay"; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + scl-gpios = <&gpio 4 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 5 GPIO_ACTIVE_HIGH>; clock-frequency = <100000>; i2c-mux@75 { /* u11 */ compatible = "nxp,pca9548"; @@ -237,10 +245,225 @@ i2csw_7: i2c@7 { }; }; +&pinctrl0 { + status = "okay"; + pinctrl_i2c1_default: i2c1-default { + mux { + groups = "i2c1_1_grp"; + function = "i2c1"; + }; + + conf { + groups = "i2c1_1_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_gpio: i2c1-gpio { + mux { + groups = "gpio0_4_grp", "gpio0_5_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_4_grp", "gpio0_5_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_sdhci0_default: sdhci0-default { + mux { + groups = "sdio0_3_grp"; + function = "sdio0"; + }; + + conf { + groups = "sdio0_3_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + + mux-cd { + groups = "sdio0_cd_0_grp"; + function = "sdio0_cd"; + }; + + conf-cd { + groups = "sdio0_cd_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_sdhci1_default: sdhci1-default { + mux { + groups = "sdio1_2_grp"; + function = "sdio1"; + }; + + conf { + groups = "sdio1_2_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + }; + + pinctrl_spi0_default: spi0-default { + mux { + groups = "spi0_3_grp"; + function = "spi0"; + }; + + conf { + groups = "spi0_3_grp"; + bias-disable; + slew-rate = ; + power-source = ; + }; + + mux-cs { + groups = "spi0_ss_9_grp"; + function = "spi0_ss"; + }; + + conf-cs { + groups = "spi0_ss_9_grp"; + bias-disable; + }; + + }; + + pinctrl_spi1_default: spi1-default { + mux { + groups = "spi1_0_grp"; + function = "spi1"; + }; + + conf { + groups = "spi1_0_grp"; + bias-disable; + slew-rate = ; + power-source = ; + }; + + mux-cs { + groups = "spi1_ss_0_grp"; + function = "spi1_ss"; + }; + + conf-cs { + groups = "spi1_ss_0_grp"; + bias-disable; + }; + + }; + + pinctrl_uart0_default: uart0-default { + mux { + groups = "uart0_0_grp"; + function = "uart0"; + }; + + conf { + groups = "uart0_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO3"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO2"; + bias-disable; + }; + }; + + pinctrl_uart1_default: uart1-default { + mux { + groups = "uart1_0_grp"; + function = "uart1"; + }; + + conf { + groups = "uart1_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO1"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO0"; + bias-disable; + }; + }; + + pinctrl_usb0_default: usb0-default { + mux { + groups = "usb0_0_grp"; + function = "usb0"; + }; + + conf { + groups = "usb0_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO52", "MIO53", "MIO55"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO54", "MIO56", "MIO57", "MIO58", "MIO59", + "MIO60", "MIO61", "MIO62", "MIO63"; + bias-disable; + }; + }; + + pinctrl_usb1_default: usb1-default { + mux { + groups = "usb1_0_grp"; + function = "usb1"; + }; + + conf { + groups = "usb1_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO64", "MIO65", "MIO67"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO66", "MIO68", "MIO69", "MIO70", "MIO71", + "MIO72", "MIO73", "MIO74", "MIO75"; + bias-disable; + }; + }; +}; + &psgtr { status = "okay"; - /* usb3, dps */ - clocks = <&si5335a_0>, <&si5335a_1>; + /* usb3, dp */ + clocks = <&si5335_0>, <&si5335_1>; clock-names = "ref0", "ref1"; }; @@ -253,12 +476,16 @@ &sdhci0 { status = "okay"; no-1-8-v; disable-wp; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci0_default>; xlnx,mio-bank = <0>; }; &sdhci1 { status = "okay"; bus-width = <0x4>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci1_default>; xlnx,mio-bank = <0>; non-removable; disable-wp; @@ -279,16 +506,22 @@ &spi0 { /* Low Speed connector */ status = "okay"; label = "LS-SPI0"; num-cs = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_spi0_default>; }; &spi1 { /* High Speed connector */ status = "okay"; label = "HS-SPI1"; num-cs = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_spi1_default>; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; bluetooth { compatible = "ti,wl1831-st"; enable-gpios = <&gpio 8 GPIO_ACTIVE_HIGH>; @@ -297,19 +530,38 @@ bluetooth { &uart1 { status = "okay"; - + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1_default>; }; /* ULPI SMSC USB3320 */ &usb0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0_default>; + phy-names = "usb3-phy"; + phys = <&psgtr 2 PHY_TYPE_USB3 0 0>; +}; + +&dwc3_0 { status = "okay"; dr_mode = "peripheral"; + maximum-speed = "super-speed"; }; /* ULPI SMSC USB3320 */ &usb1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb1_default>; + phy-names = "usb3-phy"; + phys = <&psgtr 3 PHY_TYPE_USB3 1 0>; +}; + +&dwc3_1 { status = "okay"; dr_mode = "host"; + maximum-speed = "super-speed"; }; &watchdog0 { diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts index eca6c2de84..7b9a88b125 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZCU102 RevA * - * (C) Copyright 2015 - 2019, Xilinx, Inc. + * (C) Copyright 2015 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -13,6 +13,7 @@ #include "zynqmp-clk-ccf.dtsi" #include #include +#include #include / { @@ -24,10 +25,13 @@ aliases { i2c0 = &i2c0; i2c1 = &i2c1; mmc0 = &sdhci1; + nvmem0 = &eeprom; rtc0 = &rtc; serial0 = &uart0; serial1 = &uart1; serial2 = &dcc; + spi0 = &qspi; + usb0 = &usb0; }; chosen { @@ -150,6 +154,8 @@ refhdmi: refhdmi { &can1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can1_default>; }; &dcc { @@ -192,22 +198,32 @@ &gem3 { status = "okay"; phy-handle = <&phy0>; phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gem3_default>; phy0: ethernet-phy@21 { reg = <21>; ti,rx-internal-delay = <0x8>; ti,tx-internal-delay = <0xa>; ti,fifo-depth = <0x1>; ti,dp83867-rxctrl-strap-quirk; + /* reset-gpios = <&tca6416_u97 6 GPIO_ACTIVE_LOW>; */ }; }; &gpio { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gpio_default>; }; &i2c0 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c0_default>; + pinctrl-1 = <&pinctrl_i2c0_gpio>; + scl-gpios = <&gpio 14 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 15 GPIO_ACTIVE_HIGH>; tca6416_u97: gpio@20 { compatible = "ti,tca6416"; @@ -451,7 +467,6 @@ max15303@20 { /* u8 */ status = "disabled"; /* unreachable */ reg = <0x20>; }; - max20751@72 { /* u95 */ compatible = "maxim,max20751"; reg = <0x72>; @@ -468,6 +483,11 @@ max20751@73 { /* u96 */ &i2c1 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + scl-gpios = <&gpio 16 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 17 GPIO_ACTIVE_HIGH>; /* PL i2c via PCA9306 - u45 */ i2c-mux@74 { /* u34 */ @@ -642,6 +662,269 @@ i2c@7 { }; }; +&pinctrl0 { + status = "okay"; + pinctrl_i2c0_default: i2c0-default { + mux { + groups = "i2c0_3_grp"; + function = "i2c0"; + }; + + conf { + groups = "i2c0_3_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c0_gpio: i2c0-gpio { + mux { + groups = "gpio0_14_grp", "gpio0_15_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_14_grp", "gpio0_15_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_default: i2c1-default { + mux { + groups = "i2c1_4_grp"; + function = "i2c1"; + }; + + conf { + groups = "i2c1_4_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_gpio: i2c1-gpio { + mux { + groups = "gpio0_16_grp", "gpio0_17_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_16_grp", "gpio0_17_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_uart0_default: uart0-default { + mux { + groups = "uart0_4_grp"; + function = "uart0"; + }; + + conf { + groups = "uart0_4_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO18"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO19"; + bias-disable; + }; + }; + + pinctrl_uart1_default: uart1-default { + mux { + groups = "uart1_5_grp"; + function = "uart1"; + }; + + conf { + groups = "uart1_5_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO21"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO20"; + bias-disable; + }; + }; + + pinctrl_usb0_default: usb0-default { + mux { + groups = "usb0_0_grp"; + function = "usb0"; + }; + + conf { + groups = "usb0_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO52", "MIO53", "MIO55"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO54", "MIO56", "MIO57", "MIO58", "MIO59", + "MIO60", "MIO61", "MIO62", "MIO63"; + bias-disable; + }; + }; + + pinctrl_gem3_default: gem3-default { + mux { + function = "ethernet3"; + groups = "ethernet3_0_grp"; + }; + + conf { + groups = "ethernet3_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO70", "MIO71", "MIO72", "MIO73", "MIO74", + "MIO75"; + bias-high-impedance; + low-power-disable; + }; + + conf-tx { + pins = "MIO64", "MIO65", "MIO66", "MIO67", "MIO68", + "MIO69"; + bias-disable; + low-power-enable; + }; + + mux-mdio { + function = "mdio3"; + groups = "mdio3_0_grp"; + }; + + conf-mdio { + groups = "mdio3_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + }; + + pinctrl_can1_default: can1-default { + mux { + function = "can1"; + groups = "can1_6_grp"; + }; + + conf { + groups = "can1_6_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO25"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO24"; + bias-disable; + }; + }; + + pinctrl_sdhci1_default: sdhci1-default { + mux { + groups = "sdio1_0_grp"; + function = "sdio1"; + }; + + conf { + groups = "sdio1_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + + mux-cd { + groups = "sdio1_cd_0_grp"; + function = "sdio1_cd"; + }; + + conf-cd { + groups = "sdio1_cd_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + + mux-wp { + groups = "sdio1_wp_0_grp"; + function = "sdio1_wp"; + }; + + conf-wp { + groups = "sdio1_wp_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_gpio_default: gpio-default { + mux-sw { + function = "gpio0"; + groups = "gpio0_22_grp", "gpio0_23_grp"; + }; + + conf-sw { + groups = "gpio0_22_grp", "gpio0_23_grp"; + slew-rate = ; + power-source = ; + }; + + mux-msp { + function = "gpio0"; + groups = "gpio0_13_grp", "gpio0_38_grp"; + }; + + conf-msp { + groups = "gpio0_13_grp", "gpio0_38_grp"; + slew-rate = ; + power-source = ; + }; + + conf-pull-up { + pins = "MIO22", "MIO23"; + bias-pull-up; + }; + + conf-pull-none { + pins = "MIO13", "MIO38"; + bias-disable; + }; + }; +}; + &pcie { status = "okay"; }; @@ -653,6 +936,19 @@ &psgtr { clock-names = "ref0", "ref1", "ref2", "ref3"; }; +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; /* 16MB + 16MB */ + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; /* FIXME also DUAL configuration possible */ + spi-max-frequency = <108000000>; /* Based on DC1 spec */ + }; +}; + &rtc { status = "okay"; }; @@ -675,22 +971,42 @@ &sata { /* SD1 with level shifter */ &sdhci1 { status = "okay"; + /* + * 1.0 revision has level shifter and this property should be + * removed for supporting UHS mode + */ no-1-8-v; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci1_default>; xlnx,mio-bank = <1>; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; }; &uart1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1_default>; }; /* ULPI SMSC USB3320 */ &usb0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0_default>; + phy-names = "usb3-phy"; + phys = <&psgtr 2 PHY_TYPE_USB3 0 2>; +}; + +&dwc3_0 { status = "okay"; dr_mode = "host"; + snps,usb3_lpm_capable; + maximum-speed = "super-speed"; }; &watchdog0 { diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revB.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revB.dts index d9ad8a4b20..f7d718ff11 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revB.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revB.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZCU102 RevB * - * (C) Copyright 2016 - 2018, Xilinx, Inc. + * (C) Copyright 2016 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -22,6 +22,7 @@ phyc: ethernet-phy@c { ti,tx-internal-delay = <0xa>; ti,fifo-depth = <0x1>; ti,dp83867-rxctrl-strap-quirk; + /* reset-gpios = <&tca6416_u97 6 GPIO_ACTIVE_LOW>; */ }; /* Cleanup from RevA */ /delete-node/ ethernet-phy@21; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revA.dts index 5637e1c17f..bd8f20f322 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revA.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revA.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZCU104 * - * (C) Copyright 2017 - 2019, Xilinx, Inc. + * (C) Copyright 2017 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -12,6 +12,7 @@ #include "zynqmp.dtsi" #include "zynqmp-clk-ccf.dtsi" #include +#include #include / { @@ -22,10 +23,13 @@ aliases { ethernet0 = &gem3; i2c0 = &i2c1; mmc0 = &sdhci1; + nvmem0 = &eeprom; rtc0 = &rtc; serial0 = &uart0; serial1 = &uart1; serial2 = &dcc; + spi0 = &qspi; + usb0 = &usb0; }; chosen { @@ -59,16 +63,52 @@ clock_8t49n287_3: clk27 { &can1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can1_default>; }; &dcc { status = "okay"; }; +&fpd_dma_chan1 { + status = "okay"; +}; + +&fpd_dma_chan2 { + status = "okay"; +}; + +&fpd_dma_chan3 { + status = "okay"; +}; + +&fpd_dma_chan4 { + status = "okay"; +}; + +&fpd_dma_chan5 { + status = "okay"; +}; + +&fpd_dma_chan6 { + status = "okay"; +}; + +&fpd_dma_chan7 { + status = "okay"; +}; + +&fpd_dma_chan8 { + status = "okay"; +}; + &gem3 { status = "okay"; phy-handle = <&phy0>; phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gem3_default>; phy0: ethernet-phy@c { reg = <0xc>; ti,rx-internal-delay = <0x8>; @@ -85,6 +125,11 @@ &gpio { &i2c1 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + scl-gpios = <&gpio 16 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 17 GPIO_ACTIVE_HIGH>; /* Another connection to this bus via PL i2c via PCA9306 - u45 */ i2c-mux@74 { /* u34 */ @@ -104,7 +149,7 @@ i2c@0 { * 512B - 768B address 0x56 * 768B - 1024B address 0x57 */ - eeprom@54 { /* u23 */ + eeprom: eeprom@54 { /* u23 */ compatible = "atmel,24c08"; reg = <0x54>; #address-cells = <1>; @@ -116,20 +161,20 @@ i2c@1 { #address-cells = <1>; #size-cells = <0>; reg = <1>; - clock_8t49n287: clock-generator@6c { /* 8T49N287 - u182 */ - reg = <0x6c>; - }; + /* 8T49N287 - u182 */ }; i2c@2 { #address-cells = <1>; #size-cells = <0>; reg = <2>; - irps5401_43: irps54012@43 { /* IRPS5401 - u175 */ - reg = <0x43>; + irps5401_43: irps5401@43 { /* IRPS5401 - u175 */ + compatible = "infineon,irps5401"; + reg = <0x43>; /* pmbus / i2c 0x13 */ }; - irps5401_4d: irps54012@4d { /* IRPS5401 - u180 */ - reg = <0x4d>; + irps5401_44: irps5401@44 { /* IRPS5401 - u180 */ + compatible = "infineon,irps5401"; + reg = <0x44>; /* pmbus / i2c 0x14 */ }; }; @@ -173,8 +218,202 @@ i2c@7 { }; }; -&rtc { +&pinctrl0 { status = "okay"; + + pinctrl_can1_default: can1-default { + mux { + function = "can1"; + groups = "can1_6_grp"; + }; + + conf { + groups = "can1_6_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO25"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO24"; + bias-disable; + }; + }; + + pinctrl_i2c1_default: i2c1-default { + mux { + groups = "i2c1_4_grp"; + function = "i2c1"; + }; + + conf { + groups = "i2c1_4_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + }; + + pinctrl_i2c1_gpio: i2c1-gpio { + mux { + groups = "gpio0_16_grp", "gpio0_17_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_16_grp", "gpio0_17_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + }; + + pinctrl_gem3_default: gem3-default { + mux { + function = "ethernet3"; + groups = "ethernet3_0_grp"; + }; + + conf { + groups = "ethernet3_0_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO70", "MIO71", "MIO72", "MIO73", "MIO74", + "MIO75"; + bias-high-impedance; + low-power-disable; + }; + + conf-tx { + pins = "MIO64", "MIO65", "MIO66", "MIO67", "MIO68", + "MIO69"; + bias-disable; + low-power-enable; + }; + + mux-mdio { + function = "mdio3"; + groups = "mdio3_0_grp"; + }; + + conf-mdio { + groups = "mdio3_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + }; + + pinctrl_sdhci1_default: sdhci1-default { + mux { + groups = "sdio1_0_grp"; + function = "sdio1"; + }; + + conf { + groups = "sdio1_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + drive-strength = <12>; + }; + + mux-cd { + groups = "sdio1_cd_0_grp"; + function = "sdio1_cd"; + }; + + conf-cd { + groups = "sdio1_cd_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_uart0_default: uart0-default { + mux { + groups = "uart0_4_grp"; + function = "uart0"; + }; + + conf { + groups = "uart0_4_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO18"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO19"; + bias-disable; + }; + }; + + pinctrl_uart1_default: uart1-default { + mux { + groups = "uart1_5_grp"; + function = "uart1"; + }; + + conf { + groups = "uart1_5_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO21"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO20"; + bias-disable; + }; + }; + + pinctrl_usb0_default: usb0-default { + mux { + groups = "usb0_0_grp"; + function = "usb0"; + }; + + conf { + groups = "usb0_0_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO52", "MIO53", "MIO55"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO54", "MIO56", "MIO57", "MIO58", "MIO59", + "MIO60", "MIO61", "MIO62", "MIO63"; + bias-disable; + }; + }; }; &psgtr { @@ -184,6 +423,23 @@ &psgtr { clock-names = "ref1", "ref2", "ref3"; }; +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; /* n25q512a 128MiB */ + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; + spi-max-frequency = <108000000>; /* Based on DC1 spec */ + }; +}; + +&rtc { + status = "okay"; +}; + &sata { status = "okay"; /* SATA OOB timing settings */ @@ -203,22 +459,38 @@ &sata { &sdhci1 { status = "okay"; no-1-8-v; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci1_default>; xlnx,mio-bank = <1>; disable-wp; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; }; &uart1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1_default>; }; /* ULPI SMSC USB3320 */ &usb0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0_default>; + phy-names = "usb3-phy"; + phys = <&psgtr 2 PHY_TYPE_USB3 0 2>; +}; + +&dwc3_0 { status = "okay"; dr_mode = "host"; + snps,usb3_lpm_capable; + maximum-speed = "super-speed"; }; &watchdog0 { diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revC.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revC.dts index 7f2e32831b..96feaad301 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revC.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revC.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZCU104 * - * (C) Copyright 2017 - 2020, Xilinx, Inc. + * (C) Copyright 2017 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -12,6 +12,7 @@ #include "zynqmp.dtsi" #include "zynqmp-clk-ccf.dtsi" #include +#include #include / { @@ -22,10 +23,13 @@ aliases { ethernet0 = &gem3; i2c0 = &i2c1; mmc0 = &sdhci1; + nvmem0 = &eeprom; rtc0 = &rtc; serial0 = &uart0; serial1 = &uart1; serial2 = &dcc; + spi0 = &qspi; + usb0 = &usb0; }; chosen { @@ -64,6 +68,8 @@ clock_8t49n287_3: clk27 { &can1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can1_default>; }; &dcc { @@ -106,6 +112,8 @@ &gem3 { status = "okay"; phy-handle = <&phy0>; phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gem3_default>; phy0: ethernet-phy@c { reg = <0xc>; ti,rx-internal-delay = <0x8>; @@ -122,6 +130,11 @@ &gpio { &i2c1 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + scl-gpios = <&gpio 16 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 17 GPIO_ACTIVE_HIGH>; tca6416_u97: gpio@20 { compatible = "ti,tca6416"; @@ -172,9 +185,7 @@ i2c@1 { #address-cells = <1>; #size-cells = <0>; reg = <1>; - clock_8t49n287: clock-generator@6c { /* 8T49N287 - u182 */ - reg = <0x6c>; - }; + /* 8T49N287 - u182 */ }; i2c@2 { @@ -219,18 +230,202 @@ i2c@7 { }; }; -&qspi { +&pinctrl0 { status = "okay"; - flash@0 { - compatible = "m25p80", "jedec,spi-nor"; /* n25q512a 128MiB */ - #address-cells = <1>; - #size-cells = <1>; - reg = <0x0>; - }; -}; -&rtc { - status = "okay"; + pinctrl_can1_default: can1-default { + mux { + function = "can1"; + groups = "can1_6_grp"; + }; + + conf { + groups = "can1_6_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO25"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO24"; + bias-disable; + }; + }; + + pinctrl_i2c1_default: i2c1-default { + mux { + groups = "i2c1_4_grp"; + function = "i2c1"; + }; + + conf { + groups = "i2c1_4_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + }; + + pinctrl_i2c1_gpio: i2c1-gpio { + mux { + groups = "gpio0_16_grp", "gpio0_17_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_16_grp", "gpio0_17_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + }; + + pinctrl_gem3_default: gem3-default { + mux { + function = "ethernet3"; + groups = "ethernet3_0_grp"; + }; + + conf { + groups = "ethernet3_0_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO70", "MIO71", "MIO72", "MIO73", "MIO74", + "MIO75"; + bias-high-impedance; + low-power-disable; + }; + + conf-tx { + pins = "MIO64", "MIO65", "MIO66", "MIO67", "MIO68", + "MIO69"; + bias-disable; + low-power-enable; + }; + + mux-mdio { + function = "mdio3"; + groups = "mdio3_0_grp"; + }; + + conf-mdio { + groups = "mdio3_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + }; + + pinctrl_sdhci1_default: sdhci1-default { + mux { + groups = "sdio1_0_grp"; + function = "sdio1"; + }; + + conf { + groups = "sdio1_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + drive-strength = <12>; + }; + + mux-cd { + groups = "sdio1_cd_0_grp"; + function = "sdio1_cd"; + }; + + conf-cd { + groups = "sdio1_cd_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_uart0_default: uart0-default { + mux { + groups = "uart0_4_grp"; + function = "uart0"; + }; + + conf { + groups = "uart0_4_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO18"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO19"; + bias-disable; + }; + }; + + pinctrl_uart1_default: uart1-default { + mux { + groups = "uart1_5_grp"; + function = "uart1"; + }; + + conf { + groups = "uart1_5_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO21"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO20"; + bias-disable; + }; + }; + + pinctrl_usb0_default: usb0-default { + mux { + groups = "usb0_0_grp"; + function = "usb0"; + }; + + conf { + groups = "usb0_0_grp"; + slew-rate = ; + power-source = ; + drive-strength = <12>; + }; + + conf-rx { + pins = "MIO52", "MIO53", "MIO55"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO54", "MIO56", "MIO57", "MIO58", "MIO59", + "MIO60", "MIO61", "MIO62", "MIO63"; + bias-disable; + }; + }; }; &psgtr { @@ -240,6 +435,23 @@ &psgtr { clock-names = "ref1", "ref2", "ref3"; }; +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; /* n25q512a 128MiB */ + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; + spi-max-frequency = <108000000>; /* Based on DC1 spec */ + }; +}; + +&rtc { + status = "okay"; +}; + &sata { status = "okay"; /* SATA OOB timing settings */ @@ -259,22 +471,38 @@ &sata { &sdhci1 { status = "okay"; no-1-8-v; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci1_default>; xlnx,mio-bank = <1>; disable-wp; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; }; &uart1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1_default>; }; /* ULPI SMSC USB3320 */ &usb0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0_default>; + phy-names = "usb3-phy"; + phys = <&psgtr 2 PHY_TYPE_USB3 0 2>; +}; + +&dwc3_0 { status = "okay"; dr_mode = "host"; + snps,usb3_lpm_capable; + maximum-speed = "super-speed"; }; &watchdog0 { diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts index eff7c64470..20b7c75bb1 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZCU106 * - * (C) Copyright 2016 - 2019, Xilinx, Inc. + * (C) Copyright 2016 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -13,6 +13,7 @@ #include "zynqmp-clk-ccf.dtsi" #include #include +#include #include / { @@ -24,10 +25,13 @@ aliases { i2c0 = &i2c0; i2c1 = &i2c1; mmc0 = &sdhci1; + nvmem0 = &eeprom; rtc0 = &rtc; serial0 = &uart0; serial1 = &uart1; serial2 = &dcc; + spi0 = &qspi; + usb0 = &usb0; }; chosen { @@ -150,24 +154,14 @@ refhdmi: refhdmi { &can1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can1_default>; }; &dcc { status = "okay"; }; -&zynqmp_dpdma { - status = "okay"; -}; - -&zynqmp_dpsub { - status = "okay"; - phy-names = "dp-phy0", "dp-phy1"; - phys = <&psgtr 1 PHY_TYPE_DP 0 3>, - <&psgtr 0 PHY_TYPE_DP 1 3>; -}; - -/* fpd_dma clk 667MHz, lpd_dma 500MHz */ &fpd_dma_chan1 { status = "okay"; }; @@ -204,6 +198,8 @@ &gem3 { status = "okay"; phy-handle = <&phy0>; phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gem3_default>; phy0: ethernet-phy@c { reg = <0xc>; ti,rx-internal-delay = <0x8>; @@ -215,11 +211,18 @@ phy0: ethernet-phy@c { &gpio { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gpio_default>; }; &i2c0 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c0_default>; + pinctrl-1 = <&pinctrl_i2c0_gpio>; + scl-gpios = <&gpio 14 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 15 GPIO_ACTIVE_HIGH>; tca6416_u97: gpio@20 { compatible = "ti,tca6416"; @@ -478,6 +481,11 @@ max20751@73 { /* u96 */ &i2c1 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + scl-gpios = <&gpio 16 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 17 GPIO_ACTIVE_HIGH>; /* PL i2c via PCA9306 - u45 */ i2c-mux@74 { /* u34 */ @@ -652,6 +660,269 @@ i2c@7 { }; }; +&pinctrl0 { + status = "okay"; + pinctrl_i2c0_default: i2c0-default { + mux { + groups = "i2c0_3_grp"; + function = "i2c0"; + }; + + conf { + groups = "i2c0_3_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c0_gpio: i2c0-gpio { + mux { + groups = "gpio0_14_grp", "gpio0_15_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_14_grp", "gpio0_15_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_default: i2c1-default { + mux { + groups = "i2c1_4_grp"; + function = "i2c1"; + }; + + conf { + groups = "i2c1_4_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_gpio: i2c1-gpio { + mux { + groups = "gpio0_16_grp", "gpio0_17_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_16_grp", "gpio0_17_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_uart0_default: uart0-default { + mux { + groups = "uart0_4_grp"; + function = "uart0"; + }; + + conf { + groups = "uart0_4_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO18"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO19"; + bias-disable; + }; + }; + + pinctrl_uart1_default: uart1-default { + mux { + groups = "uart1_5_grp"; + function = "uart1"; + }; + + conf { + groups = "uart1_5_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO21"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO20"; + bias-disable; + }; + }; + + pinctrl_usb0_default: usb0-default { + mux { + groups = "usb0_0_grp"; + function = "usb0"; + }; + + conf { + groups = "usb0_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO52", "MIO53", "MIO55"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO54", "MIO56", "MIO57", "MIO58", "MIO59", + "MIO60", "MIO61", "MIO62", "MIO63"; + bias-disable; + }; + }; + + pinctrl_gem3_default: gem3-default { + mux { + function = "ethernet3"; + groups = "ethernet3_0_grp"; + }; + + conf { + groups = "ethernet3_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO70", "MIO71", "MIO72", "MIO73", "MIO74", + "MIO75"; + bias-high-impedance; + low-power-disable; + }; + + conf-tx { + pins = "MIO64", "MIO65", "MIO66", "MIO67", "MIO68", + "MIO69"; + bias-disable; + low-power-enable; + }; + + mux-mdio { + function = "mdio3"; + groups = "mdio3_0_grp"; + }; + + conf-mdio { + groups = "mdio3_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + }; + + pinctrl_can1_default: can1-default { + mux { + function = "can1"; + groups = "can1_6_grp"; + }; + + conf { + groups = "can1_6_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO25"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO24"; + bias-disable; + }; + }; + + pinctrl_sdhci1_default: sdhci1-default { + mux { + groups = "sdio1_0_grp"; + function = "sdio1"; + }; + + conf { + groups = "sdio1_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + + mux-cd { + groups = "sdio1_cd_0_grp"; + function = "sdio1_cd"; + }; + + conf-cd { + groups = "sdio1_cd_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + + mux-wp { + groups = "sdio1_wp_0_grp"; + function = "sdio1_wp"; + }; + + conf-wp { + groups = "sdio1_wp_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_gpio_default: gpio-default { + mux { + function = "gpio0"; + groups = "gpio0_22_grp", "gpio0_23_grp"; + }; + + conf { + groups = "gpio0_22_grp", "gpio0_23_grp"; + slew-rate = ; + power-source = ; + }; + + mux-msp { + function = "gpio0"; + groups = "gpio0_13_grp", "gpio0_38_grp"; + }; + + conf-msp { + groups = "gpio0_13_grp", "gpio0_38_grp"; + slew-rate = ; + power-source = ; + }; + + conf-pull-up { + pins = "MIO22"; + bias-pull-up; + }; + + conf-pull-none { + pins = "MIO13", "MIO23", "MIO38"; + bias-disable; + }; + }; +}; + &psgtr { status = "okay"; /* nc, sata, usb3, dp */ @@ -659,6 +930,19 @@ &psgtr { clock-names = "ref1", "ref2", "ref3"; }; +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; /* 16MB + 16MB */ + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; /* FIXME also DUAL configuration possible */ + spi-max-frequency = <108000000>; /* Based on DC1 spec */ + }; +}; + &rtc { status = "okay"; }; @@ -681,24 +965,54 @@ &sata { /* SD1 with level shifter */ &sdhci1 { status = "okay"; + /* + * This property should be removed for supporting UHS mode + */ no-1-8-v; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci1_default>; xlnx,mio-bank = <1>; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; }; &uart1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1_default>; }; /* ULPI SMSC USB3320 */ &usb0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0_default>; + phy-names = "usb3-phy"; + phys = <&psgtr 2 PHY_TYPE_USB3 0 2>; +}; + +&dwc3_0 { status = "okay"; dr_mode = "host"; + snps,usb3_lpm_capable; + maximum-speed = "super-speed"; }; &watchdog0 { status = "okay"; }; + +&zynqmp_dpdma { + status = "okay"; +}; + +&zynqmp_dpsub { + status = "okay"; + phy-names = "dp-phy0", "dp-phy1"; + phys = <&psgtr 1 PHY_TYPE_DP 0 3>, + <&psgtr 0 PHY_TYPE_DP 1 3>; +}; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu111-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu111-revA.dts index d4b68f0d00..e36df6adbe 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu111-revA.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu111-revA.dts @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP ZCU111 * - * (C) Copyright 2017 - 2019, Xilinx, Inc. + * (C) Copyright 2017 - 2021, Xilinx, Inc. * * Michal Simek */ @@ -13,6 +13,7 @@ #include "zynqmp-clk-ccf.dtsi" #include #include +#include #include / { @@ -24,9 +25,12 @@ aliases { i2c0 = &i2c0; i2c1 = &i2c1; mmc0 = &sdhci1; + nvmem0 = &eeprom; rtc0 = &rtc; serial0 = &uart0; serial1 = &dcc; + spi0 = &qspi; + usb0 = &usb0; }; chosen { @@ -166,6 +170,8 @@ &gem3 { status = "okay"; phy-handle = <&phy0>; phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gem3_default>; phy0: ethernet-phy@c { reg = <0xc>; ti,rx-internal-delay = <0x8>; @@ -177,11 +183,18 @@ phy0: ethernet-phy@c { &gpio { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gpio_default>; }; &i2c0 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c0_default>; + pinctrl-1 = <&pinctrl_i2c0_gpio>; + scl-gpios = <&gpio 14 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 15 GPIO_ACTIVE_HIGH>; tca6416_u22: gpio@20 { compatible = "ti,tca6416"; @@ -326,13 +339,16 @@ i2c@2 { #address-cells = <1>; #size-cells = <0>; reg = <2>; - irps5401_43: irps54012@43 { /* IRPS5401 - u53 check these */ + irps5401_43: irps5401@43 { /* IRPS5401 - u53 check these */ + compatible = "infineon,irps5401"; reg = <0x43>; }; - irps5401_44: irps54012@44 { /* IRPS5401 - u55 */ + irps5401_44: irps5401@44 { /* IRPS5401 - u55 */ + compatible = "infineon,irps5401"; reg = <0x44>; }; - irps5401_45: irps54012@45 { /* IRPS5401 - u57 */ + irps5401_45: irps5401@45 { /* IRPS5401 - u57 */ + compatible = "infineon,irps5401"; reg = <0x45>; }; /* u68 IR38064 +0 */ @@ -354,6 +370,11 @@ i2c@3 { &i2c1 { status = "okay"; clock-frequency = <400000>; + pinctrl-names = "default", "gpio"; + pinctrl-0 = <&pinctrl_i2c1_default>; + pinctrl-1 = <&pinctrl_i2c1_gpio>; + scl-gpios = <&gpio 16 GPIO_ACTIVE_HIGH>; + sda-gpios = <&gpio 17 GPIO_ACTIVE_HIGH>; i2c-mux@74 { /* u26 */ compatible = "nxp,pca9548"; @@ -455,9 +476,7 @@ i2c@4 { #address-cells = <1>; #size-cells = <0>; reg = <4>; - si5382: clock-generator@69 { /* SI5382 - u48 */ - reg = <0x69>; - }; + /* SI5382 - u48 */ }; i2c@5 { #address-cells = <1>; @@ -542,13 +561,230 @@ i2c@7 { }; }; +&pinctrl0 { + status = "okay"; + pinctrl_i2c0_default: i2c0-default { + mux { + groups = "i2c0_3_grp"; + function = "i2c0"; + }; + + conf { + groups = "i2c0_3_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c0_gpio: i2c0-gpio { + mux { + groups = "gpio0_14_grp", "gpio0_15_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_14_grp", "gpio0_15_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_default: i2c1-default { + mux { + groups = "i2c1_4_grp"; + function = "i2c1"; + }; + + conf { + groups = "i2c1_4_grp"; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_i2c1_gpio: i2c1-gpio { + mux { + groups = "gpio0_16_grp", "gpio0_17_grp"; + function = "gpio0"; + }; + + conf { + groups = "gpio0_16_grp", "gpio0_17_grp"; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_uart0_default: uart0-default { + mux { + groups = "uart0_4_grp"; + function = "uart0"; + }; + + conf { + groups = "uart0_4_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO18"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO19"; + bias-disable; + }; + }; + + pinctrl_usb0_default: usb0-default { + mux { + groups = "usb0_0_grp"; + function = "usb0"; + }; + + conf { + groups = "usb0_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO52", "MIO53", "MIO55"; + bias-high-impedance; + }; + + conf-tx { + pins = "MIO54", "MIO56", "MIO57", "MIO58", "MIO59", + "MIO60", "MIO61", "MIO62", "MIO63"; + bias-disable; + }; + }; + + pinctrl_gem3_default: gem3-default { + mux { + function = "ethernet3"; + groups = "ethernet3_0_grp"; + }; + + conf { + groups = "ethernet3_0_grp"; + slew-rate = ; + power-source = ; + }; + + conf-rx { + pins = "MIO70", "MIO71", "MIO72", "MIO73", "MIO74", + "MIO75"; + bias-high-impedance; + low-power-disable; + }; + + conf-tx { + pins = "MIO64", "MIO65", "MIO66", "MIO67", "MIO68", + "MIO69"; + bias-disable; + low-power-enable; + }; + + mux-mdio { + function = "mdio3"; + groups = "mdio3_0_grp"; + }; + + conf-mdio { + groups = "mdio3_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + }; + + pinctrl_sdhci1_default: sdhci1-default { + mux { + groups = "sdio1_0_grp"; + function = "sdio1"; + }; + + conf { + groups = "sdio1_0_grp"; + slew-rate = ; + power-source = ; + bias-disable; + }; + + mux-cd { + groups = "sdio1_cd_0_grp"; + function = "sdio1_cd"; + }; + + conf-cd { + groups = "sdio1_cd_0_grp"; + bias-high-impedance; + bias-pull-up; + slew-rate = ; + power-source = ; + }; + }; + + pinctrl_gpio_default: gpio-default { + mux { + function = "gpio0"; + groups = "gpio0_22_grp", "gpio0_23_grp"; + }; + + conf { + groups = "gpio0_22_grp", "gpio0_23_grp"; + slew-rate = ; + power-source = ; + }; + + mux-msp { + function = "gpio0"; + groups = "gpio0_13_grp", "gpio0_38_grp"; + }; + + conf-msp { + groups = "gpio0_13_grp", "gpio0_38_grp"; + slew-rate = ; + power-source = ; + }; + + conf-pull-up { + pins = "MIO22"; + bias-pull-up; + }; + + conf-pull-none { + pins = "MIO13", "MIO23", "MIO38"; + bias-disable; + }; + }; +}; + &psgtr { status = "okay"; - /* nc, sata, usb3, dp */ - clocks = <&si5341 0 3>, <&si5341 0 2>, <&si5341 0 0>; + /* nc, dp, usb3, sata */ + clocks = <&si5341 0 0>, <&si5341 0 2>, <&si5341 0 3>; clock-names = "ref1", "ref2", "ref3"; }; +&qspi { + status = "okay"; + flash@0 { + compatible = "m25p80", "jedec,spi-nor"; /* 16MB + 16MB */ + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <4>; /* FIXME also DUAL configuration possible */ + spi-max-frequency = <108000000>; /* Based on DC1 spec */ + }; +}; + &rtc { status = "okay"; }; @@ -565,24 +801,42 @@ &sata { ceva,p1-burst-params = /bits/ 8 <0x13 0x08 0x4A 0x06>; ceva,p1-retry-params = /bits/ 16 <0x96A4 0x3FFC>; phy-names = "sata-phy"; - phys = <&psgtr 3 PHY_TYPE_SATA 1 1>; + phys = <&psgtr 3 PHY_TYPE_SATA 1 3>; }; /* SD1 with level shifter */ &sdhci1 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sdhci1_default>; + disable-wp; + /* + * This property should be removed for supporting UHS mode + */ no-1-8-v; xlnx,mio-bank = <1>; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart0_default>; }; /* ULPI SMSC USB3320 */ &usb0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0_default>; + phy-names = "usb3-phy"; + phys = <&psgtr 2 PHY_TYPE_USB3 0 2>; +}; + +&dwc3_0 { status = "okay"; dr_mode = "host"; + snps,usb3_lpm_capable; + maximum-speed = "super-speed"; }; &zynqmp_dpdma { diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi index 8278876ad3..74e66443e4 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi +++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi @@ -2,7 +2,7 @@ /* * dts file for Xilinx ZynqMP * - * (C) Copyright 2014 - 2019, Xilinx, Inc. + * (C) Copyright 2014 - 2021, Xilinx, Inc. * * Michal Simek * @@ -156,21 +156,6 @@ zynqmp_power: zynqmp-power { mbox-names = "tx", "rx"; }; - zynqmp_clk: clock-controller { - #clock-cells = <1>; - compatible = "xlnx,zynqmp-clk"; - clocks = <&pss_ref_clk>, - <&video_clk>, - <&pss_alt_ref_clk>, - <&aux_ref_clk>, - <>_crx_ref_clk>; - clock-names = "pss_ref_clk", - "video_clk", - "pss_alt_ref_clk", - "aux_ref_clk", - "gt_crx_ref_clk"; - }; - nvmem_firmware { compatible = "xlnx,zynqmp-nvmem-fw"; #address-cells = <1>; @@ -193,6 +178,11 @@ zynqmp_reset: reset-controller { compatible = "xlnx,zynqmp-reset"; #reset-cells = <1>; }; + + pinctrl0: pinctrl { + compatible = "xlnx,zynqmp-pinctrl"; + status = "disabled"; + }; }; }; @@ -245,6 +235,7 @@ can1: can@ff070000 { cci: cci@fd6e0000 { compatible = "arm,cci-400"; + status = "disabled"; reg = <0x0 0xfd6e0000 0x0 0x9000>; ranges = <0x0 0x0 0xfd6e0000 0x10000>; #address-cells = <1>; @@ -630,6 +621,8 @@ pcie: pcie@fd0e0000 { <0x0 0x0 0x0 0x2 &pcie_intc 0x2>, <0x0 0x0 0x0 0x3 &pcie_intc 0x3>, <0x0 0x0 0x0 0x4 &pcie_intc 0x4>; + #stream-id-cells = <1>; + iommus = <&smmu 0x4d0>; power-domains = <&zynqmp_firmware PD_PCIE>; pcie_intc: legacy-interrupt-controller { interrupt-controller; @@ -670,7 +663,7 @@ rtc: rtc@ffa60000 { interrupt-parent = <&gic>; interrupts = <0 26 4>, <0 27 4>; interrupt-names = "alarm", "sec"; - calibration = <0x8000>; + calibration = <0x7FFF>; }; sata: ahci@fd0c0000 { @@ -680,6 +673,7 @@ sata: ahci@fd0c0000 { interrupt-parent = <&gic>; interrupts = <0 133 4>; power-domains = <&zynqmp_firmware PD_SATA>; + resets = <&zynqmp_reset ZYNQMP_RESET_SATA>; #stream-id-cells = <4>; iommus = <&smmu 0x4c0>, <&smmu 0x4c1>, <&smmu 0x4c2>, <&smmu 0x4c3>; @@ -811,24 +805,58 @@ uart1: serial@ff010000 { power-domains = <&zynqmp_firmware PD_UART_1>; }; - usb0: usb@fe200000 { - compatible = "snps,dwc3"; + usb0: usb@ff9d0000 { + #address-cells = <2>; + #size-cells = <2>; status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 65 4>; - reg = <0x0 0xfe200000 0x0 0x40000>; - clock-names = "clk_xin", "clk_ahb"; + compatible = "xlnx,zynqmp-dwc3"; + reg = <0x0 0xff9d0000 0x0 0x100>; + clock-names = "bus_clk", "ref_clk"; power-domains = <&zynqmp_firmware PD_USB_0>; + resets = <&zynqmp_reset ZYNQMP_RESET_USB0_CORERESET>, + <&zynqmp_reset ZYNQMP_RESET_USB0_HIBERRESET>, + <&zynqmp_reset ZYNQMP_RESET_USB0_APB>; + reset-names = "usb_crst", "usb_hibrst", "usb_apbrst"; + ranges; + + dwc3_0: usb@fe200000 { + compatible = "snps,dwc3"; + reg = <0x0 0xfe200000 0x0 0x40000>; + interrupt-parent = <&gic>; + interrupt-names = "dwc_usb3", "otg"; + interrupts = <0 65 4>, <0 69 4>; + #stream-id-cells = <1>; + iommus = <&smmu 0x860>; + snps,quirk-frame-length-adjustment = <0x20>; + /* dma-coherent; */ + }; }; - usb1: usb@fe300000 { - compatible = "snps,dwc3"; + usb1: usb@ff9e0000 { + #address-cells = <2>; + #size-cells = <2>; status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 70 4>; - reg = <0x0 0xfe300000 0x0 0x40000>; - clock-names = "clk_xin", "clk_ahb"; + compatible = "xlnx,zynqmp-dwc3"; + reg = <0x0 0xff9e0000 0x0 0x100>; + clock-names = "bus_clk", "ref_clk"; power-domains = <&zynqmp_firmware PD_USB_1>; + resets = <&zynqmp_reset ZYNQMP_RESET_USB1_CORERESET>, + <&zynqmp_reset ZYNQMP_RESET_USB1_HIBERRESET>, + <&zynqmp_reset ZYNQMP_RESET_USB1_APB>; + reset-names = "usb_crst", "usb_hibrst", "usb_apbrst"; + ranges; + + dwc3_1: usb@fe300000 { + compatible = "snps,dwc3"; + reg = <0x0 0xfe300000 0x0 0x40000>; + interrupt-parent = <&gic>; + interrupt-names = "dwc_usb3", "otg"; + interrupts = <0 70 4>, <0 74 4>; + #stream-id-cells = <1>; + iommus = <&smmu 0x861>; + snps,quirk-frame-length-adjustment = <0x20>; + /* dma-coherent; */ + }; }; watchdog0: watchdog@fd4d0000 { @@ -837,7 +865,8 @@ watchdog0: watchdog@fd4d0000 { interrupt-parent = <&gic>; interrupts = <0 113 1>; reg = <0x0 0xfd4d0000 0x0 0x1000>; - timeout-sec = <10>; + timeout-sec = <60>; + reset-on-timeout; }; lpd_watchdog: watchdog@ff150000 { diff --git a/arch/arm64/configs/bcm2711_defconfig b/arch/arm64/configs/bcm2711_defconfig index 16d756cacc..9d69fe79d9 100644 --- a/arch/arm64/configs/bcm2711_defconfig +++ b/arch/arm64/configs/bcm2711_defconfig @@ -66,8 +66,6 @@ CONFIG_BLK_DEV_THROTTLING=y CONFIG_PARTITION_ADVANCED=y CONFIG_MAC_PARTITION=y CONFIG_BINFMT_MISC=m -CONFIG_CLEANCACHE=y -CONFIG_FRONTSWAP=y CONFIG_CMA=y CONFIG_ZSWAP=y CONFIG_Z3FOLD=m @@ -140,7 +138,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -254,7 +251,6 @@ CONFIG_IP_VS_PE_SIP=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m @@ -277,7 +273,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -457,7 +452,6 @@ CONFIG_MTD_UBI=m CONFIG_OF_CONFIGFS=y CONFIG_ZRAM=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -917,6 +911,7 @@ CONFIG_DRM_TOSHIBA_TC358762=m CONFIG_DRM_V3D=m CONFIG_DRM_VC4=m CONFIG_DRM_VC4_HDMI_CEC=y +CONFIG_DRM_PANEL_MIPI_DBI=m CONFIG_TINYDRM_HX8357D=m CONFIG_TINYDRM_ILI9225=m CONFIG_TINYDRM_ILI9341=m @@ -1217,6 +1212,7 @@ CONFIG_MMC_SDHCI_PLTFM=y CONFIG_MMC_SDHCI_IPROC=y CONFIG_MMC_SPI=m CONFIG_LEDS_CLASS=y +CONFIG_LEDS_CLASS_MULTICOLOR=m CONFIG_LEDS_PCA9532=m CONFIG_LEDS_GPIO=y CONFIG_LEDS_PCA955X=m @@ -1327,7 +1323,6 @@ CONFIG_FB_TFT_TINYLCD=m CONFIG_FB_TFT_TLS8204=m CONFIG_FB_TFT_UC1701=m CONFIG_FB_TFT_UPD161704=m -CONFIG_FB_TFT_WATTEROTT=m CONFIG_BCM2835_VCHIQ=y CONFIG_SND_BCM2835=m CONFIG_VIDEO_BCM2835=m diff --git a/arch/arm64/configs/bcmrpi3_defconfig b/arch/arm64/configs/bcmrpi3_defconfig index c98578ac5d..89439b35e4 100644 --- a/arch/arm64/configs/bcmrpi3_defconfig +++ b/arch/arm64/configs/bcmrpi3_defconfig @@ -67,8 +67,6 @@ CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_PARTITION_ADVANCED=y CONFIG_MAC_PARTITION=y CONFIG_BINFMT_MISC=y -CONFIG_CLEANCACHE=y -CONFIG_FRONTSWAP=y CONFIG_CMA=y CONFIG_ZSWAP=y CONFIG_Z3FOLD=m @@ -138,7 +136,6 @@ CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -252,7 +249,6 @@ CONFIG_IP_VS_PE_SIP=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m @@ -275,7 +271,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NFT_DUP_IPV6=m CONFIG_NFT_FIB_IPV6=m -CONFIG_NF_FLOW_TABLE_IPV6=m CONFIG_IP6_NF_IPTABLES=m CONFIG_IP6_NF_MATCH_AH=m CONFIG_IP6_NF_MATCH_EUI64=m @@ -446,7 +441,6 @@ CONFIG_MTD_UBI=m CONFIG_OF_CONFIGFS=y CONFIG_ZRAM=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -836,6 +830,7 @@ CONFIG_DRM_SIMPLE_BRIDGE=m CONFIG_DRM_TOSHIBA_TC358762=m CONFIG_DRM_VC4=m CONFIG_DRM_VC4_HDMI_CEC=y +CONFIG_DRM_PANEL_MIPI_DBI=m CONFIG_TINYDRM_HX8357D=m CONFIG_DRM_GUD=m CONFIG_FB=y @@ -1080,6 +1075,7 @@ CONFIG_MMC_SDHCI_PLTFM=y CONFIG_MMC_SDHCI_IPROC=m CONFIG_MMC_SPI=m CONFIG_LEDS_CLASS=y +CONFIG_LEDS_CLASS_MULTICOLOR=m CONFIG_LEDS_PCA9532=m CONFIG_LEDS_GPIO=y CONFIG_LEDS_PCA955X=m @@ -1184,7 +1180,6 @@ CONFIG_FB_TFT_TINYLCD=m CONFIG_FB_TFT_TLS8204=m CONFIG_FB_TFT_UC1701=m CONFIG_FB_TFT_UPD161704=m -CONFIG_FB_TFT_WATTEROTT=m CONFIG_BCM2835_VCHIQ=y CONFIG_SND_BCM2835=m CONFIG_VIDEO_BCM2835=m diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 545197bc05..30516dc0b7 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -28,8 +28,6 @@ CONFIG_KALLSYMS_ALL=y # CONFIG_COMPAT_BRK is not set CONFIG_PROFILING=y CONFIG_ARCH_ACTIONS=y -CONFIG_ARCH_AGILEX=y -CONFIG_ARCH_N5X=y CONFIG_ARCH_SUNXI=y CONFIG_ARCH_ALPINE=y CONFIG_ARCH_APPLE=y @@ -63,7 +61,6 @@ CONFIG_ARCH_UNIPHIER=y CONFIG_ARCH_VEXPRESS=y CONFIG_ARCH_VISCONTI=y CONFIG_ARCH_XGENE=y -CONFIG_ARCH_ZX=y CONFIG_ARCH_ZYNQMP=y CONFIG_ARM64_VA_BITS_48=y CONFIG_SCHED_MC=y @@ -201,6 +198,7 @@ CONFIG_BT_HCIUART=m CONFIG_BT_HCIUART_LL=y CONFIG_BT_HCIUART_BCM=y CONFIG_BT_HCIUART_QCA=y +CONFIG_BT_QCOMSMD=m CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_MAC80211_LEDS=y @@ -237,13 +235,13 @@ CONFIG_PCIE_ARMADA_8K=y CONFIG_PCIE_KIRIN=y CONFIG_PCIE_HISI_STB=y CONFIG_PCIE_TEGRA194_HOST=m +CONFIG_PCIE_VISCONTI_HOST=y CONFIG_PCI_ENDPOINT=y CONFIG_PCI_ENDPOINT_CONFIGFS=y CONFIG_PCI_EPF_TEST=m CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_FW_LOADER_USER_HELPER=y -CONFIG_FW_LOADER_USER_HELPER_FALLBACK=y CONFIG_HISILICON_LPC=y CONFIG_FSL_MC_BUS=y CONFIG_TEGRA_ACONNECT=m @@ -365,6 +363,7 @@ CONFIG_MICROSEMI_PHY=y CONFIG_AT803X_PHY=y CONFIG_REALTEK_PHY=y CONFIG_ROCKCHIP_PHY=y +CONFIG_DP83867_PHY=y CONFIG_VITESSE_PHY=y CONFIG_USB_PEGASUS=m CONFIG_USB_RTL8150=m @@ -380,6 +379,7 @@ CONFIG_USB_NET_MCS7830=m CONFIG_ATH10K=m CONFIG_ATH10K_PCI=m CONFIG_ATH10K_SNOC=m +CONFIG_WCN36XX=m CONFIG_BRCMFMAC=m CONFIG_MWIFIEX=m CONFIG_MWIFIEX_PCIE=m @@ -398,6 +398,7 @@ CONFIG_TOUCHSCREEN_EDT_FT5X06=m CONFIG_INPUT_MISC=y CONFIG_INPUT_PM8941_PWRKEY=y CONFIG_INPUT_PM8XXX_VIBRATOR=m +CONFIG_INPUT_PWM_BEEPER=m CONFIG_INPUT_PWM_VIBRA=m CONFIG_INPUT_HISI_POWERKEY=y # CONFIG_SERIO_SERPORT is not set @@ -460,7 +461,9 @@ CONFIG_I2C_PXA=y CONFIG_I2C_QCOM_CCI=m CONFIG_I2C_QCOM_GENI=m CONFIG_I2C_QUP=y +CONFIG_I2C_RIIC=y CONFIG_I2C_RK3X=y +CONFIG_I2C_S3C2410=y CONFIG_I2C_SH_MOBILE=y CONFIG_I2C_TEGRA=y CONFIG_I2C_UNIPHIER_F=y @@ -504,6 +507,7 @@ CONFIG_PINCTRL_IMX8MQ=y CONFIG_PINCTRL_IMX8QM=y CONFIG_PINCTRL_IMX8QXP=y CONFIG_PINCTRL_IMX8DXL=y +CONFIG_PINCTRL_IMX8ULP=y CONFIG_PINCTRL_MSM=y CONFIG_PINCTRL_IPQ8074=y CONFIG_PINCTRL_IPQ6018=y @@ -515,6 +519,7 @@ CONFIG_PINCTRL_QCS404=y CONFIG_PINCTRL_QDF2XXX=y CONFIG_PINCTRL_QCOM_SPMI_PMIC=y CONFIG_PINCTRL_SC7180=y +CONFIG_PINCTRL_SC7280=y CONFIG_PINCTRL_SDM845=y CONFIG_PINCTRL_SM8150=y CONFIG_PINCTRL_SM8250=y @@ -554,6 +559,7 @@ CONFIG_BATTERY_MAX17042=m CONFIG_CHARGER_BQ25890=m CONFIG_CHARGER_BQ25980=m CONFIG_SENSORS_ARM_SCPI=y +CONFIG_SENSORS_JC42=m CONFIG_SENSORS_LM90=m CONFIG_SENSORS_PWM_FAN=m CONFIG_SENSORS_RASPBERRYPI_HWMON=m @@ -579,6 +585,7 @@ CONFIG_TEGRA_BPMP_THERMAL=m CONFIG_TEGRA_SOCTHERM=m CONFIG_QCOM_TSENS=y CONFIG_QCOM_SPMI_TEMP_ALARM=m +CONFIG_QCOM_LMH=m CONFIG_UNIPHIER_THERMAL=y CONFIG_WATCHDOG=y CONFIG_SL28CPLD_WATCHDOG=m @@ -666,6 +673,7 @@ CONFIG_VIDEO_QCOM_VENUS=m CONFIG_SDR_PLATFORM_DRIVERS=y CONFIG_VIDEO_RCAR_DRIF=m CONFIG_VIDEO_IMX219=m +CONFIG_VIDEO_OV5640=m CONFIG_VIDEO_OV5645=m CONFIG_VIDEO_QCOM_CAMSS=m CONFIG_DRM=m @@ -696,6 +704,7 @@ CONFIG_DRM_MSM=m CONFIG_DRM_TEGRA=m CONFIG_DRM_PANEL_LVDS=m CONFIG_DRM_PANEL_SIMPLE=m +CONFIG_DRM_PANEL_EDP=m CONFIG_DRM_PANEL_BOE_TV101WUM_NL6=m CONFIG_DRM_PANEL_MANTIX_MLAF057WE51=m CONFIG_DRM_PANEL_RAYDIUM_RM67191=m @@ -763,6 +772,7 @@ CONFIG_SND_SOC_ROCKCHIP_RT5645=m CONFIG_SND_SOC_RK3399_GRU_SOUND=m CONFIG_SND_SOC_SAMSUNG=y CONFIG_SND_SOC_RCAR=m +CONFIG_SND_SOC_RZ=m CONFIG_SND_SUN4I_I2S=m CONFIG_SND_SUN4I_SPDIF=m CONFIG_SND_SOC_TEGRA=m @@ -771,6 +781,11 @@ CONFIG_SND_SOC_TEGRA210_DMIC=m CONFIG_SND_SOC_TEGRA210_I2S=m CONFIG_SND_SOC_TEGRA186_DSPK=m CONFIG_SND_SOC_TEGRA210_ADMAIF=m +CONFIG_SND_SOC_TEGRA210_MVC=m +CONFIG_SND_SOC_TEGRA210_SFC=m +CONFIG_SND_SOC_TEGRA210_AMX=m +CONFIG_SND_SOC_TEGRA210_ADX=m +CONFIG_SND_SOC_TEGRA210_MIXER=m CONFIG_SND_SOC_TEGRA_AUDIO_GRAPH_CARD=m CONFIG_SND_SOC_AK4613=m CONFIG_SND_SOC_ES7134=m @@ -783,10 +798,12 @@ CONFIG_SND_SOC_RT5659=m CONFIG_SND_SOC_SIMPLE_AMPLIFIER=m CONFIG_SND_SOC_SIMPLE_MUX=m CONFIG_SND_SOC_TAS571X=m +CONFIG_SND_SOC_TLV320AIC32X4_I2C=m CONFIG_SND_SOC_WCD934X=m CONFIG_SND_SOC_WM8904=m CONFIG_SND_SOC_WM8960=m CONFIG_SND_SOC_WM8962=m +CONFIG_SND_SOC_WM8978=m CONFIG_SND_SOC_WSA881X=m CONFIG_SND_SOC_LPASS_WSA_MACRO=m CONFIG_SND_SOC_LPASS_VA_MACRO=m @@ -897,6 +914,7 @@ CONFIG_RTC_DRV_DS1307=m CONFIG_RTC_DRV_HYM8563=m CONFIG_RTC_DRV_MAX77686=y CONFIG_RTC_DRV_RK808=m +CONFIG_RTC_DRV_PCF85063=m CONFIG_RTC_DRV_PCF85363=m CONFIG_RTC_DRV_M41T80=m CONFIG_RTC_DRV_RX8581=m @@ -934,6 +952,7 @@ CONFIG_QCOM_HIDMA_MGMT=y CONFIG_QCOM_HIDMA=y CONFIG_RCAR_DMAC=y CONFIG_RENESAS_USB_DMAC=m +CONFIG_RZ_DMAC=y CONFIG_TI_K3_UDMA=y CONFIG_TI_K3_UDMA_GLUE_LAYER=y CONFIG_VFIO=y @@ -947,6 +966,7 @@ CONFIG_MFD_CROS_EC_DEV=y CONFIG_STAGING=y CONFIG_STAGING_MEDIA=y CONFIG_VIDEO_HANTRO=m +CONFIG_VIDEO_IMX_MEDIA=m CONFIG_CHROME_PLATFORMS=y CONFIG_CROS_EC=y CONFIG_CROS_EC_I2C=y @@ -968,6 +988,7 @@ CONFIG_CLK_IMX8MN=y CONFIG_CLK_IMX8MP=y CONFIG_CLK_IMX8MQ=y CONFIG_CLK_IMX8QXP=y +CONFIG_CLK_IMX8ULP=y CONFIG_TI_SCI_CLK=y CONFIG_COMMON_CLK_QCOM=y CONFIG_QCOM_A53PLL=y @@ -983,6 +1004,7 @@ CONFIG_MSM_MMCC_8996=y CONFIG_MSM_GCC_8998=y CONFIG_QCS_GCC_404=y CONFIG_SC_GCC_7180=y +CONFIG_SC_GCC_7280=y CONFIG_SDM_CAMCC_845=m CONFIG_SDM_GCC_845=y CONFIG_SDM_GPUCC_845=y @@ -1015,6 +1037,8 @@ CONFIG_REMOTEPROC=y CONFIG_QCOM_Q6V5_MSS=m CONFIG_QCOM_Q6V5_PAS=m CONFIG_QCOM_SYSMON=m +CONFIG_QCOM_WCNSS_PIL=m +CONFIG_RPMSG_CHAR=m CONFIG_RPMSG_QCOM_GLINK_RPM=y CONFIG_RPMSG_QCOM_GLINK_SMEM=m CONFIG_RPMSG_QCOM_SMD=y @@ -1038,6 +1062,8 @@ CONFIG_QCOM_SMD_RPM=y CONFIG_QCOM_SMP2P=y CONFIG_QCOM_SMSM=y CONFIG_QCOM_SOCINFO=m +CONFIG_QCOM_WCNSS_CTRL=m +CONFIG_QCOM_STATS=m CONFIG_QCOM_APR=m CONFIG_ARCH_R8A774A1=y CONFIG_ARCH_R8A774B1=y @@ -1053,6 +1079,7 @@ CONFIG_ARCH_R8A77980=y CONFIG_ARCH_R8A77990=y CONFIG_ARCH_R8A77995=y CONFIG_ARCH_R8A779A0=y +CONFIG_ARCH_R8A779F0=y CONFIG_ARCH_R9A07G044=y CONFIG_ROCKCHIP_PM_DOMAINS=y CONFIG_ARCH_TEGRA_132_SOC=y @@ -1073,6 +1100,7 @@ CONFIG_MAX9611=m CONFIG_QCOM_SPMI_VADC=m CONFIG_QCOM_SPMI_ADC5=m CONFIG_ROCKCHIP_SARADC=m +CONFIG_RZG2L_ADC=m CONFIG_IIO_CROS_EC_SENSORS_CORE=m CONFIG_IIO_CROS_EC_SENSORS=m CONFIG_IIO_ST_LSM6DSX=m @@ -1090,6 +1118,7 @@ CONFIG_PWM_MESON=m CONFIG_PWM_MTK_DISP=m CONFIG_PWM_MEDIATEK=m CONFIG_PWM_RCAR=m +CONFIG_PWM_RENESAS_TPU=m CONFIG_PWM_ROCKCHIP=y CONFIG_PWM_SAMSUNG=y CONFIG_PWM_SL28CPLD=m @@ -1101,6 +1130,7 @@ CONFIG_QCOM_PDC=y CONFIG_RESET_IMX7=y CONFIG_RESET_QCOM_AOSS=y CONFIG_RESET_QCOM_PDC=m +CONFIG_RESET_RZG2L_USBPHY_CTRL=y CONFIG_RESET_TI_SCI=y CONFIG_PHY_XGENE=y CONFIG_PHY_SUN4I_USB=y @@ -1155,10 +1185,13 @@ CONFIG_SLIM_QCOM_NGD_CTRL=m CONFIG_MUX_MMIO=y CONFIG_INTERCONNECT=y CONFIG_INTERCONNECT_IMX=m +CONFIG_INTERCONNECT_IMX8MM=m +CONFIG_INTERCONNECT_IMX8MN=m CONFIG_INTERCONNECT_IMX8MQ=m CONFIG_INTERCONNECT_QCOM=y CONFIG_INTERCONNECT_QCOM_MSM8916=m CONFIG_INTERCONNECT_QCOM_OSM_L3=m +CONFIG_INTERCONNECT_QCOM_SC7280=y CONFIG_INTERCONNECT_QCOM_SDM845=y CONFIG_INTERCONNECT_QCOM_SM8150=m CONFIG_INTERCONNECT_QCOM_SM8250=m diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 55f1945009..addfa41365 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -88,16 +88,12 @@ config CRYPTO_AES_ARM64_CE_BLK depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_AES_ARM64_CE - select CRYPTO_AES_ARM64 - select CRYPTO_SIMD config CRYPTO_AES_ARM64_NEON_BLK tristate "AES in ECB/CBC/CTR/XTS modes using NEON instructions" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER - select CRYPTO_AES_ARM64 select CRYPTO_LIB_AES - select CRYPTO_SIMD config CRYPTO_CHACHA20_NEON tristate "ChaCha20, XChaCha20, and XChaCha12 stream ciphers using NEON instructions" @@ -122,8 +118,6 @@ config CRYPTO_AES_ARM64_BS depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_AES_ARM64_NEON_BLK - select CRYPTO_AES_ARM64 select CRYPTO_LIB_AES - select CRYPTO_SIMD endif diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S index 99a028e298..b03f7f71f8 100644 --- a/arch/arm64/crypto/aes-ce-ccm-core.S +++ b/arch/arm64/crypto/aes-ce-ccm-core.S @@ -12,22 +12,21 @@ .arch armv8-a+crypto /* - * void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, - * u32 *macp, u8 const rk[], u32 rounds); + * u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, + * u32 macp, u8 const rk[], u32 rounds); */ SYM_FUNC_START(ce_aes_ccm_auth_data) - ldr w8, [x3] /* leftover from prev round? */ ld1 {v0.16b}, [x0] /* load mac */ - cbz w8, 1f - sub w8, w8, #16 + cbz w3, 1f + sub w3, w3, #16 eor v1.16b, v1.16b, v1.16b 0: ldrb w7, [x1], #1 /* get 1 byte of input */ subs w2, w2, #1 - add w8, w8, #1 + add w3, w3, #1 ins v1.b[0], w7 ext v1.16b, v1.16b, v1.16b, #1 /* rotate in the input bytes */ beq 8f /* out of input? */ - cbnz w8, 0b + cbnz w3, 0b eor v0.16b, v0.16b, v1.16b 1: ld1 {v3.4s}, [x4] /* load first round key */ prfm pldl1strm, [x1] @@ -62,7 +61,7 @@ SYM_FUNC_START(ce_aes_ccm_auth_data) beq 10f adds w2, w2, #16 beq 10f - mov w8, w2 + mov w3, w2 7: ldrb w7, [x1], #1 umov w6, v0.b[0] eor w6, w6, w7 @@ -71,15 +70,15 @@ SYM_FUNC_START(ce_aes_ccm_auth_data) beq 10f ext v0.16b, v0.16b, v0.16b, #1 /* rotate out the mac bytes */ b 7b -8: cbz w8, 91f - mov w7, w8 - add w8, w8, #16 +8: cbz w3, 91f + mov w7, w3 + add w3, w3, #16 9: ext v1.16b, v1.16b, v1.16b, #1 adds w7, w7, #1 bne 9b 91: eor v0.16b, v0.16b, v1.16b st1 {v0.16b}, [x0] -10: str w8, [x3] +10: mov w0, w3 ret SYM_FUNC_END(ce_aes_ccm_auth_data) @@ -124,6 +123,7 @@ SYM_FUNC_START(ce_aes_ccm_final) SYM_FUNC_END(ce_aes_ccm_final) .macro aes_ccm_do_crypt,enc + cbz x2, 5f ldr x8, [x6, #8] /* load lower ctr */ ld1 {v0.16b}, [x5] /* load mac */ CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */ diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index f6d19b0dc8..c4f14415f5 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -6,12 +6,10 @@ */ #include -#include #include #include #include #include -#include #include #include @@ -29,8 +27,8 @@ static int num_rounds(struct crypto_aes_ctx *ctx) return 6 + ctx->key_length / 4; } -asmlinkage void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, - u32 *macp, u32 const rk[], u32 rounds); +asmlinkage u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, + u32 macp, u32 const rk[], u32 rounds); asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes, u32 const rk[], u32 rounds, u8 mac[], @@ -96,41 +94,6 @@ static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen) return 0; } -static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[], - u32 abytes, u32 *macp) -{ - if (crypto_simd_usable()) { - kernel_neon_begin(); - ce_aes_ccm_auth_data(mac, in, abytes, macp, key->key_enc, - num_rounds(key)); - kernel_neon_end(); - } else { - if (*macp > 0 && *macp < AES_BLOCK_SIZE) { - int added = min(abytes, AES_BLOCK_SIZE - *macp); - - crypto_xor(&mac[*macp], in, added); - - *macp += added; - in += added; - abytes -= added; - } - - while (abytes >= AES_BLOCK_SIZE) { - aes_encrypt(key, mac, mac); - crypto_xor(mac, in, AES_BLOCK_SIZE); - - in += AES_BLOCK_SIZE; - abytes -= AES_BLOCK_SIZE; - } - - if (abytes > 0) { - aes_encrypt(key, mac, mac); - crypto_xor(mac, in, abytes); - *macp = abytes; - } - } -} - static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) { struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -150,7 +113,8 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) ltag.len = 6; } - ccm_update_mac(ctx, mac, (u8 *)<ag, ltag.len, &macp); + macp = ce_aes_ccm_auth_data(mac, (u8 *)<ag, ltag.len, macp, + ctx->key_enc, num_rounds(ctx)); scatterwalk_start(&walk, req->src); do { @@ -161,8 +125,16 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) scatterwalk_start(&walk, sg_next(walk.sg)); n = scatterwalk_clamp(&walk, len); } + n = min_t(u32, n, SZ_4K); /* yield NEON at least every 4k */ p = scatterwalk_map(&walk); - ccm_update_mac(ctx, mac, p, n, &macp); + + macp = ce_aes_ccm_auth_data(mac, p, n, macp, ctx->key_enc, + num_rounds(ctx)); + + if (len / SZ_4K > (len - n) / SZ_4K) { + kernel_neon_end(); + kernel_neon_begin(); + } len -= n; scatterwalk_unmap(p); @@ -171,54 +143,6 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) } while (len); } -static int ccm_crypt_fallback(struct skcipher_walk *walk, u8 mac[], u8 iv0[], - struct crypto_aes_ctx *ctx, bool enc) -{ - u8 buf[AES_BLOCK_SIZE]; - int err = 0; - - while (walk->nbytes) { - int blocks = walk->nbytes / AES_BLOCK_SIZE; - u32 tail = walk->nbytes % AES_BLOCK_SIZE; - u8 *dst = walk->dst.virt.addr; - u8 *src = walk->src.virt.addr; - u32 nbytes = walk->nbytes; - - if (nbytes == walk->total && tail > 0) { - blocks++; - tail = 0; - } - - do { - u32 bsize = AES_BLOCK_SIZE; - - if (nbytes < AES_BLOCK_SIZE) - bsize = nbytes; - - crypto_inc(walk->iv, AES_BLOCK_SIZE); - aes_encrypt(ctx, buf, walk->iv); - aes_encrypt(ctx, mac, mac); - if (enc) - crypto_xor(mac, src, bsize); - crypto_xor_cpy(dst, src, buf, bsize); - if (!enc) - crypto_xor(mac, dst, bsize); - dst += bsize; - src += bsize; - nbytes -= bsize; - } while (--blocks); - - err = skcipher_walk_done(walk, tail); - } - - if (!err) { - aes_encrypt(ctx, buf, iv0); - aes_encrypt(ctx, mac, mac); - crypto_xor(mac, buf, AES_BLOCK_SIZE); - } - return err; -} - static int ccm_encrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -233,42 +157,42 @@ static int ccm_encrypt(struct aead_request *req) if (err) return err; - if (req->assoclen) - ccm_calculate_auth_mac(req, mac); - /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); err = skcipher_walk_aead_encrypt(&walk, req, false); - - if (crypto_simd_usable()) { - while (walk.nbytes) { - u32 tail = walk.nbytes % AES_BLOCK_SIZE; - - if (walk.nbytes == walk.total) - tail = 0; - - kernel_neon_begin(); - ce_aes_ccm_encrypt(walk.dst.virt.addr, - walk.src.virt.addr, - walk.nbytes - tail, ctx->key_enc, - num_rounds(ctx), mac, walk.iv); - kernel_neon_end(); - - err = skcipher_walk_done(&walk, tail); - } - if (!err) { - kernel_neon_begin(); - ce_aes_ccm_final(mac, buf, ctx->key_enc, - num_rounds(ctx)); - kernel_neon_end(); - } - } else { - err = ccm_crypt_fallback(&walk, mac, buf, ctx, true); - } - if (err) + if (unlikely(err)) return err; + kernel_neon_begin(); + + if (req->assoclen) + ccm_calculate_auth_mac(req, mac); + + do { + u32 tail = walk.nbytes % AES_BLOCK_SIZE; + + if (walk.nbytes == walk.total) + tail = 0; + + ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes - tail, ctx->key_enc, + num_rounds(ctx), mac, walk.iv); + + if (walk.nbytes == walk.total) + ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); + + kernel_neon_end(); + + if (walk.nbytes) { + err = skcipher_walk_done(&walk, tail); + if (unlikely(err)) + return err; + if (unlikely(walk.nbytes)) + kernel_neon_begin(); + } + } while (walk.nbytes); + /* copy authtag to end of dst */ scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen, crypto_aead_authsize(aead), 1); @@ -291,43 +215,42 @@ static int ccm_decrypt(struct aead_request *req) if (err) return err; - if (req->assoclen) - ccm_calculate_auth_mac(req, mac); - /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); err = skcipher_walk_aead_decrypt(&walk, req, false); - - if (crypto_simd_usable()) { - while (walk.nbytes) { - u32 tail = walk.nbytes % AES_BLOCK_SIZE; - - if (walk.nbytes == walk.total) - tail = 0; - - kernel_neon_begin(); - ce_aes_ccm_decrypt(walk.dst.virt.addr, - walk.src.virt.addr, - walk.nbytes - tail, ctx->key_enc, - num_rounds(ctx), mac, walk.iv); - kernel_neon_end(); - - err = skcipher_walk_done(&walk, tail); - } - if (!err) { - kernel_neon_begin(); - ce_aes_ccm_final(mac, buf, ctx->key_enc, - num_rounds(ctx)); - kernel_neon_end(); - } - } else { - err = ccm_crypt_fallback(&walk, mac, buf, ctx, false); - } - - if (err) + if (unlikely(err)) return err; + kernel_neon_begin(); + + if (req->assoclen) + ccm_calculate_auth_mac(req, mac); + + do { + u32 tail = walk.nbytes % AES_BLOCK_SIZE; + + if (walk.nbytes == walk.total) + tail = 0; + + ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes - tail, ctx->key_enc, + num_rounds(ctx), mac, walk.iv); + + if (walk.nbytes == walk.total) + ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); + + kernel_neon_end(); + + if (walk.nbytes) { + err = skcipher_walk_done(&walk, tail); + if (unlikely(err)) + return err; + if (unlikely(walk.nbytes)) + kernel_neon_begin(); + } + } while (walk.nbytes); + /* compare calculated auth tag with the stored one */ scatterwalk_map_and_copy(buf, req->src, req->assoclen + req->cryptlen - authsize, diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 445082aff0..ff91c6b11e 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -444,7 +444,7 @@ static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req) return err ?: cbc_decrypt_walk(req, &walk); } -static int ctr_encrypt(struct skcipher_request *req) +static int __maybe_unused ctr_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -485,29 +485,6 @@ static int ctr_encrypt(struct skcipher_request *req) return err; } -static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) -{ - const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); - unsigned long flags; - - /* - * Temporarily disable interrupts to avoid races where - * cachelines are evicted when the CPU is interrupted - * to do something else. - */ - local_irq_save(flags); - aes_encrypt(ctx, dst, src); - local_irq_restore(flags); -} - -static int __maybe_unused ctr_encrypt_sync(struct skcipher_request *req) -{ - if (!crypto_simd_usable()) - return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); - - return ctr_encrypt(req); -} - static int __maybe_unused xts_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -656,10 +633,9 @@ static int __maybe_unused xts_decrypt(struct skcipher_request *req) static struct skcipher_alg aes_algs[] = { { #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) .base = { - .cra_name = "__ecb(aes)", - .cra_driver_name = "__ecb-aes-" MODE, + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_module = THIS_MODULE, @@ -671,10 +647,9 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = ecb_decrypt, }, { .base = { - .cra_name = "__cbc(aes)", - .cra_driver_name = "__cbc-aes-" MODE, + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_module = THIS_MODULE, @@ -687,10 +662,9 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__ctr(aes)", - .cra_driver_name = "__ctr-aes-" MODE, + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_module = THIS_MODULE, @@ -704,26 +678,9 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = ctr_encrypt, }, { .base = { - .cra_name = "ctr(aes)", - .cra_driver_name = "ctr-aes-" MODE, - .cra_priority = PRIO - 1, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct crypto_aes_ctx), - .cra_module = THIS_MODULE, - }, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .chunksize = AES_BLOCK_SIZE, - .setkey = skcipher_aes_setkey, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base = { - .cra_name = "__xts(aes)", - .cra_driver_name = "__xts-aes-" MODE, + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), .cra_module = THIS_MODULE, @@ -738,10 +695,9 @@ static struct skcipher_alg aes_algs[] = { { }, { #endif .base = { - .cra_name = "__cts(cbc(aes))", - .cra_driver_name = "__cts-cbc-aes-" MODE, + .cra_name = "cts(cbc(aes))", + .cra_driver_name = "cts-cbc-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_module = THIS_MODULE, @@ -755,10 +711,9 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = cts_cbc_decrypt, }, { .base = { - .cra_name = "__essiv(cbc(aes),sha256)", - .cra_driver_name = "__essiv-cbc-aes-sha256-" MODE, + .cra_name = "essiv(cbc(aes),sha256)", + .cra_driver_name = "essiv-cbc-aes-sha256-" MODE, .cra_priority = PRIO + 1, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_essiv_cbc_ctx), .cra_module = THIS_MODULE, @@ -997,28 +952,15 @@ static struct shash_alg mac_algs[] = { { .descsize = sizeof(struct mac_desc_ctx), } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; - static void aes_exit(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) - if (aes_simd_algs[i]) - simd_skcipher_free(aes_simd_algs[i]); - crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - struct simd_skcipher_alg *simd; - const char *basename; - const char *algname; - const char *drvname; int err; - int i; err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); if (err) @@ -1028,26 +970,8 @@ static int __init aes_init(void) if (err) goto unregister_ciphers; - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) - continue; - - algname = aes_algs[i].base.cra_name + 2; - drvname = aes_algs[i].base.cra_driver_name + 2; - basename = aes_algs[i].base.cra_driver_name; - simd = simd_skcipher_create_compat(algname, drvname, basename); - err = PTR_ERR(simd); - if (IS_ERR(simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } - return 0; -unregister_simds: - aes_exit(); - return err; unregister_ciphers: crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); return err; diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index b495de22bb..ff01f0167b 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -363,15 +363,15 @@ ST5( mov v4.16b, vctr.16b ) adr x16, 1f sub x16, x16, x12, lsl #3 br x16 - hint 34 // bti c + bti c mov v0.d[0], vctr.d[0] - hint 34 // bti c + bti c mov v1.d[0], vctr.d[0] - hint 34 // bti c + bti c mov v2.d[0], vctr.d[0] - hint 34 // bti c + bti c mov v3.d[0], vctr.d[0] -ST5( hint 34 ) +ST5( bti c ) ST5( mov v4.d[0], vctr.d[0] ) 1: b 2f .previous diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c index cc52829d42..23c209ec51 100644 --- a/arch/arm64/crypto/aes-neonbs-glue.c +++ b/arch/arm64/crypto/aes-neonbs-glue.c @@ -58,11 +58,6 @@ struct aesbs_cbc_ctx { u32 enc[AES_MAX_KEYLENGTH_U32]; }; -struct aesbs_ctr_ctx { - struct aesbs_ctx key; /* must be first member */ - struct crypto_aes_ctx fallback; -}; - struct aesbs_xts_ctx { struct aesbs_ctx key; u32 twkey[AES_MAX_KEYLENGTH_U32]; @@ -202,25 +197,6 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = aes_expandkey(&ctx->fallback, in_key, key_len); - if (err) - return err; - - ctx->key.rounds = 6 + key_len / 4; - - kernel_neon_begin(); - aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds); - kernel_neon_end(); - - return 0; -} - static int ctr_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -287,29 +263,6 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, return aesbs_setkey(tfm, in_key, key_len); } -static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) -{ - struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - unsigned long flags; - - /* - * Temporarily disable interrupts to avoid races where - * cachelines are evicted when the CPU is interrupted - * to do something else. - */ - local_irq_save(flags); - aes_encrypt(&ctx->fallback, dst, src); - local_irq_restore(flags); -} - -static int ctr_encrypt_sync(struct skcipher_request *req) -{ - if (!crypto_simd_usable()) - return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); - - return ctr_encrypt(req); -} - static int __xts_crypt(struct skcipher_request *req, bool encrypt, void (*fn)(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[])) @@ -426,13 +379,12 @@ static int xts_decrypt(struct skcipher_request *req) } static struct skcipher_alg aes_algs[] = { { - .base.cra_name = "__ecb(aes)", - .base.cra_driver_name = "__ecb-aes-neonbs", + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -441,13 +393,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(aes)", - .base.cra_driver_name = "__cbc-aes-neonbs", + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -457,13 +408,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, { - .base.cra_name = "__ctr(aes)", - .base.cra_driver_name = "__ctr-aes-neonbs", + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aesbs_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -474,29 +424,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-neonbs", - .base.cra_priority = 250 - 1, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct aesbs_ctr_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .chunksize = AES_BLOCK_SIZE, - .walksize = 8 * AES_BLOCK_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = aesbs_ctr_setkey_sync, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base.cra_name = "__xts(aes)", - .base.cra_driver_name = "__xts-aes-neonbs", + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -507,54 +440,17 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = xts_decrypt, } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; - static void aes_exit(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) - if (aes_simd_algs[i]) - simd_skcipher_free(aes_simd_algs[i]); - crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - struct simd_skcipher_alg *simd; - const char *basename; - const char *algname; - const char *drvname; - int err; - int i; - if (!cpu_have_named_feature(ASIMD)) return -ENODEV; - err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - if (err) - return err; - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) - continue; - - algname = aes_algs[i].base.cra_name + 2; - drvname = aes_algs[i].base.cra_driver_name + 2; - basename = aes_algs[i].base.cra_driver_name; - simd = simd_skcipher_create_compat(algname, drvname, basename); - err = PTR_ERR(simd); - if (IS_ERR(simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } - return 0; - -unregister_simds: - aes_exit(); - return err; + return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } module_init(aes_init); diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 720cd3a58d..15794fe21a 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -362,84 +362,36 @@ static int gcm_encrypt(struct aead_request *req) err = skcipher_walk_aead_encrypt(&walk, req, false); - if (likely(crypto_simd_usable())) { - do { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - int nbytes = walk.nbytes; - - tag = (u8 *)&lengths; - - if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { - src = dst = memcpy(buf + sizeof(buf) - nbytes, - src, nbytes); - } else if (nbytes < walk.total) { - nbytes &= ~(AES_BLOCK_SIZE - 1); - tag = NULL; - } - - kernel_neon_begin(); - pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h, - dg, iv, ctx->aes_key.key_enc, nrounds, - tag); - kernel_neon_end(); - - if (unlikely(!nbytes)) - break; - - if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) - memcpy(walk.dst.virt.addr, - buf + sizeof(buf) - nbytes, nbytes); - - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } while (walk.nbytes); - } else { - while (walk.nbytes >= AES_BLOCK_SIZE) { - int blocks = walk.nbytes / AES_BLOCK_SIZE; - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - int remaining = blocks; - - do { - aes_encrypt(&ctx->aes_key, buf, iv); - crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE); - crypto_inc(iv, AES_BLOCK_SIZE); - - dst += AES_BLOCK_SIZE; - src += AES_BLOCK_SIZE; - } while (--remaining > 0); - - ghash_do_update(blocks, dg, walk.dst.virt.addr, - &ctx->ghash_key, NULL); - - err = skcipher_walk_done(&walk, - walk.nbytes % AES_BLOCK_SIZE); - } - - /* handle the tail */ - if (walk.nbytes) { - aes_encrypt(&ctx->aes_key, buf, iv); - - crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, - buf, walk.nbytes); - - memcpy(buf, walk.dst.virt.addr, walk.nbytes); - memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes); - } + do { + const u8 *src = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + int nbytes = walk.nbytes; tag = (u8 *)&lengths; - ghash_do_update(1, dg, tag, &ctx->ghash_key, - walk.nbytes ? buf : NULL); - if (walk.nbytes) - err = skcipher_walk_done(&walk, 0); + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { + src = dst = memcpy(buf + sizeof(buf) - nbytes, + src, nbytes); + } else if (nbytes < walk.total) { + nbytes &= ~(AES_BLOCK_SIZE - 1); + tag = NULL; + } - put_unaligned_be64(dg[1], tag); - put_unaligned_be64(dg[0], tag + 8); - put_unaligned_be32(1, iv + GCM_IV_SIZE); - aes_encrypt(&ctx->aes_key, iv, iv); - crypto_xor(tag, iv, AES_BLOCK_SIZE); - } + kernel_neon_begin(); + pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h, + dg, iv, ctx->aes_key.key_enc, nrounds, + tag); + kernel_neon_end(); + + if (unlikely(!nbytes)) + break; + + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) + memcpy(walk.dst.virt.addr, + buf + sizeof(buf) - nbytes, nbytes); + + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + } while (walk.nbytes); if (err) return err; @@ -464,6 +416,7 @@ static int gcm_decrypt(struct aead_request *req) u64 dg[2] = {}; be128 lengths; u8 *tag; + int ret; int err; lengths.a = cpu_to_be64(req->assoclen * 8); @@ -481,101 +434,41 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_aead_decrypt(&walk, req, false); - if (likely(crypto_simd_usable())) { - int ret; - - do { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - int nbytes = walk.nbytes; - - tag = (u8 *)&lengths; - - if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { - src = dst = memcpy(buf + sizeof(buf) - nbytes, - src, nbytes); - } else if (nbytes < walk.total) { - nbytes &= ~(AES_BLOCK_SIZE - 1); - tag = NULL; - } - - kernel_neon_begin(); - ret = pmull_gcm_decrypt(nbytes, dst, src, - ctx->ghash_key.h, - dg, iv, ctx->aes_key.key_enc, - nrounds, tag, otag, authsize); - kernel_neon_end(); - - if (unlikely(!nbytes)) - break; - - if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) - memcpy(walk.dst.virt.addr, - buf + sizeof(buf) - nbytes, nbytes); - - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } while (walk.nbytes); - - if (err) - return err; - if (ret) - return -EBADMSG; - } else { - while (walk.nbytes >= AES_BLOCK_SIZE) { - int blocks = walk.nbytes / AES_BLOCK_SIZE; - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - - ghash_do_update(blocks, dg, walk.src.virt.addr, - &ctx->ghash_key, NULL); - - do { - aes_encrypt(&ctx->aes_key, buf, iv); - crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE); - crypto_inc(iv, AES_BLOCK_SIZE); - - dst += AES_BLOCK_SIZE; - src += AES_BLOCK_SIZE; - } while (--blocks > 0); - - err = skcipher_walk_done(&walk, - walk.nbytes % AES_BLOCK_SIZE); - } - - /* handle the tail */ - if (walk.nbytes) { - memcpy(buf, walk.src.virt.addr, walk.nbytes); - memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes); - } + do { + const u8 *src = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + int nbytes = walk.nbytes; tag = (u8 *)&lengths; - ghash_do_update(1, dg, tag, &ctx->ghash_key, - walk.nbytes ? buf : NULL); - if (walk.nbytes) { - aes_encrypt(&ctx->aes_key, buf, iv); - - crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, - buf, walk.nbytes); - - err = skcipher_walk_done(&walk, 0); + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { + src = dst = memcpy(buf + sizeof(buf) - nbytes, + src, nbytes); + } else if (nbytes < walk.total) { + nbytes &= ~(AES_BLOCK_SIZE - 1); + tag = NULL; } - if (err) - return err; + kernel_neon_begin(); + ret = pmull_gcm_decrypt(nbytes, dst, src, ctx->ghash_key.h, + dg, iv, ctx->aes_key.key_enc, + nrounds, tag, otag, authsize); + kernel_neon_end(); - put_unaligned_be64(dg[1], tag); - put_unaligned_be64(dg[0], tag + 8); - put_unaligned_be32(1, iv + GCM_IV_SIZE); - aes_encrypt(&ctx->aes_key, iv, iv); - crypto_xor(tag, iv, AES_BLOCK_SIZE); + if (unlikely(!nbytes)) + break; - if (crypto_memneq(tag, otag, authsize)) { - memzero_explicit(tag, AES_BLOCK_SIZE); - return -EBADMSG; - } - } - return 0; + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) + memcpy(walk.dst.virt.addr, + buf + sizeof(buf) - nbytes, nbytes); + + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + } while (walk.nbytes); + + if (err) + return err; + + return ret ? -EBADMSG : 0; } static struct aead_alg gcm_aes_alg = { diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index 88d20f04c6..af1fafbe7e 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -32,7 +32,7 @@ ({ \ const struct arch_timer_erratum_workaround *__wa; \ __wa = __this_cpu_read(timer_unstable_counter_workaround); \ - (__wa && __wa->h) ? __wa->h : arch_timer_##h; \ + (__wa && __wa->h) ? ({ isb(); __wa->h;}) : arch_timer_##h; \ }) #else @@ -52,8 +52,6 @@ struct arch_timer_erratum_workaround { enum arch_timer_erratum_match_type match_type; const void *id; const char *desc; - u32 (*read_cntp_tval_el0)(void); - u32 (*read_cntv_tval_el0)(void); u64 (*read_cntpct_el0)(void); u64 (*read_cntvct_el0)(void); int (*set_next_event_phys)(unsigned long, struct clock_event_device *); @@ -64,25 +62,28 @@ struct arch_timer_erratum_workaround { DECLARE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround); -/* inline sysreg accessors that make erratum_handler() work */ -static inline notrace u32 arch_timer_read_cntp_tval_el0(void) -{ - return read_sysreg(cntp_tval_el0); -} - -static inline notrace u32 arch_timer_read_cntv_tval_el0(void) -{ - return read_sysreg(cntv_tval_el0); -} - static inline notrace u64 arch_timer_read_cntpct_el0(void) { - return read_sysreg(cntpct_el0); + u64 cnt; + + asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0", + "nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0), + ARM64_HAS_ECV) + : "=r" (cnt)); + + return cnt; } static inline notrace u64 arch_timer_read_cntvct_el0(void) { - return read_sysreg(cntvct_el0); + u64 cnt; + + asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0", + "nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0), + ARM64_HAS_ECV) + : "=r" (cnt)); + + return cnt; } #define arch_timer_reg_read_stable(reg) \ @@ -102,51 +103,58 @@ static inline notrace u64 arch_timer_read_cntvct_el0(void) * the code. */ static __always_inline -void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u32 val) +void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u64 val) { if (access == ARCH_TIMER_PHYS_ACCESS) { switch (reg) { case ARCH_TIMER_REG_CTRL: write_sysreg(val, cntp_ctl_el0); + isb(); break; - case ARCH_TIMER_REG_TVAL: - write_sysreg(val, cntp_tval_el0); + case ARCH_TIMER_REG_CVAL: + write_sysreg(val, cntp_cval_el0); break; + default: + BUILD_BUG(); } } else if (access == ARCH_TIMER_VIRT_ACCESS) { switch (reg) { case ARCH_TIMER_REG_CTRL: write_sysreg(val, cntv_ctl_el0); + isb(); break; - case ARCH_TIMER_REG_TVAL: - write_sysreg(val, cntv_tval_el0); + case ARCH_TIMER_REG_CVAL: + write_sysreg(val, cntv_cval_el0); break; + default: + BUILD_BUG(); } + } else { + BUILD_BUG(); } - - isb(); } static __always_inline -u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg) +u64 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg) { if (access == ARCH_TIMER_PHYS_ACCESS) { switch (reg) { case ARCH_TIMER_REG_CTRL: return read_sysreg(cntp_ctl_el0); - case ARCH_TIMER_REG_TVAL: - return arch_timer_reg_read_stable(cntp_tval_el0); + default: + BUILD_BUG(); } } else if (access == ARCH_TIMER_VIRT_ACCESS) { switch (reg) { case ARCH_TIMER_REG_CTRL: return read_sysreg(cntv_ctl_el0); - case ARCH_TIMER_REG_TVAL: - return arch_timer_reg_read_stable(cntv_tval_el0); + default: + BUILD_BUG(); } } - BUG(); + BUILD_BUG(); + unreachable(); } static inline u32 arch_timer_get_cntfrq(void) @@ -169,7 +177,6 @@ static __always_inline u64 __arch_counter_get_cntpct_stable(void) { u64 cnt; - isb(); cnt = arch_timer_reg_read_stable(cntpct_el0); arch_counter_enforce_ordering(cnt); return cnt; @@ -179,8 +186,10 @@ static __always_inline u64 __arch_counter_get_cntpct(void) { u64 cnt; - isb(); - cnt = read_sysreg(cntpct_el0); + asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0", + "nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0), + ARM64_HAS_ECV) + : "=r" (cnt)); arch_counter_enforce_ordering(cnt); return cnt; } @@ -189,7 +198,6 @@ static __always_inline u64 __arch_counter_get_cntvct_stable(void) { u64 cnt; - isb(); cnt = arch_timer_reg_read_stable(cntvct_el0); arch_counter_enforce_ordering(cnt); return cnt; @@ -199,8 +207,10 @@ static __always_inline u64 __arch_counter_get_cntvct(void) { u64 cnt; - isb(); - cnt = read_sysreg(cntvct_el0); + asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0", + "nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0), + ARM64_HAS_ECV) + : "=r" (cnt)); arch_counter_enforce_ordering(cnt); return cnt; } diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h index ccedf548da..0557af834e 100644 --- a/arch/arm64/include/asm/asm-uaccess.h +++ b/arch/arm64/include/asm/asm-uaccess.h @@ -3,10 +3,11 @@ #define __ASM_ASM_UACCESS_H #include +#include +#include #include #include #include -#include /* * User access enabling/disabling macros. @@ -58,6 +59,10 @@ alternative_else_nop_endif .endm #endif +#define USER(l, x...) \ +9999: x; \ + _asm_extable 9999b, l + /* * Generate the assembly for LDTR/STTR with exception table entries. * This is complicated as there is no post-increment or pair versions of the diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 448a575db8..6ebdc0f834 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -14,9 +14,10 @@ #include -#include #include #include +#include +#include #include #include #include @@ -136,32 +137,6 @@ alternative_endif .endr .endm -/* - * Create an exception table entry for `insn`, which will branch to `fixup` - * when an unhandled fault is taken. - */ - .macro _asm_extable, insn, fixup - .pushsection __ex_table, "a" - .align 3 - .long (\insn - .), (\fixup - .) - .popsection - .endm - -/* - * Create an exception table entry for `insn` if `fixup` is provided. Otherwise - * do nothing. - */ - .macro _cond_extable, insn, fixup - .ifnc \fixup, - _asm_extable \insn, \fixup - .endif - .endm - - -#define USER(l, x...) \ -9999: x; \ - _asm_extable 9999b, l - /* * Register aliases. */ @@ -412,19 +387,19 @@ alternative_endif /* * Macro to perform a data cache maintenance for the interval - * [start, end) + * [start, end) with dcache line size explicitly provided. * * op: operation passed to dc instruction * domain: domain used in dsb instruciton * start: starting virtual address of the region * end: end virtual address of the region + * linesz: dcache line size * fixup: optional label to branch to on user fault - * Corrupts: start, end, tmp1, tmp2 + * Corrupts: start, end, tmp */ - .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup - dcache_line_size \tmp1, \tmp2 - sub \tmp2, \tmp1, #1 - bic \start, \start, \tmp2 + .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup + sub \tmp, \linesz, #1 + bic \start, \start, \tmp .Ldcache_op\@: .ifc \op, cvau __dcache_op_workaround_clean_cache \op, \start @@ -443,7 +418,7 @@ alternative_endif .endif .endif .endif - add \start, \start, \tmp1 + add \start, \start, \linesz cmp \start, \end b.lo .Ldcache_op\@ dsb \domain @@ -451,6 +426,22 @@ alternative_endif _cond_extable .Ldcache_op\@, \fixup .endm +/* + * Macro to perform a data cache maintenance for the interval + * [start, end) + * + * op: operation passed to dc instruction + * domain: domain used in dsb instruciton + * start: starting virtual address of the region + * end: end virtual address of the region + * fixup: optional label to branch to on user fault + * Corrupts: start, end, tmp1, tmp2 + */ + .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup + dcache_line_size \tmp1, \tmp2 + dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup + .endm + /* * Macro to perform an instruction cache maintenance for the interval * [start, end) @@ -474,6 +465,25 @@ alternative_endif _cond_extable .Licache_op\@, \fixup .endm +/* + * To prevent the possibility of old and new partial table walks being visible + * in the tlb, switch the ttbr to a zero page when we invalidate the old + * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i + * Even switching to our copied tables will cause a changed output address at + * each stage of the walk. + */ + .macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2 + phys_to_ttbr \tmp, \zero_page + msr ttbr1_el1, \tmp + isb + tlbi vmalle1 + dsb nsh + phys_to_ttbr \tmp, \page_table + offset_ttbr1 \tmp, \tmp2 + msr ttbr1_el1, \tmp + isb + .endm + /* * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present */ @@ -787,6 +797,16 @@ alternative_endif .Lnoyield_\@: .endm +/* + * Branch Target Identifier (BTI) + */ + .macro bti, targets + .equ .L__bti_targets_c, 34 + .equ .L__bti_targets_j, 36 + .equ .L__bti_targets_jc,38 + hint #.L__bti_targets_\targets + .endm + /* * This macro emits a program property note section identifying * architecture features which require special handling, mainly for diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h index 13869b76b5..fe0db8d416 100644 --- a/arch/arm64/include/asm/atomic_ll_sc.h +++ b/arch/arm64/include/asm/atomic_ll_sc.h @@ -44,11 +44,11 @@ __ll_sc_atomic_##op(int i, atomic_t *v) \ \ asm volatile("// atomic_" #op "\n" \ __LL_SC_FALLBACK( \ -" prfm pstl1strm, %2\n" \ -"1: ldxr %w0, %2\n" \ -" " #asm_op " %w0, %w0, %w3\n" \ -" stxr %w1, %w0, %2\n" \ -" cbnz %w1, 1b\n") \ + " prfm pstl1strm, %2\n" \ + "1: ldxr %w0, %2\n" \ + " " #asm_op " %w0, %w0, %w3\n" \ + " stxr %w1, %w0, %2\n" \ + " cbnz %w1, 1b\n") \ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i)); \ } @@ -62,12 +62,12 @@ __ll_sc_atomic_##op##_return##name(int i, atomic_t *v) \ \ asm volatile("// atomic_" #op "_return" #name "\n" \ __LL_SC_FALLBACK( \ -" prfm pstl1strm, %2\n" \ -"1: ld" #acq "xr %w0, %2\n" \ -" " #asm_op " %w0, %w0, %w3\n" \ -" st" #rel "xr %w1, %w0, %2\n" \ -" cbnz %w1, 1b\n" \ -" " #mb ) \ + " prfm pstl1strm, %2\n" \ + "1: ld" #acq "xr %w0, %2\n" \ + " " #asm_op " %w0, %w0, %w3\n" \ + " st" #rel "xr %w1, %w0, %2\n" \ + " cbnz %w1, 1b\n" \ + " " #mb ) \ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i) \ : cl); \ @@ -84,12 +84,12 @@ __ll_sc_atomic_fetch_##op##name(int i, atomic_t *v) \ \ asm volatile("// atomic_fetch_" #op #name "\n" \ __LL_SC_FALLBACK( \ -" prfm pstl1strm, %3\n" \ -"1: ld" #acq "xr %w0, %3\n" \ -" " #asm_op " %w1, %w0, %w4\n" \ -" st" #rel "xr %w2, %w1, %3\n" \ -" cbnz %w2, 1b\n" \ -" " #mb ) \ + " prfm pstl1strm, %3\n" \ + "1: ld" #acq "xr %w0, %3\n" \ + " " #asm_op " %w1, %w0, %w4\n" \ + " st" #rel "xr %w2, %w1, %3\n" \ + " cbnz %w2, 1b\n" \ + " " #mb ) \ : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i) \ : cl); \ @@ -143,11 +143,11 @@ __ll_sc_atomic64_##op(s64 i, atomic64_t *v) \ \ asm volatile("// atomic64_" #op "\n" \ __LL_SC_FALLBACK( \ -" prfm pstl1strm, %2\n" \ -"1: ldxr %0, %2\n" \ -" " #asm_op " %0, %0, %3\n" \ -" stxr %w1, %0, %2\n" \ -" cbnz %w1, 1b") \ + " prfm pstl1strm, %2\n" \ + "1: ldxr %0, %2\n" \ + " " #asm_op " %0, %0, %3\n" \ + " stxr %w1, %0, %2\n" \ + " cbnz %w1, 1b") \ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i)); \ } @@ -161,12 +161,12 @@ __ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v) \ \ asm volatile("// atomic64_" #op "_return" #name "\n" \ __LL_SC_FALLBACK( \ -" prfm pstl1strm, %2\n" \ -"1: ld" #acq "xr %0, %2\n" \ -" " #asm_op " %0, %0, %3\n" \ -" st" #rel "xr %w1, %0, %2\n" \ -" cbnz %w1, 1b\n" \ -" " #mb ) \ + " prfm pstl1strm, %2\n" \ + "1: ld" #acq "xr %0, %2\n" \ + " " #asm_op " %0, %0, %3\n" \ + " st" #rel "xr %w1, %0, %2\n" \ + " cbnz %w1, 1b\n" \ + " " #mb ) \ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i) \ : cl); \ @@ -176,19 +176,19 @@ __ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v) \ #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\ static inline long \ -__ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v) \ +__ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v) \ { \ s64 result, val; \ unsigned long tmp; \ \ asm volatile("// atomic64_fetch_" #op #name "\n" \ __LL_SC_FALLBACK( \ -" prfm pstl1strm, %3\n" \ -"1: ld" #acq "xr %0, %3\n" \ -" " #asm_op " %1, %0, %4\n" \ -" st" #rel "xr %w2, %1, %3\n" \ -" cbnz %w2, 1b\n" \ -" " #mb ) \ + " prfm pstl1strm, %3\n" \ + "1: ld" #acq "xr %0, %3\n" \ + " " #asm_op " %1, %0, %4\n" \ + " st" #rel "xr %w2, %1, %3\n" \ + " cbnz %w2, 1b\n" \ + " " #mb ) \ : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i) \ : cl); \ @@ -241,14 +241,14 @@ __ll_sc_atomic64_dec_if_positive(atomic64_t *v) asm volatile("// atomic64_dec_if_positive\n" __LL_SC_FALLBACK( -" prfm pstl1strm, %2\n" -"1: ldxr %0, %2\n" -" subs %0, %0, #1\n" -" b.lt 2f\n" -" stlxr %w1, %0, %2\n" -" cbnz %w1, 1b\n" -" dmb ish\n" -"2:") + " prfm pstl1strm, %2\n" + "1: ldxr %0, %2\n" + " subs %0, %0, #1\n" + " b.lt 2f\n" + " stlxr %w1, %0, %2\n" + " cbnz %w1, 1b\n" + " dmb ish\n" + "2:") : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) : : "cc", "memory"); diff --git a/arch/arm64/include/asm/atomic_lse.h b/arch/arm64/include/asm/atomic_lse.h index da3280f639..5d460f6b76 100644 --- a/arch/arm64/include/asm/atomic_lse.h +++ b/arch/arm64/include/asm/atomic_lse.h @@ -11,13 +11,13 @@ #define __ASM_ATOMIC_LSE_H #define ATOMIC_OP(op, asm_op) \ -static inline void __lse_atomic_##op(int i, atomic_t *v) \ +static inline void __lse_atomic_##op(int i, atomic_t *v) \ { \ asm volatile( \ __LSE_PREAMBLE \ -" " #asm_op " %w[i], %[v]\n" \ - : [i] "+r" (i), [v] "+Q" (v->counter) \ - : "r" (v)); \ + " " #asm_op " %w[i], %[v]\n" \ + : [v] "+Q" (v->counter) \ + : [i] "r" (i)); \ } ATOMIC_OP(andnot, stclr) @@ -25,19 +25,27 @@ ATOMIC_OP(or, stset) ATOMIC_OP(xor, steor) ATOMIC_OP(add, stadd) +static inline void __lse_atomic_sub(int i, atomic_t *v) +{ + __lse_atomic_add(-i, v); +} + #undef ATOMIC_OP #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \ static inline int __lse_atomic_fetch_##op##name(int i, atomic_t *v) \ { \ + int old; \ + \ asm volatile( \ __LSE_PREAMBLE \ -" " #asm_op #mb " %w[i], %w[i], %[v]" \ - : [i] "+r" (i), [v] "+Q" (v->counter) \ - : "r" (v) \ + " " #asm_op #mb " %w[i], %w[old], %[v]" \ + : [v] "+Q" (v->counter), \ + [old] "=r" (old) \ + : [i] "r" (i) \ : cl); \ \ - return i; \ + return old; \ } #define ATOMIC_FETCH_OPS(op, asm_op) \ @@ -54,51 +62,46 @@ ATOMIC_FETCH_OPS(add, ldadd) #undef ATOMIC_FETCH_OP #undef ATOMIC_FETCH_OPS -#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \ -static inline int __lse_atomic_add_return##name(int i, atomic_t *v) \ +#define ATOMIC_FETCH_OP_SUB(name) \ +static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v) \ { \ - u32 tmp; \ - \ - asm volatile( \ - __LSE_PREAMBLE \ - " ldadd" #mb " %w[i], %w[tmp], %[v]\n" \ - " add %w[i], %w[i], %w[tmp]" \ - : [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ - : "r" (v) \ - : cl); \ - \ - return i; \ + return __lse_atomic_fetch_add##name(-i, v); \ } -ATOMIC_OP_ADD_RETURN(_relaxed, ) -ATOMIC_OP_ADD_RETURN(_acquire, a, "memory") -ATOMIC_OP_ADD_RETURN(_release, l, "memory") -ATOMIC_OP_ADD_RETURN( , al, "memory") +ATOMIC_FETCH_OP_SUB(_relaxed) +ATOMIC_FETCH_OP_SUB(_acquire) +ATOMIC_FETCH_OP_SUB(_release) +ATOMIC_FETCH_OP_SUB( ) -#undef ATOMIC_OP_ADD_RETURN +#undef ATOMIC_FETCH_OP_SUB + +#define ATOMIC_OP_ADD_SUB_RETURN(name) \ +static inline int __lse_atomic_add_return##name(int i, atomic_t *v) \ +{ \ + return __lse_atomic_fetch_add##name(i, v) + i; \ +} \ + \ +static inline int __lse_atomic_sub_return##name(int i, atomic_t *v) \ +{ \ + return __lse_atomic_fetch_sub(i, v) - i; \ +} + +ATOMIC_OP_ADD_SUB_RETURN(_relaxed) +ATOMIC_OP_ADD_SUB_RETURN(_acquire) +ATOMIC_OP_ADD_SUB_RETURN(_release) +ATOMIC_OP_ADD_SUB_RETURN( ) + +#undef ATOMIC_OP_ADD_SUB_RETURN static inline void __lse_atomic_and(int i, atomic_t *v) { - asm volatile( - __LSE_PREAMBLE - " mvn %w[i], %w[i]\n" - " stclr %w[i], %[v]" - : [i] "+&r" (i), [v] "+Q" (v->counter) - : "r" (v)); + return __lse_atomic_andnot(~i, v); } #define ATOMIC_FETCH_OP_AND(name, mb, cl...) \ static inline int __lse_atomic_fetch_and##name(int i, atomic_t *v) \ { \ - asm volatile( \ - __LSE_PREAMBLE \ - " mvn %w[i], %w[i]\n" \ - " ldclr" #mb " %w[i], %w[i], %[v]" \ - : [i] "+&r" (i), [v] "+Q" (v->counter) \ - : "r" (v) \ - : cl); \ - \ - return i; \ + return __lse_atomic_fetch_andnot##name(~i, v); \ } ATOMIC_FETCH_OP_AND(_relaxed, ) @@ -108,69 +111,14 @@ ATOMIC_FETCH_OP_AND( , al, "memory") #undef ATOMIC_FETCH_OP_AND -static inline void __lse_atomic_sub(int i, atomic_t *v) -{ - asm volatile( - __LSE_PREAMBLE - " neg %w[i], %w[i]\n" - " stadd %w[i], %[v]" - : [i] "+&r" (i), [v] "+Q" (v->counter) - : "r" (v)); -} - -#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \ -static inline int __lse_atomic_sub_return##name(int i, atomic_t *v) \ -{ \ - u32 tmp; \ - \ - asm volatile( \ - __LSE_PREAMBLE \ - " neg %w[i], %w[i]\n" \ - " ldadd" #mb " %w[i], %w[tmp], %[v]\n" \ - " add %w[i], %w[i], %w[tmp]" \ - : [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ - : "r" (v) \ - : cl); \ - \ - return i; \ -} - -ATOMIC_OP_SUB_RETURN(_relaxed, ) -ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") -ATOMIC_OP_SUB_RETURN(_release, l, "memory") -ATOMIC_OP_SUB_RETURN( , al, "memory") - -#undef ATOMIC_OP_SUB_RETURN - -#define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \ -static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v) \ -{ \ - asm volatile( \ - __LSE_PREAMBLE \ - " neg %w[i], %w[i]\n" \ - " ldadd" #mb " %w[i], %w[i], %[v]" \ - : [i] "+&r" (i), [v] "+Q" (v->counter) \ - : "r" (v) \ - : cl); \ - \ - return i; \ -} - -ATOMIC_FETCH_OP_SUB(_relaxed, ) -ATOMIC_FETCH_OP_SUB(_acquire, a, "memory") -ATOMIC_FETCH_OP_SUB(_release, l, "memory") -ATOMIC_FETCH_OP_SUB( , al, "memory") - -#undef ATOMIC_FETCH_OP_SUB - #define ATOMIC64_OP(op, asm_op) \ static inline void __lse_atomic64_##op(s64 i, atomic64_t *v) \ { \ asm volatile( \ __LSE_PREAMBLE \ -" " #asm_op " %[i], %[v]\n" \ - : [i] "+r" (i), [v] "+Q" (v->counter) \ - : "r" (v)); \ + " " #asm_op " %[i], %[v]\n" \ + : [v] "+Q" (v->counter) \ + : [i] "r" (i)); \ } ATOMIC64_OP(andnot, stclr) @@ -178,19 +126,27 @@ ATOMIC64_OP(or, stset) ATOMIC64_OP(xor, steor) ATOMIC64_OP(add, stadd) +static inline void __lse_atomic64_sub(s64 i, atomic64_t *v) +{ + __lse_atomic64_add(-i, v); +} + #undef ATOMIC64_OP #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \ static inline long __lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)\ { \ + s64 old; \ + \ asm volatile( \ __LSE_PREAMBLE \ -" " #asm_op #mb " %[i], %[i], %[v]" \ - : [i] "+r" (i), [v] "+Q" (v->counter) \ - : "r" (v) \ + " " #asm_op #mb " %[i], %[old], %[v]" \ + : [v] "+Q" (v->counter), \ + [old] "=r" (old) \ + : [i] "r" (i) \ : cl); \ \ - return i; \ + return old; \ } #define ATOMIC64_FETCH_OPS(op, asm_op) \ @@ -207,51 +163,46 @@ ATOMIC64_FETCH_OPS(add, ldadd) #undef ATOMIC64_FETCH_OP #undef ATOMIC64_FETCH_OPS -#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \ -static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\ +#define ATOMIC64_FETCH_OP_SUB(name) \ +static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \ { \ - unsigned long tmp; \ - \ - asm volatile( \ - __LSE_PREAMBLE \ - " ldadd" #mb " %[i], %x[tmp], %[v]\n" \ - " add %[i], %[i], %x[tmp]" \ - : [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ - : "r" (v) \ - : cl); \ - \ - return i; \ + return __lse_atomic64_fetch_add##name(-i, v); \ } -ATOMIC64_OP_ADD_RETURN(_relaxed, ) -ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory") -ATOMIC64_OP_ADD_RETURN(_release, l, "memory") -ATOMIC64_OP_ADD_RETURN( , al, "memory") +ATOMIC64_FETCH_OP_SUB(_relaxed) +ATOMIC64_FETCH_OP_SUB(_acquire) +ATOMIC64_FETCH_OP_SUB(_release) +ATOMIC64_FETCH_OP_SUB( ) -#undef ATOMIC64_OP_ADD_RETURN +#undef ATOMIC64_FETCH_OP_SUB + +#define ATOMIC64_OP_ADD_SUB_RETURN(name) \ +static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\ +{ \ + return __lse_atomic64_fetch_add##name(i, v) + i; \ +} \ + \ +static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v)\ +{ \ + return __lse_atomic64_fetch_sub##name(i, v) - i; \ +} + +ATOMIC64_OP_ADD_SUB_RETURN(_relaxed) +ATOMIC64_OP_ADD_SUB_RETURN(_acquire) +ATOMIC64_OP_ADD_SUB_RETURN(_release) +ATOMIC64_OP_ADD_SUB_RETURN( ) + +#undef ATOMIC64_OP_ADD_SUB_RETURN static inline void __lse_atomic64_and(s64 i, atomic64_t *v) { - asm volatile( - __LSE_PREAMBLE - " mvn %[i], %[i]\n" - " stclr %[i], %[v]" - : [i] "+&r" (i), [v] "+Q" (v->counter) - : "r" (v)); + return __lse_atomic64_andnot(~i, v); } #define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \ static inline long __lse_atomic64_fetch_and##name(s64 i, atomic64_t *v) \ { \ - asm volatile( \ - __LSE_PREAMBLE \ - " mvn %[i], %[i]\n" \ - " ldclr" #mb " %[i], %[i], %[v]" \ - : [i] "+&r" (i), [v] "+Q" (v->counter) \ - : "r" (v) \ - : cl); \ - \ - return i; \ + return __lse_atomic64_fetch_andnot##name(~i, v); \ } ATOMIC64_FETCH_OP_AND(_relaxed, ) @@ -261,61 +212,6 @@ ATOMIC64_FETCH_OP_AND( , al, "memory") #undef ATOMIC64_FETCH_OP_AND -static inline void __lse_atomic64_sub(s64 i, atomic64_t *v) -{ - asm volatile( - __LSE_PREAMBLE - " neg %[i], %[i]\n" - " stadd %[i], %[v]" - : [i] "+&r" (i), [v] "+Q" (v->counter) - : "r" (v)); -} - -#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \ -static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v) \ -{ \ - unsigned long tmp; \ - \ - asm volatile( \ - __LSE_PREAMBLE \ - " neg %[i], %[i]\n" \ - " ldadd" #mb " %[i], %x[tmp], %[v]\n" \ - " add %[i], %[i], %x[tmp]" \ - : [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ - : "r" (v) \ - : cl); \ - \ - return i; \ -} - -ATOMIC64_OP_SUB_RETURN(_relaxed, ) -ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") -ATOMIC64_OP_SUB_RETURN(_release, l, "memory") -ATOMIC64_OP_SUB_RETURN( , al, "memory") - -#undef ATOMIC64_OP_SUB_RETURN - -#define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \ -static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \ -{ \ - asm volatile( \ - __LSE_PREAMBLE \ - " neg %[i], %[i]\n" \ - " ldadd" #mb " %[i], %[i], %[v]" \ - : [i] "+&r" (i), [v] "+Q" (v->counter) \ - : "r" (v) \ - : cl); \ - \ - return i; \ -} - -ATOMIC64_FETCH_OP_SUB(_relaxed, ) -ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory") -ATOMIC64_FETCH_OP_SUB(_release, l, "memory") -ATOMIC64_FETCH_OP_SUB( , al, "memory") - -#undef ATOMIC64_FETCH_OP_SUB - static inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v) { unsigned long tmp; @@ -353,7 +249,7 @@ __lse__cmpxchg_case_##name##sz(volatile void *ptr, \ " mov %" #w "[tmp], %" #w "[old]\n" \ " cas" #mb #sfx "\t%" #w "[tmp], %" #w "[new], %[v]\n" \ " mov %" #w "[ret], %" #w "[tmp]" \ - : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr), \ + : [ret] "+r" (x0), [v] "+Q" (*(u##sz *)ptr), \ [tmp] "=&r" (tmp) \ : [old] "r" (x1), [new] "r" (x2) \ : cl); \ diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 451e11e5fd..62217be362 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -23,9 +23,17 @@ #define dsb(opt) asm volatile("dsb " #opt : : : "memory") #define psb_csync() asm volatile("hint #17" : : : "memory") -#define tsb_csync() asm volatile("hint #18" : : : "memory") +#define __tsb_csync() asm volatile("hint #18" : : : "memory") #define csdb() asm volatile("hint #20" : : : "memory") +/* + * Data Gathering Hint: + * This instruction prevents merging memory accesses with Normal-NC or + * Device-GRE attributes before the hint instruction with any memory accesses + * appearing after the hint instruction. + */ +#define dgh() asm volatile("hint #6" : : : "memory") + #ifdef CONFIG_ARM64_PSEUDO_NMI #define pmr_sync() \ do { \ @@ -46,6 +54,21 @@ #define dma_rmb() dmb(oshld) #define dma_wmb() dmb(oshst) +#define io_stop_wc() dgh() + +#define tsb_csync() \ + do { \ + /* \ + * CPUs affected by Arm Erratum 2054223 or 2067961 needs \ + * another TSB to ensure the trace is flushed. The barriers \ + * don't have to be strictly back to back, as long as the \ + * CPU is in trace prohibited state. \ + */ \ + if (cpus_have_final_cap(ARM64_WORKAROUND_TSB_FLUSH_FAILURE)) \ + __tsb_csync(); \ + __tsb_csync(); \ + } while (0) + /* * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz * and 0 otherwise. diff --git a/arch/arm64/include/asm/bitops.h b/arch/arm64/include/asm/bitops.h index 81a3e519b0..9b3c787132 100644 --- a/arch/arm64/include/asm/bitops.h +++ b/arch/arm64/include/asm/bitops.h @@ -18,7 +18,6 @@ #include #include -#include #include #include diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index f9bef42c14..497acf134d 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -243,7 +243,7 @@ static inline void __cmpwait_case_##sz(volatile void *ptr, \ " cbnz %" #w "[tmp], 1f\n" \ " wfe\n" \ "1:" \ - : [tmp] "=&r" (tmp), [v] "+Q" (*(unsigned long *)ptr) \ + : [tmp] "=&r" (tmp), [v] "+Q" (*(u##sz *)ptr) \ : [val] "r" (val)); \ } diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 8f59bbeba7..d52a0b269e 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -228,6 +228,9 @@ #define ESR_ELx_SYS64_ISS_SYS_CNTVCT (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 2, 14, 0) | \ ESR_ELx_SYS64_ISS_DIR_READ) +#define ESR_ELx_SYS64_ISS_SYS_CNTVCTSS (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 6, 14, 0) | \ + ESR_ELx_SYS64_ISS_DIR_READ) + #define ESR_ELx_SYS64_ISS_SYS_CNTFRQ (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 0, 14, 0) | \ ESR_ELx_SYS64_ISS_DIR_READ) @@ -318,6 +321,9 @@ #define ESR_ELx_CP15_64_ISS_SYS_CNTVCT (ESR_ELx_CP15_64_ISS_SYS_VAL(1, 14) | \ ESR_ELx_CP15_64_ISS_DIR_READ) +#define ESR_ELx_CP15_64_ISS_SYS_CNTVCTSS (ESR_ELx_CP15_64_ISS_SYS_VAL(9, 14) | \ + ESR_ELx_CP15_64_ISS_DIR_READ) + #define ESR_ELx_CP15_32_ISS_SYS_CNTFRQ (ESR_ELx_CP15_32_ISS_SYS_VAL(0, 0, 14, 0) |\ ESR_ELx_CP15_32_ISS_DIR_READ) diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h index 840a35ed92..72b0e71cc3 100644 --- a/arch/arm64/include/asm/extable.h +++ b/arch/arm64/include/asm/extable.h @@ -18,21 +18,32 @@ struct exception_table_entry { int insn, fixup; + short type, data; }; #define ARCH_HAS_RELATIVE_EXTABLE +#define swap_ex_entry_fixup(a, b, tmp, delta) \ +do { \ + (a)->fixup = (b)->fixup + (delta); \ + (b)->fixup = (tmp).fixup - (delta); \ + (a)->type = (b)->type; \ + (b)->type = (tmp).type; \ + (a)->data = (b)->data; \ + (b)->data = (tmp).data; \ +} while (0) + #ifdef CONFIG_BPF_JIT -int arm64_bpf_fixup_exception(const struct exception_table_entry *ex, - struct pt_regs *regs); +bool ex_handler_bpf(const struct exception_table_entry *ex, + struct pt_regs *regs); #else /* !CONFIG_BPF_JIT */ static inline -int arm64_bpf_fixup_exception(const struct exception_table_entry *ex, - struct pt_regs *regs) +bool ex_handler_bpf(const struct exception_table_entry *ex, + struct pt_regs *regs) { - return 0; + return false; } #endif /* !CONFIG_BPF_JIT */ -extern int fixup_exception(struct pt_regs *regs); +bool fixup_exception(struct pt_regs *regs); #endif diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 9a62884183..cb24385e36 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -51,8 +51,8 @@ extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state, extern void fpsimd_flush_task_state(struct task_struct *target); extern void fpsimd_save_and_flush_cpu_state(void); -/* Maximum VL that SVE VL-agnostic software can transparently support */ -#define SVE_VL_ARCH_MAX 0x100 +/* Maximum VL that SVE/SME VL-agnostic software can transparently support */ +#define VL_ARCH_MAX 0x100 /* Offset of FFR in the SVE register dump */ static inline size_t sve_ffr_offset(int vl) @@ -62,15 +62,13 @@ static inline size_t sve_ffr_offset(int vl) static inline void *sve_pffr(struct thread_struct *thread) { - return (char *)thread->sve_state + sve_ffr_offset(thread->sve_vl); + return (char *)thread->sve_state + sve_ffr_offset(thread_get_sve_vl(thread)); } -extern void sve_save_state(void *state, u32 *pfpsr); +extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr); extern void sve_load_state(void const *state, u32 const *pfpsr, - unsigned long vq_minus_1); -extern void sve_flush_live(unsigned long vq_minus_1); -extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state, - unsigned long vq_minus_1); + int restore_ffr); +extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1); extern unsigned int sve_get_vl(void); extern void sve_set_vq(unsigned long vq_minus_1); @@ -79,10 +77,6 @@ extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused); extern u64 read_zcr_features(void); -extern int __ro_after_init sve_max_vl; -extern int __ro_after_init sve_max_virtualisable_vl; -extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX); - /* * Helpers to translate bit indices in sve_vq_map to VQ values (and * vice versa). This allows find_next_bit() to be used to find the @@ -98,23 +92,37 @@ static inline unsigned int __bit_to_vq(unsigned int bit) return SVE_VQ_MAX - bit; } -/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */ -static inline bool sve_vq_available(unsigned int vq) -{ - return test_bit(__vq_to_bit(vq), sve_vq_map); -} + +struct vl_info { + enum vec_type type; + const char *name; /* For display purposes */ + + /* Minimum supported vector length across all CPUs */ + int min_vl; + + /* Maximum supported vector length across all CPUs */ + int max_vl; + int max_virtualisable_vl; + + /* + * Set of available vector lengths, + * where length vq encoded as bit __vq_to_bit(vq): + */ + DECLARE_BITMAP(vq_map, SVE_VQ_MAX); + + /* Set of vector lengths present on at least one cpu: */ + DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX); +}; #ifdef CONFIG_ARM64_SVE -extern size_t sve_state_size(struct task_struct const *task); - extern void sve_alloc(struct task_struct *task); extern void fpsimd_release_task(struct task_struct *task); extern void fpsimd_sync_to_sve(struct task_struct *task); extern void sve_sync_to_fpsimd(struct task_struct *task); extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task); -extern int sve_set_vector_length(struct task_struct *task, +extern int vec_set_vector_length(struct task_struct *task, enum vec_type type, unsigned long vl, unsigned long flags); extern int sve_set_current_vl(unsigned long arg); @@ -143,11 +151,63 @@ static inline void sve_user_enable(void) * Probing and setup functions. * Calls to these functions must be serialised with one another. */ -extern void __init sve_init_vq_map(void); -extern void sve_update_vq_map(void); -extern int sve_verify_vq_map(void); +enum vec_type; + +extern void __init vec_init_vq_map(enum vec_type type); +extern void vec_update_vq_map(enum vec_type type); +extern int vec_verify_vq_map(enum vec_type type); extern void __init sve_setup(void); +extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX]; + +static inline void write_vl(enum vec_type type, u64 val) +{ + u64 tmp; + + switch (type) { +#ifdef CONFIG_ARM64_SVE + case ARM64_VEC_SVE: + tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK; + write_sysreg_s(tmp | val, SYS_ZCR_EL1); + break; +#endif + default: + WARN_ON_ONCE(1); + break; + } +} + +static inline int vec_max_vl(enum vec_type type) +{ + return vl_info[type].max_vl; +} + +static inline int vec_max_virtualisable_vl(enum vec_type type) +{ + return vl_info[type].max_virtualisable_vl; +} + +static inline int sve_max_vl(void) +{ + return vec_max_vl(ARM64_VEC_SVE); +} + +static inline int sve_max_virtualisable_vl(void) +{ + return vec_max_virtualisable_vl(ARM64_VEC_SVE); +} + +/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */ +static inline bool vq_available(enum vec_type type, unsigned int vq) +{ + return test_bit(__vq_to_bit(vq), vl_info[type].vq_map); +} + +static inline bool sve_vq_available(unsigned int vq) +{ + return vq_available(ARM64_VEC_SVE, vq); +} + #else /* ! CONFIG_ARM64_SVE */ static inline void sve_alloc(struct task_struct *task) { } @@ -155,6 +215,11 @@ static inline void fpsimd_release_task(struct task_struct *task) { } static inline void sve_sync_to_fpsimd(struct task_struct *task) { } static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { } +static inline int sve_max_virtualisable_vl(void) +{ + return 0; +} + static inline int sve_set_current_vl(unsigned long arg) { return -EINVAL; @@ -165,14 +230,21 @@ static inline int sve_get_current_vl(void) return -EINVAL; } +static inline int sve_max_vl(void) +{ + return -EINVAL; +} + +static inline bool sve_vq_available(unsigned int vq) { return false; } + static inline void sve_user_disable(void) { BUILD_BUG(); } static inline void sve_user_enable(void) { BUILD_BUG(); } #define sve_cond_update_zcr_vq(val, reg) do { } while (0) -static inline void sve_init_vq_map(void) { } -static inline void sve_update_vq_map(void) { } -static inline int sve_verify_vq_map(void) { return 0; } +static inline void vec_init_vq_map(enum vec_type t) { } +static inline void vec_update_vq_map(enum vec_type t) { } +static inline int vec_verify_vq_map(enum vec_type t) { return 0; } static inline void sve_setup(void) { } #endif /* ! CONFIG_ARM64_SVE */ diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index 00a2c0b69c..2509d7dde5 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -217,28 +217,36 @@ .macro sve_flush_z _for n, 0, 31, _sve_flush_z \n .endm -.macro sve_flush_p_ffr +.macro sve_flush_p _for n, 0, 15, _sve_pfalse \n +.endm +.macro sve_flush_ffr _sve_wrffr 0 .endm -.macro sve_save nxbase, xpfpsr, nxtmp +.macro sve_save nxbase, xpfpsr, save_ffr, nxtmp _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34 _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 + cbz \save_ffr, 921f _sve_rdffr 0 _sve_str_p 0, \nxbase _sve_ldr_p 0, \nxbase, -16 - + b 922f +921: + str xzr, [x\nxbase] // Zero out FFR +922: mrs x\nxtmp, fpsr str w\nxtmp, [\xpfpsr] mrs x\nxtmp, fpcr str w\nxtmp, [\xpfpsr, #4] .endm -.macro __sve_load nxbase, xpfpsr, nxtmp +.macro sve_load nxbase, xpfpsr, restore_ffr, nxtmp _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 + cbz \restore_ffr, 921f _sve_ldr_p 0, \nxbase _sve_wrffr 0 +921: _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16 ldr w\nxtmp, [\xpfpsr] @@ -246,8 +254,3 @@ ldr w\nxtmp, [\xpfpsr, #4] msr fpcr, x\nxtmp .endm - -.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2 - sve_load_vq \xvqminus1, x\nxtmp, \xtmp2 - __sve_load \nxbase, \xpfpsr, \nxtmp -.endm diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index 91fa4baa1a..1494cfa863 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -12,10 +12,21 @@ #define HAVE_FUNCTION_GRAPH_FP_TEST +/* + * HAVE_FUNCTION_GRAPH_RET_ADDR_PTR means that the architecture can provide a + * "return address pointer" which can be used to uniquely identify a return + * address which has been overwritten. + * + * On arm64 we use the address of the caller's frame record, which remains the + * same for the lifetime of the instrumented function, unlike the return + * address in the LR. + */ +#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS #define ARCH_SUPPORTS_FTRACE_OPS 1 #else -#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_ADDR ((unsigned long)function_nocfi(_mcount)) #endif /* The BL at the callsite's adjusted rec->ip */ diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index 8e41faa37c..bc06691d20 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -25,19 +25,14 @@ do { \ " cbz %w0, 3f\n" \ " sub %w4, %w4, %w0\n" \ " cbnz %w4, 1b\n" \ -" mov %w0, %w7\n" \ +" mov %w0, %w6\n" \ "3:\n" \ " dmb ish\n" \ -" .pushsection .fixup,\"ax\"\n" \ -" .align 2\n" \ -"4: mov %w0, %w6\n" \ -" b 3b\n" \ -" .popsection\n" \ - _ASM_EXTABLE(1b, 4b) \ - _ASM_EXTABLE(2b, 4b) \ + _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \ + _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0) \ : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \ "+r" (loops) \ - : "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN) \ + : "r" (oparg), "Ir" (-EAGAIN) \ : "memory"); \ uaccess_disable_privileged(); \ } while (0) @@ -105,18 +100,14 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr, " cbz %w3, 3f\n" " sub %w4, %w4, %w3\n" " cbnz %w4, 1b\n" -" mov %w0, %w8\n" +" mov %w0, %w7\n" "3:\n" " dmb ish\n" "4:\n" -" .pushsection .fixup,\"ax\"\n" -"5: mov %w0, %w7\n" -" b 4b\n" -" .popsection\n" - _ASM_EXTABLE(1b, 5b) - _ASM_EXTABLE(2b, 5b) + _ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0) + _ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0) : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops) - : "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN) + : "r" (oldval), "r" (newval), "Ir" (-EAGAIN) : "memory"); uaccess_disable_privileged(); diff --git a/arch/arm64/include/asm/hyperv-tlfs.h b/arch/arm64/include/asm/hyperv-tlfs.h index 4d964a7f02..bc6c7ac934 100644 --- a/arch/arm64/include/asm/hyperv-tlfs.h +++ b/arch/arm64/include/asm/hyperv-tlfs.h @@ -64,6 +64,15 @@ #define HV_REGISTER_STIMER0_CONFIG 0x000B0000 #define HV_REGISTER_STIMER0_COUNT 0x000B0001 +union hv_msi_entry { + u64 as_uint64[2]; + struct { + u64 address; + u32 data; + u32 reserved; + } __packed; +}; + #include #endif diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h index 00dbcc71ae..9839bfc163 100644 --- a/arch/arm64/include/asm/kexec.h +++ b/arch/arm64/include/asm/kexec.h @@ -90,12 +90,24 @@ static inline void crash_prepare_suspend(void) {} static inline void crash_post_resume(void) {} #endif +#if defined(CONFIG_KEXEC_CORE) +void cpu_soft_restart(unsigned long el2_switch, unsigned long entry, + unsigned long arg0, unsigned long arg1, + unsigned long arg2); +#endif + #define ARCH_HAS_KIMAGE_ARCH struct kimage_arch { void *dtb; phys_addr_t dtb_mem; phys_addr_t kern_reloc; + phys_addr_t el2_vectors; + phys_addr_t ttbr0; + phys_addr_t ttbr1; + phys_addr_t zero_page; + unsigned long phys_offset; + unsigned long t0sz; }; #ifdef CONFIG_KEXEC_FILE diff --git a/arch/arm64/include/asm/kprobes.h b/arch/arm64/include/asm/kprobes.h index 5d38ff4a48..05cd82eeca 100644 --- a/arch/arm64/include/asm/kprobes.h +++ b/arch/arm64/include/asm/kprobes.h @@ -39,7 +39,7 @@ void arch_remove_kprobe(struct kprobe *); int kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr); int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data); -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); void __kprobes *trampoline_probe_handler(struct pt_regs *regs); #endif /* CONFIG_KPROBES */ diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index f67a561e09..01d47c5886 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -295,6 +295,7 @@ #define MDCR_EL2_HPMFZO (UL(1) << 29) #define MDCR_EL2_MTPME (UL(1) << 28) #define MDCR_EL2_TDCC (UL(1) << 27) +#define MDCR_EL2_HLP (UL(1) << 26) #define MDCR_EL2_HCCD (UL(1) << 23) #define MDCR_EL2_TTRF (UL(1) << 19) #define MDCR_EL2_HPMD (UL(1) << 17) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index e86045ac43..d5b0386ef7 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -44,31 +44,40 @@ #define KVM_HOST_SMCCC_FUNC(name) KVM_HOST_SMCCC_ID(__KVM_HOST_SMCCC_FUNC_##name) #define __KVM_HOST_SMCCC_FUNC___kvm_hyp_init 0 -#define __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run 1 -#define __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context 2 -#define __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa 3 -#define __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid 4 -#define __KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context 5 -#define __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff 6 -#define __KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs 7 -#define __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config 8 -#define __KVM_HOST_SMCCC_FUNC___vgic_v3_read_vmcr 9 -#define __KVM_HOST_SMCCC_FUNC___vgic_v3_write_vmcr 10 -#define __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs 11 -#define __KVM_HOST_SMCCC_FUNC___kvm_get_mdcr_el2 12 -#define __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs 13 -#define __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs 14 -#define __KVM_HOST_SMCCC_FUNC___pkvm_init 15 -#define __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp 16 -#define __KVM_HOST_SMCCC_FUNC___pkvm_create_private_mapping 17 -#define __KVM_HOST_SMCCC_FUNC___pkvm_cpu_set_vector 18 -#define __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize 19 -#define __KVM_HOST_SMCCC_FUNC___kvm_adjust_pc 20 #ifndef __ASSEMBLY__ #include +enum __kvm_host_smccc_func { + /* Hypercalls available only prior to pKVM finalisation */ + /* __KVM_HOST_SMCCC_FUNC___kvm_hyp_init */ + __KVM_HOST_SMCCC_FUNC___kvm_get_mdcr_el2 = __KVM_HOST_SMCCC_FUNC___kvm_hyp_init + 1, + __KVM_HOST_SMCCC_FUNC___pkvm_init, + __KVM_HOST_SMCCC_FUNC___pkvm_create_private_mapping, + __KVM_HOST_SMCCC_FUNC___pkvm_cpu_set_vector, + __KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs, + __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs, + __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config, + __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize, + + /* Hypercalls available after pKVM finalisation */ + __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp, + __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp, + __KVM_HOST_SMCCC_FUNC___kvm_adjust_pc, + __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run, + __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid, + __KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context, + __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff, + __KVM_HOST_SMCCC_FUNC___vgic_v3_read_vmcr, + __KVM_HOST_SMCCC_FUNC___vgic_v3_write_vmcr, + __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs, + __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs, + __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_init_traps, +}; + #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] #define DECLARE_KVM_NVHE_SYM(sym) extern char kvm_nvhe_sym(sym)[] @@ -263,9 +272,10 @@ extern u64 __kvm_get_mdcr_el2(void); /* * KVM extable for unexpected exceptions. - * In the same format _asm_extable, but output to a different section so that - * it can be mapped to EL2. The KVM version is not sorted. The caller must - * ensure: + * Create a struct kvm_exception_table_entry output to a section that can be + * mapped by EL2. The table is not sorted. + * + * The caller must ensure: * x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented * code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup. */ diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index fd418955e3..d62405ce3e 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -41,6 +41,8 @@ void kvm_inject_vabt(struct kvm_vcpu *vcpu); void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr); void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr); +void kvm_vcpu_wfi(struct kvm_vcpu *vcpu); + static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) { return !(vcpu->arch.hcr_el2 & HCR_RW); @@ -386,7 +388,7 @@ static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) *vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT; } else { u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); - sctlr |= (1 << 25); + sctlr |= SCTLR_ELx_EE; vcpu_write_sys_reg(vcpu, sctlr, SCTLR_EL1); } } @@ -396,7 +398,10 @@ static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) if (vcpu_mode_is_32bit(vcpu)) return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT); - return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & (1 << 25)); + if (vcpu_mode_priv(vcpu)) + return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_ELx_EE); + else + return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_EL1_E0E); } static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu, diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index fc6ee6c597..031e3a2537 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -26,7 +26,6 @@ #include #include #include -#include #define __KVM_HAVE_ARCH_INTC_INITIALIZED @@ -58,6 +57,7 @@ enum kvm_mode { KVM_MODE_DEFAULT, KVM_MODE_PROTECTED, + KVM_MODE_NONE, }; enum kvm_mode kvm_get_mode(void); @@ -297,9 +297,6 @@ struct kvm_vcpu_arch { /* Exception Information */ struct kvm_vcpu_fault_info fault; - /* State of various workarounds, see kvm_asm.h for bit assignment */ - u64 workaround_flags; - /* Miscellaneous vcpu state flags */ u64 flags; @@ -320,8 +317,8 @@ struct kvm_vcpu_arch { struct kvm_guest_debug_arch vcpu_debug_state; struct kvm_guest_debug_arch external_debug_state; - struct thread_info *host_thread_info; /* hyp VA */ struct user_fpsimd_state *host_fpsimd_state; /* hyp VA */ + struct task_struct *parent_task; struct { /* {Break,watch}point registers */ @@ -366,9 +363,6 @@ struct kvm_vcpu_arch { int target; DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES); - /* Detect first run of a vcpu */ - bool has_run_once; - /* Virtual SError ESR to restore when HCR_EL2.VSE is set */ u64 vsesr_el2; @@ -410,20 +404,17 @@ struct kvm_vcpu_arch { #define KVM_ARM64_DEBUG_DIRTY (1 << 0) #define KVM_ARM64_FP_ENABLED (1 << 1) /* guest FP regs loaded */ #define KVM_ARM64_FP_HOST (1 << 2) /* host FP regs loaded */ -#define KVM_ARM64_HOST_SVE_IN_USE (1 << 3) /* backup for host TIF_SVE */ #define KVM_ARM64_HOST_SVE_ENABLED (1 << 4) /* SVE enabled for EL0 */ #define KVM_ARM64_GUEST_HAS_SVE (1 << 5) /* SVE exposed to guest */ #define KVM_ARM64_VCPU_SVE_FINALIZED (1 << 6) /* SVE config completed */ #define KVM_ARM64_GUEST_HAS_PTRAUTH (1 << 7) /* PTRAUTH exposed to guest */ #define KVM_ARM64_PENDING_EXCEPTION (1 << 8) /* Exception pending */ +/* + * Overlaps with KVM_ARM64_EXCEPT_MASK on purpose so that it can't be + * set together with an exception... + */ +#define KVM_ARM64_INCREMENT_PC (1 << 9) /* Increment PC */ #define KVM_ARM64_EXCEPT_MASK (7 << 9) /* Target EL/MODE */ -#define KVM_ARM64_DEBUG_STATE_SAVE_SPE (1 << 12) /* Save SPE context if active */ -#define KVM_ARM64_DEBUG_STATE_SAVE_TRBE (1 << 13) /* Save TRBE context if active */ - -#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ - KVM_GUESTDBG_USE_SW_BP | \ - KVM_GUESTDBG_USE_HW | \ - KVM_GUESTDBG_SINGLESTEP) /* * When KVM_ARM64_PENDING_EXCEPTION is set, KVM_ARM64_EXCEPT_MASK can * take the following values: @@ -441,11 +432,14 @@ struct kvm_vcpu_arch { #define KVM_ARM64_EXCEPT_AA64_EL1 (0 << 11) #define KVM_ARM64_EXCEPT_AA64_EL2 (1 << 11) -/* - * Overlaps with KVM_ARM64_EXCEPT_MASK on purpose so that it can't be - * set together with an exception... - */ -#define KVM_ARM64_INCREMENT_PC (1 << 9) /* Increment PC */ +#define KVM_ARM64_DEBUG_STATE_SAVE_SPE (1 << 12) /* Save SPE context if active */ +#define KVM_ARM64_DEBUG_STATE_SAVE_TRBE (1 << 13) /* Save TRBE context if active */ +#define KVM_ARM64_FP_FOREIGN_FPSTATE (1 << 14) + +#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ + KVM_GUESTDBG_USE_SW_BP | \ + KVM_GUESTDBG_USE_HW | \ + KVM_GUESTDBG_SINGLESTEP) #define vcpu_has_sve(vcpu) (system_supports_sve() && \ ((vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_SVE)) @@ -583,7 +577,7 @@ struct kvm_vcpu_stat { u64 exits; }; -int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init); +void kvm_vcpu_preferred_target(struct kvm_vcpu_init *init); unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); @@ -605,6 +599,8 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, void kvm_arm_halt_guest(struct kvm *kvm); void kvm_arm_resume_guest(struct kvm *kvm); +#define vcpu_has_run_once(vcpu) !!rcu_access_pointer((vcpu)->pid) + #ifndef __KVM_NVHE_HYPERVISOR__ #define kvm_call_hyp_nvhe(f, ...) \ ({ \ @@ -674,8 +670,15 @@ unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len); int kvm_handle_mmio_return(struct kvm_vcpu *vcpu); int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa); -int kvm_perf_init(void); -int kvm_perf_teardown(void); +/* + * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event, + * arrived in guest context. For arm64, any event that arrives while a vCPU is + * loaded is considered to be "in guest". + */ +static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) +{ + return IS_ENABLED(CONFIG_GUEST_PERF_EVENTS) && !!vcpu; +} long kvm_hypercall_pv_features(struct kvm_vcpu *vcpu); gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu); @@ -721,7 +724,6 @@ void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu); static inline void kvm_arch_hardware_unsetup(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} -static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} void kvm_arm_init_debug(void); void kvm_arm_vcpu_init_debug(struct kvm_vcpu *vcpu); @@ -741,8 +743,10 @@ long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, /* Guest/host FPSIMD coordination helpers */ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu); +void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu); static inline bool kvm_pmu_counter_deferred(struct perf_event_attr *attr) { @@ -753,12 +757,7 @@ static inline bool kvm_pmu_counter_deferred(struct perf_event_attr *attr) void kvm_arch_vcpu_load_debug_state_flags(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_put_debug_state_flags(struct kvm_vcpu *vcpu); -#ifdef CONFIG_KVM /* Avoid conflicts with core headers if CONFIG_KVM=n */ -static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) -{ - return kvm_arch_vcpu_run_map_fp(vcpu); -} - +#ifdef CONFIG_KVM void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr); void kvm_clr_pmu_events(u32 clr); @@ -776,7 +775,6 @@ int kvm_set_ipa_limit(void); #define __KVM_HAVE_ARCH_VM_ALLOC struct kvm *kvm_arch_alloc_vm(void); -void kvm_arch_free_vm(struct kvm *kvm); int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type); @@ -785,6 +783,8 @@ static inline bool kvm_vm_is_protected(struct kvm *kvm) return false; } +void kvm_init_protected_traps(struct kvm_vcpu *vcpu); + int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 657d0c94cf..462882f356 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -90,7 +90,6 @@ void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); void __fpsimd_save_state(struct user_fpsimd_state *fp_regs); void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs); -void __sve_save_state(void *sve_pffr, u32 *fpsr); void __sve_restore_state(void *sve_pffr, u32 *fpsr); #ifndef __KVM_NVHE_HYPERVISOR__ @@ -115,7 +114,12 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus, void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); #endif +extern u64 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val); +extern u64 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val); +extern u64 kvm_nvhe_sym(id_aa64isar0_el1_sys_val); +extern u64 kvm_nvhe_sym(id_aa64isar1_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val); +extern u64 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val); #endif /* __ARM64_KVM_HYP_H__ */ diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 02d3788877..81839e9a8a 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -150,6 +150,8 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) #include #include +int kvm_share_hyp(void *from, void *to); +void kvm_unshare_hyp(void *from, void *to); int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot); int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, void __iomem **kaddr, diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 0277838295..9f339dffbc 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -251,6 +251,27 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt); int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, enum kvm_pgtable_prot prot); +/** + * kvm_pgtable_hyp_unmap() - Remove a mapping from a hypervisor stage-1 page-table. + * @pgt: Page-table structure initialised by kvm_pgtable_hyp_init(). + * @addr: Virtual address from which to remove the mapping. + * @size: Size of the mapping. + * + * The offset of @addr within a page is ignored, @size is rounded-up to + * the next page boundary and @phys is rounded-down to the previous page + * boundary. + * + * TLB invalidation is performed for each page-table entry cleared during the + * unmapping operation and the reference count for the page-table page + * containing the cleared entry is decremented, with unreferenced pages being + * freed. The unmapping operation will stop early if it encounters either an + * invalid page-table entry or a valid block mapping which maps beyond the range + * being unmapped. + * + * Return: Number of bytes unmapped, which may be 0. + */ +u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size); + /** * kvm_get_vtcr() - Helper to construct VTCR_EL2 * @mmfr0: Sanitized value of SYS_ID_AA64MMFR0_EL1 register. @@ -270,8 +291,7 @@ u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift); /** * __kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table. * @pgt: Uninitialised page-table structure to initialise. - * @arch: Arch-specific KVM structure representing the guest virtual - * machine. + * @mmu: S2 MMU context for this S2 translation * @mm_ops: Memory management callbacks. * @flags: Stage-2 configuration flags. * @force_pte_cb: Function that returns true if page level mappings must @@ -279,13 +299,13 @@ u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift); * * Return: 0 on success, negative error code on failure. */ -int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_arch *arch, +int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, struct kvm_pgtable_mm_ops *mm_ops, enum kvm_pgtable_stage2_flags flags, kvm_pgtable_force_pte_cb_t force_pte_cb); -#define kvm_pgtable_stage2_init(pgt, arch, mm_ops) \ - __kvm_pgtable_stage2_init(pgt, arch, mm_ops, 0, NULL) +#define kvm_pgtable_stage2_init(pgt, mmu, mm_ops) \ + __kvm_pgtable_stage2_init(pgt, mmu, mm_ops, 0, NULL) /** * kvm_pgtable_stage2_destroy() - Destroy an unused guest stage-2 page-table. diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index 9906541a68..b77e9b3f53 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -1,48 +1,43 @@ #ifndef __ASM_LINKAGE_H #define __ASM_LINKAGE_H +#ifdef __ASSEMBLY__ +#include +#endif + #define __ALIGN .align 2 #define __ALIGN_STR ".align 2" -#if defined(CONFIG_ARM64_BTI_KERNEL) && defined(__aarch64__) - /* - * Since current versions of gas reject the BTI instruction unless we - * set the architecture version to v8.5 we use the hint instruction - * instead. - */ -#define BTI_C hint 34 ; - -/* - * When using in-kernel BTI we need to ensure that PCS-conformant assembly - * functions have suitable annotations. Override SYM_FUNC_START to insert - * a BTI landing pad at the start of everything. + * When using in-kernel BTI we need to ensure that PCS-conformant + * assembly functions have suitable annotations. Override + * SYM_FUNC_START to insert a BTI landing pad at the start of + * everything, the override is done unconditionally so we're more + * likely to notice any drift from the overridden definitions. */ #define SYM_FUNC_START(name) \ SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ - BTI_C + bti c ; #define SYM_FUNC_START_NOALIGN(name) \ SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \ - BTI_C + bti c ; #define SYM_FUNC_START_LOCAL(name) \ SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \ - BTI_C + bti c ; #define SYM_FUNC_START_LOCAL_NOALIGN(name) \ SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \ - BTI_C + bti c ; #define SYM_FUNC_START_WEAK(name) \ SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \ - BTI_C + bti c ; #define SYM_FUNC_START_WEAK_NOALIGN(name) \ SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ - BTI_C - -#endif + bti c ; /* * Annotate a function as position independent, i.e., safe to be called before diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index 05886322c3..0af70d9abe 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -240,6 +240,7 @@ static inline const void *__tag_set(const void *addr, u8 tag) #ifdef CONFIG_KASAN_HW_TAGS #define arch_enable_tagging_sync() mte_enable_kernel_sync() #define arch_enable_tagging_async() mte_enable_kernel_async() +#define arch_enable_tagging_asymm() mte_enable_kernel_asymm() #define arch_force_async_tag_fault() mte_check_tfsr_exit() #define arch_get_random_tag() mte_get_random_tag() #define arch_get_mem_tag(addr) mte_get_mem_tag(addr) diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index e9c30859f8..48f8466a4b 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -15,6 +15,7 @@ #ifndef __ASSEMBLY__ #include +#include typedef struct { atomic64_t id; diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index f4ba93d4ff..6770667b34 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -115,6 +115,30 @@ static inline void cpu_install_idmap(void) cpu_switch_mm(lm_alias(idmap_pg_dir), &init_mm); } +/* + * Load our new page tables. A strict BBM approach requires that we ensure that + * TLBs are free of any entries that may overlap with the global mappings we are + * about to install. + * + * For a real hibernate/resume/kexec cycle TTBR0 currently points to a zero + * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI runtime + * services), while for a userspace-driven test_resume cycle it points to + * userspace page tables (and we must point it at a zero page ourselves). + * + * We change T0SZ as part of installing the idmap. This is undone by + * cpu_uninstall_idmap() in __cpu_suspend_exit(). + */ +static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz) +{ + cpu_set_reserved_ttbr0(); + local_flush_tlb_all(); + __cpu_set_tcr_t0sz(t0sz); + + /* avoid cpu_switch_mm() and its SW-PAN and CNP interactions */ + write_sysreg(ttbr0, ttbr0_el1); + isb(); +} + /* * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD, * avoiding the possibility of conflicting TLB entries being allocated. diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h index 592aabb25b..a857bcacf0 100644 --- a/arch/arm64/include/asm/mte-kasan.h +++ b/arch/arm64/include/asm/mte-kasan.h @@ -133,6 +133,7 @@ static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag, void mte_enable_kernel_sync(void); void mte_enable_kernel_async(void); +void mte_enable_kernel_asymm(void); #else /* CONFIG_ARM64_MTE */ @@ -164,6 +165,10 @@ static inline void mte_enable_kernel_async(void) { } +static inline void mte_enable_kernel_asymm(void) +{ +} + #endif /* CONFIG_ARM64_MTE */ #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 02511650cf..075539f5f1 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -88,11 +88,11 @@ static inline int mte_ptrace_copy_tags(struct task_struct *child, #ifdef CONFIG_KASAN_HW_TAGS /* Whether the MTE asynchronous mode is enabled. */ -DECLARE_STATIC_KEY_FALSE(mte_async_mode); +DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode); -static inline bool system_uses_mte_async_mode(void) +static inline bool system_uses_mte_async_or_asymm_mode(void) { - return static_branch_unlikely(&mte_async_mode); + return static_branch_unlikely(&mte_async_or_asymm_mode); } void mte_check_tfsr_el1(void); @@ -121,7 +121,7 @@ static inline void mte_check_tfsr_exit(void) mte_check_tfsr_el1(); } #else -static inline bool system_uses_mte_async_mode(void) +static inline bool system_uses_mte_async_or_asymm_mode(void) { return false; } diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index f98c91bbd7..993a27ea6f 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -41,7 +41,6 @@ void tag_clear_highpage(struct page *to); typedef struct page *pgtable_t; -int pfn_valid(unsigned long pfn); int pfn_is_map_memory(unsigned long pfn); #include diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 08363d3cc1..94e147e545 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1017,6 +1017,12 @@ static inline bool arch_wants_old_prefaulted_pte(void) } #define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte +static inline bool pud_sect_supported(void) +{ + return PAGE_SIZE == SZ_4K; +} + + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_PGTABLE_H */ diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index ee2bdc1b9f..6f41b65f99 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -115,6 +115,11 @@ struct debug_info { #endif }; +enum vec_type { + ARM64_VEC_SVE = 0, + ARM64_VEC_MAX, +}; + struct cpu_context { unsigned long x19; unsigned long x20; @@ -147,8 +152,8 @@ struct thread_struct { unsigned int fpsimd_cpu; void *sve_state; /* SVE registers, if any */ - unsigned int sve_vl; /* SVE vector length */ - unsigned int sve_vl_onexec; /* SVE vl after next exec */ + unsigned int vl[ARM64_VEC_MAX]; /* vector length */ + unsigned int vl_onexec[ARM64_VEC_MAX]; /* vl after next exec */ unsigned long fault_address; /* fault info */ unsigned long fault_code; /* ESR_EL1 value */ struct debug_info debug; /* debugging */ @@ -164,6 +169,46 @@ struct thread_struct { u64 sctlr_user; }; +static inline unsigned int thread_get_vl(struct thread_struct *thread, + enum vec_type type) +{ + return thread->vl[type]; +} + +static inline unsigned int thread_get_sve_vl(struct thread_struct *thread) +{ + return thread_get_vl(thread, ARM64_VEC_SVE); +} + +unsigned int task_get_vl(const struct task_struct *task, enum vec_type type); +void task_set_vl(struct task_struct *task, enum vec_type type, + unsigned long vl); +void task_set_vl_onexec(struct task_struct *task, enum vec_type type, + unsigned long vl); +unsigned int task_get_vl_onexec(const struct task_struct *task, + enum vec_type type); + +static inline unsigned int task_get_sve_vl(const struct task_struct *task) +{ + return task_get_vl(task, ARM64_VEC_SVE); +} + +static inline void task_set_sve_vl(struct task_struct *task, unsigned long vl) +{ + task_set_vl(task, ARM64_VEC_SVE, vl); +} + +static inline unsigned int task_get_sve_vl_onexec(const struct task_struct *task) +{ + return task_get_vl_onexec(task, ARM64_VEC_SVE); +} + +static inline void task_set_sve_vl_onexec(struct task_struct *task, + unsigned long vl) +{ + task_set_vl_onexec(task, ARM64_VEC_SVE, vl); +} + #define SCTLR_USER_MASK \ (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB | \ SCTLR_EL1_TCF0_MASK) @@ -257,7 +302,7 @@ struct task_struct; /* Free all resources held by a thread. */ extern void release_thread(struct task_struct *); -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); void update_sctlr_el1(u64 sctlr); diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h index 552891e626..40971ac130 100644 --- a/arch/arm64/include/asm/sections.h +++ b/arch/arm64/include/asm/sections.h @@ -21,6 +21,7 @@ extern char __exittext_begin[], __exittext_end[]; extern char __irqentry_text_start[], __irqentry_text_end[]; extern char __mmuoff_data_start[], __mmuoff_data_end[]; extern char __entry_tramp_text_start[], __entry_tramp_text_end[]; +extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[]; static inline size_t entry_tramp_text_size(void) { diff --git a/arch/arm64/include/asm/setup.h b/arch/arm64/include/asm/setup.h index d3320618ed..6437df6617 100644 --- a/arch/arm64/include/asm/setup.h +++ b/arch/arm64/include/asm/setup.h @@ -8,4 +8,10 @@ void *get_early_fdt_ptr(void); void early_fdt_map(u64 dt_phys); +/* + * These two variables are used in the head.S file. + */ +extern phys_addr_t __fdt_pointer __initdata; +extern u64 __cacheline_aligned boot_args[4]; + #endif diff --git a/arch/arm64/include/asm/spinlock_types.h b/arch/arm64/include/asm/spinlock_types.h index 18782f0c47..11ab1c0776 100644 --- a/arch/arm64/include/asm/spinlock_types.h +++ b/arch/arm64/include/asm/spinlock_types.h @@ -5,7 +5,7 @@ #ifndef __ASM_SPINLOCK_TYPES_H #define __ASM_SPINLOCK_TYPES_H -#if !defined(__LINUX_SPINLOCK_TYPES_H) && !defined(__ASM_SPINLOCK_H) +#if !defined(__LINUX_SPINLOCK_TYPES_RAW_H) && !defined(__ASM_SPINLOCK_H) # error "please don't include this file directly" #endif diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/stacktrace.h index 8aebc00c17..e77cdef9ca 100644 --- a/arch/arm64/include/asm/stacktrace.h +++ b/arch/arm64/include/asm/stacktrace.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -47,8 +48,9 @@ struct stack_info { * value of STACK_TYPE_UNKNOWN. This is used to detect a * transition from one stack to another. * - * @graph: When FUNCTION_GRAPH_TRACER is selected, holds the index of a - * replacement lr value in the ftrace graph stack. + * @kr_cur: When KRETPROBES is selected, holds the kretprobe instance + * associated with the most recently encountered replacement lr + * value. */ struct stackframe { unsigned long fp; @@ -56,14 +58,11 @@ struct stackframe { DECLARE_BITMAP(stacks_done, __NR_STACK_TYPES); unsigned long prev_fp; enum stack_type prev_type; -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - int graph; +#ifdef CONFIG_KRETPROBES + struct llist_node *kr_cur; #endif }; -extern int unwind_frame(struct task_struct *tsk, struct stackframe *frame); -extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame, - bool (*fn)(void *, unsigned long), void *data); extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, const char *loglvl); @@ -148,7 +147,4 @@ static inline bool on_accessible_stack(const struct task_struct *tsk, return false; } -void start_backtrace(struct stackframe *frame, unsigned long fp, - unsigned long pc); - #endif /* __ASM_STACKTRACE_H */ diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h index 03e2089545..4cfe9b4970 100644 --- a/arch/arm64/include/asm/syscall.h +++ b/arch/arm64/include/asm/syscall.h @@ -73,16 +73,6 @@ static inline void syscall_get_arguments(struct task_struct *task, memcpy(args, ®s->regs[1], 5 * sizeof(args[0])); } -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - const unsigned long *args) -{ - regs->orig_x0 = args[0]; - args++; - - memcpy(®s->regs[1], args, 5 * sizeof(args[0])); -} - /* * We don't care about endianness (__AUDIT_ARCH_LE bit) here because * AArch64 has the same system calls both on little- and big- endian. diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 394fc5998a..932d45b178 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -13,6 +13,8 @@ #include #include +#include + /* * ARMv8 ARM reserves the following encoding for system registers: * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview", @@ -508,6 +510,9 @@ #define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0) +#define SYS_CNTPCTSS_EL0 sys_reg(3, 3, 14, 0, 5) +#define SYS_CNTVCTSS_EL0 sys_reg(3, 3, 14, 0, 6) + #define SYS_CNTP_TVAL_EL0 sys_reg(3, 3, 14, 2, 0) #define SYS_CNTP_CTL_EL0 sys_reg(3, 3, 14, 2, 1) #define SYS_CNTP_CVAL_EL0 sys_reg(3, 3, 14, 2, 2) @@ -622,6 +627,7 @@ #define SCTLR_ELx_TCF_NONE (UL(0x0) << SCTLR_ELx_TCF_SHIFT) #define SCTLR_ELx_TCF_SYNC (UL(0x1) << SCTLR_ELx_TCF_SHIFT) #define SCTLR_ELx_TCF_ASYNC (UL(0x2) << SCTLR_ELx_TCF_SHIFT) +#define SCTLR_ELx_TCF_ASYMM (UL(0x3) << SCTLR_ELx_TCF_SHIFT) #define SCTLR_ELx_TCF_MASK (UL(0x3) << SCTLR_ELx_TCF_SHIFT) #define SCTLR_ELx_ENIA_SHIFT 31 @@ -667,6 +673,7 @@ #define SCTLR_EL1_TCF0_NONE (UL(0x0) << SCTLR_EL1_TCF0_SHIFT) #define SCTLR_EL1_TCF0_SYNC (UL(0x1) << SCTLR_EL1_TCF0_SHIFT) #define SCTLR_EL1_TCF0_ASYNC (UL(0x2) << SCTLR_EL1_TCF0_SHIFT) +#define SCTLR_EL1_TCF0_ASYMM (UL(0x3) << SCTLR_EL1_TCF0_SHIFT) #define SCTLR_EL1_TCF0_MASK (UL(0x3) << SCTLR_EL1_TCF0_SHIFT) #define SCTLR_EL1_BT1 (BIT(36)) @@ -823,6 +830,7 @@ #define ID_AA64PFR1_MTE_NI 0x0 #define ID_AA64PFR1_MTE_EL0 0x1 #define ID_AA64PFR1_MTE 0x2 +#define ID_AA64PFR1_MTE_ASYMM 0x3 /* id_aa64zfr0 */ #define ID_AA64ZFR0_F64MM_SHIFT 56 @@ -947,6 +955,7 @@ #define ID_AA64DFR0_PMUVER_8_1 0x4 #define ID_AA64DFR0_PMUVER_8_4 0x5 #define ID_AA64DFR0_PMUVER_8_5 0x6 +#define ID_AA64DFR0_PMUVER_8_7 0x7 #define ID_AA64DFR0_PMUVER_IMP_DEF 0xf #define ID_AA64DFR0_PMSVER_8_2 0x1 @@ -1170,6 +1179,7 @@ #define ICH_HCR_TC (1 << 10) #define ICH_HCR_TALL0 (1 << 11) #define ICH_HCR_TALL1 (1 << 12) +#define ICH_HCR_TDIR (1 << 14) #define ICH_HCR_EOIcount_SHIFT 27 #define ICH_HCR_EOIcount_MASK (0x1f << ICH_HCR_EOIcount_SHIFT) @@ -1202,6 +1212,8 @@ #define ICH_VTR_SEIS_MASK (1 << ICH_VTR_SEIS_SHIFT) #define ICH_VTR_A3V_SHIFT 21 #define ICH_VTR_A3V_MASK (1 << ICH_VTR_A3V_SHIFT) +#define ICH_VTR_TDS_SHIFT 19 +#define ICH_VTR_TDS_MASK (1 << ICH_VTR_TDS_SHIFT) #define ARM64_FEATURE_FIELD_BITS 4 @@ -1210,17 +1222,12 @@ #ifdef __ASSEMBLY__ - .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 - .equ .L__reg_num_x\num, \num - .endr - .equ .L__reg_num_xzr, 31 - .macro mrs_s, rt, sreg - __emit_inst(0xd5200000|(\sreg)|(.L__reg_num_\rt)) + __emit_inst(0xd5200000|(\sreg)|(.L__gpr_num_\rt)) .endm .macro msr_s, sreg, rt - __emit_inst(0xd5000000|(\sreg)|(.L__reg_num_\rt)) + __emit_inst(0xd5000000|(\sreg)|(.L__gpr_num_\rt)) .endm #else @@ -1229,22 +1236,16 @@ #include #include -#define __DEFINE_MRS_MSR_S_REGNUM \ -" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n" \ -" .equ .L__reg_num_x\\num, \\num\n" \ -" .endr\n" \ -" .equ .L__reg_num_xzr, 31\n" - #define DEFINE_MRS_S \ - __DEFINE_MRS_MSR_S_REGNUM \ + __DEFINE_ASM_GPR_NUMS \ " .macro mrs_s, rt, sreg\n" \ - __emit_inst(0xd5200000|(\\sreg)|(.L__reg_num_\\rt)) \ + __emit_inst(0xd5200000|(\\sreg)|(.L__gpr_num_\\rt)) \ " .endm\n" #define DEFINE_MSR_S \ - __DEFINE_MRS_MSR_S_REGNUM \ + __DEFINE_ASM_GPR_NUMS \ " .macro msr_s, sreg, rt\n" \ - __emit_inst(0xd5000000|(\\sreg)|(.L__reg_num_\\rt)) \ + __emit_inst(0xd5000000|(\\sreg)|(.L__gpr_num_\\rt)) \ " .endm\n" #define UNDEFINE_MRS_S \ diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 6623c99f09..e1317b7c45 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -42,6 +42,7 @@ struct thread_info { void *scs_base; void *scs_sp; #endif + u32 cpu; }; #define thread_saved_pc(tsk) \ @@ -78,7 +79,7 @@ int arch_dup_task_struct(struct task_struct *dst, #define TIF_SINGLESTEP 21 #define TIF_32BIT 22 /* 32bit process */ #define TIF_SVE 23 /* Scalable Vector Extension in use */ -#define TIF_SVE_VL_INHERIT 24 /* Inherit sve_vl_onexec across exec */ +#define TIF_SVE_VL_INHERIT 24 /* Inherit SVE vl_onexec across exec */ #define TIF_SSBD 25 /* Wants SSB mitigation */ #define TIF_TAGGED_ADDR 26 /* Allow tagged user addresses */ diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index ec2db3419c..f386b90a79 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -32,7 +32,7 @@ void update_freq_counters_refs(void); /* Replace task scheduler's default thermal pressure API */ #define arch_scale_thermal_pressure topology_get_thermal_pressure -#define arch_set_thermal_pressure topology_set_thermal_pressure +#define arch_update_thermal_pressure topology_update_thermal_pressure #include diff --git a/arch/arm64/include/asm/trans_pgd.h b/arch/arm64/include/asm/trans_pgd.h index 5d08e5adf3..033d400a4e 100644 --- a/arch/arm64/include/asm/trans_pgd.h +++ b/arch/arm64/include/asm/trans_pgd.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2020, Microsoft Corporation. - * Pavel Tatashin + * Copyright (c) 2021, Microsoft Corporation. + * Pasha Tatashin */ #ifndef _ASM_TRANS_TABLE_H @@ -15,7 +15,7 @@ /* * trans_alloc_page * - Allocator that should return exactly one zeroed page, if this - * allocator fails, trans_pgd_create_copy() and trans_pgd_map_page() + * allocator fails, trans_pgd_create_copy() and trans_pgd_idmap_page() * return -ENOMEM error. * * trans_alloc_arg @@ -30,10 +30,12 @@ struct trans_pgd_info { int trans_pgd_create_copy(struct trans_pgd_info *info, pgd_t **trans_pgd, unsigned long start, unsigned long end); -int trans_pgd_map_page(struct trans_pgd_info *info, pgd_t *trans_pgd, - void *page, unsigned long dst_addr, pgprot_t pgprot); - int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0, unsigned long *t0sz, void *page); +int trans_pgd_copy_el2_vectors(struct trans_pgd_info *info, + phys_addr_t *el2_vectors); + +extern char trans_pgd_stub_vectors[]; + #endif /* _ASM_TRANS_TABLE_H */ diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 0fd6056ba4..3a5ff5e205 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -70,12 +71,6 @@ static inline unsigned long __range_ok(const void __user *addr, unsigned long si #define access_ok(addr, size) __range_ok(addr, size) -#define _ASM_EXTABLE(from, to) \ - " .pushsection __ex_table, \"a\"\n" \ - " .align 3\n" \ - " .long (" #from " - .), (" #to " - .)\n" \ - " .popsection\n" - /* * User access enabling/disabling. */ @@ -196,13 +191,13 @@ static inline void __uaccess_enable_tco(void) */ static inline void __uaccess_disable_tco_async(void) { - if (system_uses_mte_async_mode()) + if (system_uses_mte_async_or_asymm_mode()) __uaccess_disable_tco(); } static inline void __uaccess_enable_tco_async(void) { - if (system_uses_mte_async_mode()) + if (system_uses_mte_async_or_asymm_mode()) __uaccess_enable_tco(); } @@ -260,15 +255,9 @@ static inline void __user *__uaccess_mask_ptr(const void __user *ptr) asm volatile( \ "1: " load " " reg "1, [%2]\n" \ "2:\n" \ - " .section .fixup, \"ax\"\n" \ - " .align 2\n" \ - "3: mov %w0, %3\n" \ - " mov %1, #0\n" \ - " b 2b\n" \ - " .previous\n" \ - _ASM_EXTABLE(1b, 3b) \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %w0, %w1) \ : "+r" (err), "=&r" (x) \ - : "r" (addr), "i" (-EFAULT)) + : "r" (addr)) #define __raw_get_mem(ldr, x, ptr, err) \ do { \ @@ -355,14 +344,9 @@ do { \ asm volatile( \ "1: " store " " reg "1, [%2]\n" \ "2:\n" \ - " .section .fixup,\"ax\"\n" \ - " .align 2\n" \ - "3: mov %w0, %3\n" \ - " b 2b\n" \ - " .previous\n" \ - _ASM_EXTABLE(1b, 3b) \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \ : "+r" (err) \ - : "r" (x), "r" (addr), "i" (-EFAULT)) + : "r" (x), "r" (addr)) #define __raw_put_mem(str, x, ptr, err) \ do { \ diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index 844f6ae586..604a2053d0 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -903,6 +903,10 @@ __SYSCALL(__NR_landlock_add_rule, sys_landlock_add_rule) __SYSCALL(__NR_landlock_restrict_self, sys_landlock_restrict_self) #define __NR_process_mrelease 448 __SYSCALL(__NR_process_mrelease, sys_process_mrelease) +#define __NR_futex_waitv 449 +__SYSCALL(__NR_futex_waitv, sys_futex_waitv) +#define __NR_set_mempolicy_home_node 450 +__SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) /* * Please add new compat syscalls above this comment and update diff --git a/arch/arm64/include/asm/vdso/compat_barrier.h b/arch/arm64/include/asm/vdso/compat_barrier.h index 3fd8fd6d8f..3ac35f4a66 100644 --- a/arch/arm64/include/asm/vdso/compat_barrier.h +++ b/arch/arm64/include/asm/vdso/compat_barrier.h @@ -20,16 +20,9 @@ #define dmb(option) __asm__ __volatile__ ("dmb " #option : : : "memory") -#if __LINUX_ARM_ARCH__ >= 8 && defined(CONFIG_AS_DMB_ISHLD) #define aarch32_smp_mb() dmb(ish) #define aarch32_smp_rmb() dmb(ishld) #define aarch32_smp_wmb() dmb(ishst) -#else -#define aarch32_smp_mb() dmb(ish) -#define aarch32_smp_rmb() aarch32_smp_mb() -#define aarch32_smp_wmb() dmb(ishst) -#endif - #undef smp_mb #undef smp_rmb diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h index f64613a96d..bc9a2145f4 100644 --- a/arch/arm64/include/asm/vectors.h +++ b/arch/arm64/include/asm/vectors.h @@ -56,14 +56,14 @@ enum arm64_bp_harden_el1_vectors { DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector); #ifndef CONFIG_UNMAP_KERNEL_AT_EL0 -#define TRAMP_VALIAS 0 +#define TRAMP_VALIAS 0ul #endif static inline const char * arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot) { if (arm64_kernel_unmapped_at_el0()) - return (char *)TRAMP_VALIAS + SZ_2K * slot; + return (char *)(TRAMP_VALIAS + SZ_2K * slot); WARN_ON_ONCE(slot == EL1_VECTOR_KPTI); diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h index 7379f35ae2..3c8af033a9 100644 --- a/arch/arm64/include/asm/virt.h +++ b/arch/arm64/include/asm/virt.h @@ -67,6 +67,8 @@ */ extern u32 __boot_cpu_mode[2]; +#define ARM64_VECTOR_TABLE_LEN SZ_2K + void __hyp_set_vectors(phys_addr_t phys_vector_base); void __hyp_reset_vectors(void); @@ -128,6 +130,11 @@ static __always_inline bool is_protected_kvm_enabled(void) return cpus_have_final_cap(ARM64_KVM_PROTECTED_MODE); } +static inline bool is_hyp_nvhe(void) +{ + return is_hyp_mode_available() && !is_kernel_in_hyp_mode(); +} + #endif /* __ASSEMBLY__ */ #endif /* ! __ASM__VIRT_H */ diff --git a/arch/arm64/include/asm/vmalloc.h b/arch/arm64/include/asm/vmalloc.h index 7a22aeea9b..b9185503fe 100644 --- a/arch/arm64/include/asm/vmalloc.h +++ b/arch/arm64/include/asm/vmalloc.h @@ -2,6 +2,7 @@ #define _ASM_ARM64_VMALLOC_H #include +#include #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP @@ -9,10 +10,9 @@ static inline bool arch_vmap_pud_supported(pgprot_t prot) { /* - * Only 4k granule supports level 1 block mappings. * SW table walks can't handle removal of intermediate entries. */ - return IS_ENABLED(CONFIG_ARM64_4K_PAGES) && + return pud_sect_supported() && !IS_ENABLED(CONFIG_PTDUMP_DEBUGFS); } diff --git a/arch/arm64/include/asm/word-at-a-time.h b/arch/arm64/include/asm/word-at-a-time.h index 2dcb104c64..1c8e4f2490 100644 --- a/arch/arm64/include/asm/word-at-a-time.h +++ b/arch/arm64/include/asm/word-at-a-time.h @@ -53,29 +53,16 @@ static inline unsigned long find_zero(unsigned long mask) */ static inline unsigned long load_unaligned_zeropad(const void *addr) { - unsigned long ret, tmp; + unsigned long ret; __uaccess_enable_tco_async(); /* Load word from unaligned pointer addr */ asm( - "1: ldr %0, %3\n" + "1: ldr %0, %2\n" "2:\n" - " .pushsection .fixup,\"ax\"\n" - " .align 2\n" - "3: bic %1, %2, #0x7\n" - " ldr %0, [%1]\n" - " and %1, %2, #0x7\n" - " lsl %1, %1, #0x3\n" -#ifndef __AARCH64EB__ - " lsr %0, %0, %1\n" -#else - " lsl %0, %0, %1\n" -#endif - " b 2b\n" - " .popsection\n" - _ASM_EXTABLE(1b, 3b) - : "=&r" (ret), "=&r" (tmp) + _ASM_EXTABLE_LOAD_UNALIGNED_ZEROPAD(1b, 2b, %0, %1) + : "=&r" (ret) : "r" (addr), "Q" (*(unsigned long *)addr)); __uaccess_disable_tco_async(); diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 3f1490bfb9..88b3e2a214 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -81,3 +81,6 @@ extra-y += $(head-y) vmlinux.lds ifeq ($(CONFIG_DEBUG_EFI),y) AFLAGS_head.o += -DVMLINUX_PATH="\"$(realpath $(objtree)/vmlinux)\"" endif + +# for cleaning +subdir- += vdso vdso32 diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index f3851724fe..e4dea8db69 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -62,29 +63,22 @@ static int __init parse_acpi(char *arg) } early_param("acpi", parse_acpi); -static int __init dt_scan_depth1_nodes(unsigned long node, - const char *uname, int depth, - void *data) +static bool __init dt_is_stub(void) { - /* - * Ignore anything not directly under the root node; we'll - * catch its parent instead. - */ - if (depth != 1) - return 0; + int node; - if (strcmp(uname, "chosen") == 0) - return 0; + fdt_for_each_subnode(node, initial_boot_params, 0) { + const char *name = fdt_get_name(initial_boot_params, node, NULL); + if (strcmp(name, "chosen") == 0) + continue; + if (strcmp(name, "hypervisor") == 0 && + of_flat_dt_is_compatible(node, "xen,xen")) + continue; - if (strcmp(uname, "hypervisor") == 0 && - of_flat_dt_is_compatible(node, "xen,xen")) - return 0; + return false; + } - /* - * This node at depth 1 is neither a chosen node nor a xen node, - * which we do not expect. - */ - return 1; + return true; } /* @@ -205,8 +199,7 @@ void __init acpi_boot_table_init(void) * and ACPI has not been [force] enabled (acpi=on|force) */ if (param_acpi_off || - (!param_acpi_on && !param_acpi_force && - of_scan_flat_dt(dt_scan_depth1_nodes, NULL))) + (!param_acpi_on && !param_acpi_force && !dt_is_stub())) goto done; /* diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c index 62078b06fe..443a0c4d76 100644 --- a/arch/arm64/kernel/armv8_deprecated.c +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -284,7 +284,7 @@ static void __init register_insn_emulation_sysctl(void) do { \ uaccess_enable_privileged(); \ __asm__ __volatile__( \ - " mov %w3, %w7\n" \ + " mov %w3, %w6\n" \ "0: ldxr"B" %w2, [%4]\n" \ "1: stxr"B" %w0, %w1, [%4]\n" \ " cbz %w0, 2f\n" \ @@ -295,16 +295,10 @@ do { \ "2:\n" \ " mov %w1, %w2\n" \ "3:\n" \ - " .pushsection .fixup,\"ax\"\n" \ - " .align 2\n" \ - "4: mov %w0, %w6\n" \ - " b 3b\n" \ - " .popsection" \ - _ASM_EXTABLE(0b, 4b) \ - _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE_UACCESS_ERR(0b, 3b, %w0) \ + _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \ : "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2) \ : "r" ((unsigned long)addr), "i" (-EAGAIN), \ - "i" (-EFAULT), \ "i" (__SWP_LL_SC_LOOPS) \ : "memory"); \ uaccess_disable_privileged(); \ diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 551427ae8c..1197e76798 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -27,8 +28,8 @@ int main(void) { DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); - DEFINE(TSK_CPU, offsetof(struct task_struct, cpu)); BLANK(); + DEFINE(TSK_TI_CPU, offsetof(struct task_struct, thread_info.cpu)); DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count)); #ifdef CONFIG_ARM64_SW_TTBR0_PAN @@ -110,7 +111,6 @@ int main(void) #ifdef CONFIG_KVM DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt)); DEFINE(VCPU_FAULT_DISR, offsetof(struct kvm_vcpu, arch.fault.disr_el1)); - DEFINE(VCPU_WORKAROUND_FLAGS, offsetof(struct kvm_vcpu, arch.workaround_flags)); DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2)); DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_cpu_context, regs)); DEFINE(CPU_RGSR_EL1, offsetof(struct kvm_cpu_context, sys_regs[RGSR_EL1])); @@ -170,6 +170,16 @@ int main(void) DEFINE(PTRAUTH_KERNEL_KEY_APIA, offsetof(struct ptrauth_keys_kernel, apia)); #endif BLANK(); +#endif +#ifdef CONFIG_KEXEC_CORE + DEFINE(KIMAGE_ARCH_DTB_MEM, offsetof(struct kimage, arch.dtb_mem)); + DEFINE(KIMAGE_ARCH_EL2_VECTORS, offsetof(struct kimage, arch.el2_vectors)); + DEFINE(KIMAGE_ARCH_ZERO_PAGE, offsetof(struct kimage, arch.zero_page)); + DEFINE(KIMAGE_ARCH_PHYS_OFFSET, offsetof(struct kimage, arch.phys_offset)); + DEFINE(KIMAGE_ARCH_TTBR1, offsetof(struct kimage, arch.ttbr1)); + DEFINE(KIMAGE_HEAD, offsetof(struct kimage, head)); + DEFINE(KIMAGE_START, offsetof(struct kimage, start)); + BLANK(); #endif return 0; } diff --git a/arch/arm64/kernel/cpu-reset.S b/arch/arm64/kernel/cpu-reset.S index d47ff63a5b..48a8af97fa 100644 --- a/arch/arm64/kernel/cpu-reset.S +++ b/arch/arm64/kernel/cpu-reset.S @@ -16,8 +16,7 @@ .pushsection .idmap.text, "awx" /* - * __cpu_soft_restart(el2_switch, entry, arg0, arg1, arg2) - Helper for - * cpu_soft_restart. + * cpu_soft_restart(el2_switch, entry, arg0, arg1, arg2) * * @el2_switch: Flag to indicate a switch to EL2 is needed. * @entry: Location to jump to for soft reset. @@ -29,7 +28,7 @@ * branch to what would be the reset vector. It must be executed with the * flat identity mapping. */ -SYM_CODE_START(__cpu_soft_restart) +SYM_CODE_START(cpu_soft_restart) mov_q x12, INIT_SCTLR_EL1_MMU_OFF pre_disable_mmu_workaround /* @@ -48,6 +47,6 @@ SYM_CODE_START(__cpu_soft_restart) mov x1, x3 // arg1 mov x2, x4 // arg2 br x8 -SYM_CODE_END(__cpu_soft_restart) +SYM_CODE_END(cpu_soft_restart) .popsection diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index a33d7b8f3b..146fa2e768 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -340,6 +340,44 @@ static const struct midr_range erratum_1463225[] = { }; #endif +#ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE +static const struct midr_range trbe_overwrite_fill_mode_cpus[] = { +#ifdef CONFIG_ARM64_ERRATUM_2139208 + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), +#endif +#ifdef CONFIG_ARM64_ERRATUM_2119858 + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_RANGE(MIDR_CORTEX_X2, 0, 0, 2, 0), +#endif + {}, +}; +#endif /* CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE */ + +#ifdef CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE +static const struct midr_range tsb_flush_fail_cpus[] = { +#ifdef CONFIG_ARM64_ERRATUM_2067961 + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), +#endif +#ifdef CONFIG_ARM64_ERRATUM_2054223 + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), +#endif + {}, +}; +#endif /* CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE */ + +#ifdef CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE +static struct midr_range trbe_write_out_of_range_cpus[] = { +#ifdef CONFIG_ARM64_ERRATUM_2253138 + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), +#endif +#ifdef CONFIG_ARM64_ERRATUM_2224489 + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_RANGE(MIDR_CORTEX_X2, 0, 0, 2, 0), +#endif + {}, +}; +#endif /* CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE */ + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -540,6 +578,68 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .capability = ARM64_WORKAROUND_NVIDIA_CARMEL_CNP, ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), }, +#endif +#ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE + { + /* + * The erratum work around is handled within the TRBE + * driver and can be applied per-cpu. So, we can allow + * a late CPU to come online with this erratum. + */ + .desc = "ARM erratum 2119858 or 2139208", + .capability = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE, + .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, + CAP_MIDR_RANGE_LIST(trbe_overwrite_fill_mode_cpus), + }, +#endif +#ifdef CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE + { + .desc = "ARM erratum 2067961 or 2054223", + .capability = ARM64_WORKAROUND_TSB_FLUSH_FAILURE, + ERRATA_MIDR_RANGE_LIST(tsb_flush_fail_cpus), + }, +#endif +#ifdef CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE + { + .desc = "ARM erratum 2253138 or 2224489", + .capability = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE, + .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, + CAP_MIDR_RANGE_LIST(trbe_write_out_of_range_cpus), + }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_2077057 + { + .desc = "ARM erratum 2077057", + .capability = ARM64_WORKAROUND_2077057, + ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2), + }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_2064142 + { + .desc = "ARM erratum 2064142", + .capability = ARM64_WORKAROUND_2064142, + + /* Cortex-A510 r0p0 - r0p2 */ + ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2) + }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_2038923 + { + .desc = "ARM erratum 2038923", + .capability = ARM64_WORKAROUND_2038923, + + /* Cortex-A510 r0p0 - r0p2 */ + ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2) + }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_1902691 + { + .desc = "ARM erratum 1902691", + .capability = ARM64_WORKAROUND_1902691, + + /* Cortex-A510 r0p0 - r0p1 */ + ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 1) + }, #endif { } diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index e71c9cfb46..d33687673f 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -959,7 +959,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) { init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr); - sve_init_vq_map(); + vec_init_vq_map(ARM64_VEC_SVE); } if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) @@ -1195,7 +1195,7 @@ void update_cpu_features(int cpu, /* Probe vector lengths, unless we already gave up on SVE */ if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) && !system_capabilities_finalized()) - sve_update_vq_map(); + vec_update_vq_map(ARM64_VEC_SVE); } /* @@ -1657,6 +1657,9 @@ static bool cpu_has_broken_dbm(void) MIDR_ALL_VERSIONS(MIDR_CORTEX_A55), /* Kryo4xx Silver (rdpe => r1p0) */ MIDR_REV(MIDR_QCOM_KRYO_4XX_SILVER, 0xd, 0xe), +#endif +#ifdef CONFIG_ARM64_ERRATUM_2051678 + MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2), #endif {}, }; @@ -1957,6 +1960,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sign = FTR_UNSIGNED, .min_field_value = 1, }, + { + .desc = "Enhanced Counter Virtualization", + .capability = ARM64_HAS_ECV, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64MMFR0_EL1, + .field_pos = ID_AA64MMFR0_ECV_SHIFT, + .sign = FTR_UNSIGNED, + .min_field_value = 1, + }, #ifdef CONFIG_ARM64_PAN { .desc = "Privileged Access Never", @@ -2348,6 +2361,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sign = FTR_UNSIGNED, .cpu_enable = cpu_enable_mte, }, + { + .desc = "Asymmetric MTE Tag Check Fault", + .capability = ARM64_MTE_ASYMM, + .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64PFR1_EL1, + .field_pos = ID_AA64PFR1_MTE_SHIFT, + .min_field_value = ID_AA64PFR1_MTE_ASYMM, + .sign = FTR_UNSIGNED, + }, #endif /* CONFIG_ARM64_MTE */ { .desc = "RCpc load-acquire (LDAPR)", @@ -2769,7 +2792,7 @@ static void verify_sve_features(void) unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK; unsigned int len = zcr & ZCR_ELx_LEN_MASK; - if (len < safe_len || sve_verify_vq_map()) { + if (len < safe_len || vec_verify_vq_map(ARM64_VEC_SVE)) { pr_crit("CPU%d: SVE: vector length support mismatch\n", smp_processor_id()); cpu_die_early(); @@ -2873,6 +2896,7 @@ bool this_cpu_has_cap(unsigned int n) return false; } +EXPORT_SYMBOL_GPL(this_cpu_has_cap); /* * This helper function is used in a narrow window when, diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 32f9796c4f..ef7fcefb96 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -128,7 +129,7 @@ static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs) local_daif_mask(); - flags = READ_ONCE(current_thread_info()->flags); + flags = read_thread_flags(); if (unlikely(flags & _TIF_WORK_MASK)) do_notify_resume(regs, flags); } @@ -219,22 +220,6 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs) lockdep_hardirqs_on(CALLER_ADDR0); } -static void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs) -{ - if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) - arm64_enter_nmi(regs); - else - enter_from_kernel_mode(regs); -} - -static void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs) -{ - if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) - arm64_exit_nmi(regs); - else - exit_to_kernel_mode(regs); -} - static void __sched arm64_preempt_schedule_irq(void) { lockdep_assert_irqs_disabled(); @@ -263,10 +248,14 @@ static void __sched arm64_preempt_schedule_irq(void) static void do_interrupt_handler(struct pt_regs *regs, void (*handler)(struct pt_regs *)) { + struct pt_regs *old_regs = set_irq_regs(regs); + if (on_thread_stack()) call_on_irq_stack(regs, handler); else handler(regs); + + set_irq_regs(old_regs); } extern void (*handle_arch_irq)(struct pt_regs *); @@ -432,13 +421,22 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) } } -static void noinstr el1_interrupt(struct pt_regs *regs, - void (*handler)(struct pt_regs *)) +static __always_inline void __el1_pnmi(struct pt_regs *regs, + void (*handler)(struct pt_regs *)) { - write_sysreg(DAIF_PROCCTX_NOIRQ, daif); - - enter_el1_irq_or_nmi(regs); + arm64_enter_nmi(regs); do_interrupt_handler(regs, handler); + arm64_exit_nmi(regs); +} + +static __always_inline void __el1_irq(struct pt_regs *regs, + void (*handler)(struct pt_regs *)) +{ + enter_from_kernel_mode(regs); + + irq_enter_rcu(); + do_interrupt_handler(regs, handler); + irq_exit_rcu(); /* * Note: thread_info::preempt_count includes both thread_info::count @@ -449,7 +447,17 @@ static void noinstr el1_interrupt(struct pt_regs *regs, READ_ONCE(current_thread_info()->preempt_count) == 0) arm64_preempt_schedule_irq(); - exit_el1_irq_or_nmi(regs); + exit_to_kernel_mode(regs); +} +static void noinstr el1_interrupt(struct pt_regs *regs, + void (*handler)(struct pt_regs *)) +{ + write_sysreg(DAIF_PROCCTX_NOIRQ, daif); + + if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) + __el1_pnmi(regs, handler); + else + __el1_irq(regs, handler); } asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) @@ -667,7 +675,9 @@ static void noinstr el0_interrupt(struct pt_regs *regs, if (regs->pc & BIT(55)) arm64_apply_bp_hardening(); + irq_enter_rcu(); do_interrupt_handler(regs, handler); + irq_exit_rcu(); exit_to_user_mode(regs); } diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index 196e921f61..dc242e269f 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -38,9 +38,10 @@ SYM_FUNC_END(fpsimd_load_state) * * x0 - pointer to buffer for state * x1 - pointer to storage for FPSR + * x2 - Save FFR if non-zero */ SYM_FUNC_START(sve_save_state) - sve_save 0, x1, 2 + sve_save 0, x1, x2, 3 ret SYM_FUNC_END(sve_save_state) @@ -49,10 +50,10 @@ SYM_FUNC_END(sve_save_state) * * x0 - pointer to buffer for state * x1 - pointer to storage for FPSR - * x2 - VQ-1 + * x2 - Restore FFR if non-zero */ SYM_FUNC_START(sve_load_state) - sve_load 0, x1, x2, 3, x4 + sve_load 0, x1, x2, 4 ret SYM_FUNC_END(sve_load_state) @@ -66,35 +67,22 @@ SYM_FUNC_START(sve_set_vq) ret SYM_FUNC_END(sve_set_vq) -/* - * Load SVE state from FPSIMD state. - * - * x0 = pointer to struct fpsimd_state - * x1 = VQ - 1 - * - * Each SVE vector will be loaded with the first 128-bits taken from FPSIMD - * and the rest zeroed. All the other SVE registers will be zeroed. - */ -SYM_FUNC_START(sve_load_from_fpsimd_state) - sve_load_vq x1, x2, x3 - fpsimd_restore x0, 8 - sve_flush_p_ffr - ret -SYM_FUNC_END(sve_load_from_fpsimd_state) - /* * Zero all SVE registers but the first 128-bits of each vector * * VQ must already be configured by caller, any further updates of VQ * will need to ensure that the register state remains valid. * - * x0 = VQ - 1 + * x0 = include FFR? + * x1 = VQ - 1 */ SYM_FUNC_START(sve_flush_live) - cbz x0, 1f // A VQ-1 of 0 is 128 bits so no extra Z state + cbz x1, 1f // A VQ-1 of 0 is 128 bits so no extra Z state sve_flush_z -1: sve_flush_p_ffr - ret +1: sve_flush_p + tbz x0, #0, 2f + sve_flush_ffr +2: ret SYM_FUNC_END(sve_flush_live) #endif /* CONFIG_ARM64_SVE */ diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 8cf970d219..e535480a40 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -77,17 +77,13 @@ .endm SYM_CODE_START(ftrace_regs_caller) -#ifdef BTI_C - BTI_C -#endif + bti c ftrace_regs_entry 1 b ftrace_common SYM_CODE_END(ftrace_regs_caller) SYM_CODE_START(ftrace_caller) -#ifdef BTI_C - BTI_C -#endif + bti c ftrace_regs_entry 0 b ftrace_common SYM_CODE_END(ftrace_caller) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 34e38eb00f..4a3a653df0 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -175,9 +175,9 @@ alternative_else_nop_endif .macro mte_set_kernel_gcr, tmp, tmp2 #ifdef CONFIG_KASAN_HW_TAGS -alternative_if_not ARM64_MTE +alternative_cb kasan_hw_tags_enable b 1f -alternative_else_nop_endif +alternative_cb_end mov \tmp, KERNEL_GCR_EL1 msr_s SYS_GCR_EL1, \tmp 1: @@ -185,10 +185,10 @@ alternative_else_nop_endif .endm .macro mte_set_user_gcr, tsk, tmp, tmp2 -#ifdef CONFIG_ARM64_MTE -alternative_if_not ARM64_MTE +#ifdef CONFIG_KASAN_HW_TAGS +alternative_cb kasan_hw_tags_enable b 1f -alternative_else_nop_endif +alternative_cb_end ldr \tmp, [\tsk, #THREAD_MTE_CTRL] mte_set_gcr \tmp, \tmp2 @@ -1066,8 +1066,10 @@ SYM_CODE_START(__sdei_asm_handler) mov sp, x1 mov x1, x0 // address to complete_and_resume - /* x0 = (x0 <= 1) ? EVENT_COMPLETE:EVENT_COMPLETE_AND_RESUME */ - cmp x0, #1 + /* x0 = (x0 <= SDEI_EV_FAILED) ? + * EVENT_COMPLETE:EVENT_COMPLETE_AND_RESUME + */ + cmp x0, #SDEI_EV_FAILED mov_q x2, SDEI_1_0_FN_SDEI_EVENT_COMPLETE mov_q x3, SDEI_1_0_FN_SDEI_EVENT_COMPLETE_AND_RESUME csel x0, x2, x3, ls diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index ff4962750b..5280e098cf 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -78,7 +79,11 @@ * indicate whether or not the userland FPSIMD state of the current task is * present in the registers. The flag is set unless the FPSIMD registers of this * CPU currently contain the most recent userland FPSIMD state of the current - * task. + * task. If the task is behaving as a VMM, then this is will be managed by + * KVM which will clear it to indicate that the vcpu FPSIMD state is currently + * loaded on the CPU, allowing the state to be saved if a FPSIMD-aware + * softirq kicks in. Upon vcpu_put(), KVM will save the vcpu FP state and + * flag the register state as invalid. * * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may * save the task's FPSIMD context back to task_struct from softirq context. @@ -121,40 +126,62 @@ struct fpsimd_last_state_struct { static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state); -/* Default VL for tasks that don't set it explicitly: */ -static int __sve_default_vl = -1; +__ro_after_init struct vl_info vl_info[ARM64_VEC_MAX] = { +#ifdef CONFIG_ARM64_SVE + [ARM64_VEC_SVE] = { + .type = ARM64_VEC_SVE, + .name = "SVE", + .min_vl = SVE_VL_MIN, + .max_vl = SVE_VL_MIN, + .max_virtualisable_vl = SVE_VL_MIN, + }, +#endif +}; -static int get_sve_default_vl(void) +static unsigned int vec_vl_inherit_flag(enum vec_type type) { - return READ_ONCE(__sve_default_vl); + switch (type) { + case ARM64_VEC_SVE: + return TIF_SVE_VL_INHERIT; + default: + WARN_ON_ONCE(1); + return 0; + } +} + +struct vl_config { + int __default_vl; /* Default VL for tasks */ +}; + +static struct vl_config vl_config[ARM64_VEC_MAX]; + +static inline int get_default_vl(enum vec_type type) +{ + return READ_ONCE(vl_config[type].__default_vl); } #ifdef CONFIG_ARM64_SVE -static void set_sve_default_vl(int val) +static inline int get_sve_default_vl(void) { - WRITE_ONCE(__sve_default_vl, val); + return get_default_vl(ARM64_VEC_SVE); } -/* Maximum supported vector length across all CPUs (initially poisoned) */ -int __ro_after_init sve_max_vl = SVE_VL_MIN; -int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN; +static inline void set_default_vl(enum vec_type type, int val) +{ + WRITE_ONCE(vl_config[type].__default_vl, val); +} -/* - * Set of available vector lengths, - * where length vq encoded as bit __vq_to_bit(vq): - */ -__ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX); -/* Set of vector lengths present on at least one cpu: */ -static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX); +static inline void set_sve_default_vl(int val) +{ + set_default_vl(ARM64_VEC_SVE, val); +} static void __percpu *efi_sve_state; #else /* ! CONFIG_ARM64_SVE */ /* Dummy declaration for code that will be optimised out: */ -extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX); -extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX); extern void __percpu *efi_sve_state; #endif /* ! CONFIG_ARM64_SVE */ @@ -228,6 +255,29 @@ static void sve_free(struct task_struct *task) __sve_free(task); } +unsigned int task_get_vl(const struct task_struct *task, enum vec_type type) +{ + return task->thread.vl[type]; +} + +void task_set_vl(struct task_struct *task, enum vec_type type, + unsigned long vl) +{ + task->thread.vl[type] = vl; +} + +unsigned int task_get_vl_onexec(const struct task_struct *task, + enum vec_type type) +{ + return task->thread.vl_onexec[type]; +} + +void task_set_vl_onexec(struct task_struct *task, enum vec_type type, + unsigned long vl) +{ + task->thread.vl_onexec[type] = vl; +} + /* * TIF_SVE controls whether a task can use SVE without trapping while * in userspace, and also the way a task's FPSIMD/SVE state is stored @@ -287,12 +337,13 @@ static void task_fpsimd_load(void) WARN_ON(!system_supports_fpsimd()); WARN_ON(!have_cpu_fpsimd_context()); - if (IS_ENABLED(CONFIG_ARM64_SVE) && test_thread_flag(TIF_SVE)) + if (IS_ENABLED(CONFIG_ARM64_SVE) && test_thread_flag(TIF_SVE)) { + sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1); sve_load_state(sve_pffr(¤t->thread), - ¤t->thread.uw.fpsimd_state.fpsr, - sve_vq_from_vl(current->thread.sve_vl) - 1); - else + ¤t->thread.uw.fpsimd_state.fpsr, true); + } else { fpsimd_load_state(¤t->thread.uw.fpsimd_state); + } } /* @@ -308,24 +359,26 @@ static void fpsimd_save(void) WARN_ON(!system_supports_fpsimd()); WARN_ON(!have_cpu_fpsimd_context()); - if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - if (IS_ENABLED(CONFIG_ARM64_SVE) && - test_thread_flag(TIF_SVE)) { - if (WARN_ON(sve_get_vl() != last->sve_vl)) { - /* - * Can't save the user regs, so current would - * re-enter user with corrupt state. - * There's no way to recover, so kill it: - */ - force_signal_inject(SIGKILL, SI_KERNEL, 0, 0); - return; - } + if (test_thread_flag(TIF_FOREIGN_FPSTATE)) + return; - sve_save_state((char *)last->sve_state + - sve_ffr_offset(last->sve_vl), - &last->st->fpsr); - } else - fpsimd_save_state(last->st); + if (IS_ENABLED(CONFIG_ARM64_SVE) && + test_thread_flag(TIF_SVE)) { + if (WARN_ON(sve_get_vl() != last->sve_vl)) { + /* + * Can't save the user regs, so current would + * re-enter user with corrupt state. + * There's no way to recover, so kill it: + */ + force_signal_inject(SIGKILL, SI_KERNEL, 0, 0); + return; + } + + sve_save_state((char *)last->sve_state + + sve_ffr_offset(last->sve_vl), + &last->st->fpsr, true); + } else { + fpsimd_save_state(last->st); } } @@ -335,32 +388,36 @@ static void fpsimd_save(void) * If things go wrong there's a bug somewhere, but try to fall back to a * safe choice. */ -static unsigned int find_supported_vector_length(unsigned int vl) +static unsigned int find_supported_vector_length(enum vec_type type, + unsigned int vl) { + struct vl_info *info = &vl_info[type]; int bit; - int max_vl = sve_max_vl; + int max_vl = info->max_vl; if (WARN_ON(!sve_vl_valid(vl))) - vl = SVE_VL_MIN; + vl = info->min_vl; if (WARN_ON(!sve_vl_valid(max_vl))) - max_vl = SVE_VL_MIN; + max_vl = info->min_vl; if (vl > max_vl) vl = max_vl; - bit = find_next_bit(sve_vq_map, SVE_VQ_MAX, + bit = find_next_bit(info->vq_map, SVE_VQ_MAX, __vq_to_bit(sve_vq_from_vl(vl))); return sve_vl_from_vq(__bit_to_vq(bit)); } #if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL) -static int sve_proc_do_default_vl(struct ctl_table *table, int write, +static int vec_proc_do_default_vl(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { + struct vl_info *info = table->extra1; + enum vec_type type = info->type; int ret; - int vl = get_sve_default_vl(); + int vl = get_default_vl(type); struct ctl_table tmp_table = { .data = &vl, .maxlen = sizeof(vl), @@ -372,12 +429,12 @@ static int sve_proc_do_default_vl(struct ctl_table *table, int write, /* Writing -1 has the special meaning "set to max": */ if (vl == -1) - vl = sve_max_vl; + vl = info->max_vl; if (!sve_vl_valid(vl)) return -EINVAL; - set_sve_default_vl(find_supported_vector_length(vl)); + set_default_vl(type, find_supported_vector_length(type, vl)); return 0; } @@ -385,7 +442,8 @@ static struct ctl_table sve_default_vl_table[] = { { .procname = "sve_default_vector_length", .mode = 0644, - .proc_handler = sve_proc_do_default_vl, + .proc_handler = vec_proc_do_default_vl, + .extra1 = &vl_info[ARM64_VEC_SVE], }, { } }; @@ -456,7 +514,7 @@ static void fpsimd_to_sve(struct task_struct *task) if (!system_supports_sve()) return; - vq = sve_vq_from_vl(task->thread.sve_vl); + vq = sve_vq_from_vl(task_get_sve_vl(task)); __fpsimd_to_sve(sst, fst, vq); } @@ -482,7 +540,7 @@ static void sve_to_fpsimd(struct task_struct *task) if (!system_supports_sve()) return; - vq = sve_vq_from_vl(task->thread.sve_vl); + vq = sve_vq_from_vl(task_get_sve_vl(task)); for (i = 0; i < SVE_NUM_ZREGS; ++i) { p = (__uint128_t const *)ZREG(sst, vq, i); fst->vregs[i] = arm64_le128_to_cpu(*p); @@ -495,9 +553,9 @@ static void sve_to_fpsimd(struct task_struct *task) * Return how many bytes of memory are required to store the full SVE * state for task, given task's currently configured vector length. */ -size_t sve_state_size(struct task_struct const *task) +static size_t sve_state_size(struct task_struct const *task) { - return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl)); + return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task_get_sve_vl(task))); } /* @@ -572,13 +630,13 @@ void sve_sync_from_fpsimd_zeropad(struct task_struct *task) if (!test_tsk_thread_flag(task, TIF_SVE)) return; - vq = sve_vq_from_vl(task->thread.sve_vl); + vq = sve_vq_from_vl(task_get_sve_vl(task)); memset(sst, 0, SVE_SIG_REGS_SIZE(vq)); __fpsimd_to_sve(sst, fst, vq); } -int sve_set_vector_length(struct task_struct *task, +int vec_set_vector_length(struct task_struct *task, enum vec_type type, unsigned long vl, unsigned long flags) { if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT | @@ -589,33 +647,35 @@ int sve_set_vector_length(struct task_struct *task, return -EINVAL; /* - * Clamp to the maximum vector length that VL-agnostic SVE code can - * work with. A flag may be assigned in the future to allow setting - * of larger vector lengths without confusing older software. + * Clamp to the maximum vector length that VL-agnostic code + * can work with. A flag may be assigned in the future to + * allow setting of larger vector lengths without confusing + * older software. */ - if (vl > SVE_VL_ARCH_MAX) - vl = SVE_VL_ARCH_MAX; + if (vl > VL_ARCH_MAX) + vl = VL_ARCH_MAX; - vl = find_supported_vector_length(vl); + vl = find_supported_vector_length(type, vl); if (flags & (PR_SVE_VL_INHERIT | PR_SVE_SET_VL_ONEXEC)) - task->thread.sve_vl_onexec = vl; + task_set_vl_onexec(task, type, vl); else /* Reset VL to system default on next exec: */ - task->thread.sve_vl_onexec = 0; + task_set_vl_onexec(task, type, 0); /* Only actually set the VL if not deferred: */ if (flags & PR_SVE_SET_VL_ONEXEC) goto out; - if (vl == task->thread.sve_vl) + if (vl == task_get_vl(task, type)) goto out; /* * To ensure the FPSIMD bits of the SVE vector registers are preserved, * write any live register state back to task_struct, and convert to a - * non-SVE thread. + * regular FPSIMD thread. Since the vector length can only be changed + * with a syscall we can't be in streaming mode while reconfiguring. */ if (task == current) { get_cpu_fpsimd_context(); @@ -636,10 +696,10 @@ int sve_set_vector_length(struct task_struct *task, */ sve_free(task); - task->thread.sve_vl = vl; + task_set_vl(task, type, vl); out: - update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT, + update_tsk_thread_flag(task, vec_vl_inherit_flag(type), flags & PR_SVE_VL_INHERIT); return 0; @@ -647,20 +707,21 @@ int sve_set_vector_length(struct task_struct *task, /* * Encode the current vector length and flags for return. - * This is only required for prctl(): ptrace has separate fields + * This is only required for prctl(): ptrace has separate fields. + * SVE and SME use the same bits for _ONEXEC and _INHERIT. * - * flags are as for sve_set_vector_length(). + * flags are as for vec_set_vector_length(). */ -static int sve_prctl_status(unsigned long flags) +static int vec_prctl_status(enum vec_type type, unsigned long flags) { int ret; if (flags & PR_SVE_SET_VL_ONEXEC) - ret = current->thread.sve_vl_onexec; + ret = task_get_vl_onexec(current, type); else - ret = current->thread.sve_vl; + ret = task_get_vl(current, type); - if (test_thread_flag(TIF_SVE_VL_INHERIT)) + if (test_thread_flag(vec_vl_inherit_flag(type))) ret |= PR_SVE_VL_INHERIT; return ret; @@ -678,11 +739,11 @@ int sve_set_current_vl(unsigned long arg) if (!system_supports_sve() || is_compat_task()) return -EINVAL; - ret = sve_set_vector_length(current, vl, flags); + ret = vec_set_vector_length(current, ARM64_VEC_SVE, vl, flags); if (ret) return ret; - return sve_prctl_status(flags); + return vec_prctl_status(ARM64_VEC_SVE, flags); } /* PR_SVE_GET_VL */ @@ -691,21 +752,18 @@ int sve_get_current_vl(void) if (!system_supports_sve() || is_compat_task()) return -EINVAL; - return sve_prctl_status(0); + return vec_prctl_status(ARM64_VEC_SVE, 0); } -static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX)) +static void vec_probe_vqs(struct vl_info *info, + DECLARE_BITMAP(map, SVE_VQ_MAX)) { unsigned int vq, vl; - unsigned long zcr; bitmap_zero(map, SVE_VQ_MAX); - zcr = ZCR_ELx_LEN_MASK; - zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr; - for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) { - write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */ + write_vl(info->type, vq - 1); /* self-syncing */ vl = sve_get_vl(); vq = sve_vq_from_vl(vl); /* skip intervening lengths */ set_bit(__vq_to_bit(vq), map); @@ -716,10 +774,11 @@ static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX)) * Initialise the set of known supported VQs for the boot CPU. * This is called during kernel boot, before secondary CPUs are brought up. */ -void __init sve_init_vq_map(void) +void __init vec_init_vq_map(enum vec_type type) { - sve_probe_vqs(sve_vq_map); - bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX); + struct vl_info *info = &vl_info[type]; + vec_probe_vqs(info, info->vq_map); + bitmap_copy(info->vq_partial_map, info->vq_map, SVE_VQ_MAX); } /* @@ -727,30 +786,33 @@ void __init sve_init_vq_map(void) * those not supported by the current CPU. * This function is called during the bring-up of early secondary CPUs only. */ -void sve_update_vq_map(void) +void vec_update_vq_map(enum vec_type type) { + struct vl_info *info = &vl_info[type]; DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); - sve_probe_vqs(tmp_map); - bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX); - bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX); + vec_probe_vqs(info, tmp_map); + bitmap_and(info->vq_map, info->vq_map, tmp_map, SVE_VQ_MAX); + bitmap_or(info->vq_partial_map, info->vq_partial_map, tmp_map, + SVE_VQ_MAX); } /* * Check whether the current CPU supports all VQs in the committed set. * This function is called during the bring-up of late secondary CPUs only. */ -int sve_verify_vq_map(void) +int vec_verify_vq_map(enum vec_type type) { + struct vl_info *info = &vl_info[type]; DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); unsigned long b; - sve_probe_vqs(tmp_map); + vec_probe_vqs(info, tmp_map); bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX); - if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) { - pr_warn("SVE: cpu%d: Required vector length(s) missing\n", - smp_processor_id()); + if (bitmap_intersects(tmp_map, info->vq_map, SVE_VQ_MAX)) { + pr_warn("%s: cpu%d: Required vector length(s) missing\n", + info->name, smp_processor_id()); return -EINVAL; } @@ -766,7 +828,7 @@ int sve_verify_vq_map(void) /* Recover the set of supported VQs: */ bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX); /* Find VQs supported that are not globally supported: */ - bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX); + bitmap_andnot(tmp_map, tmp_map, info->vq_map, SVE_VQ_MAX); /* Find the lowest such VQ, if any: */ b = find_last_bit(tmp_map, SVE_VQ_MAX); @@ -777,9 +839,9 @@ int sve_verify_vq_map(void) * Mismatches above sve_max_virtualisable_vl are fine, since * no guest is allowed to configure ZCR_EL2.LEN to exceed this: */ - if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) { - pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n", - smp_processor_id()); + if (sve_vl_from_vq(__bit_to_vq(b)) <= info->max_virtualisable_vl) { + pr_warn("%s: cpu%d: Unsupported vector length(s) present\n", + info->name, smp_processor_id()); return -EINVAL; } @@ -788,6 +850,8 @@ int sve_verify_vq_map(void) static void __init sve_efi_setup(void) { + struct vl_info *info = &vl_info[ARM64_VEC_SVE]; + if (!IS_ENABLED(CONFIG_EFI)) return; @@ -796,11 +860,11 @@ static void __init sve_efi_setup(void) * This is evidence of a crippled system and we are returning void, * so no attempt is made to handle this situation here. */ - if (!sve_vl_valid(sve_max_vl)) + if (!sve_vl_valid(info->max_vl)) goto fail; efi_sve_state = __alloc_percpu( - SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES); + SVE_SIG_REGS_SIZE(sve_vq_from_vl(info->max_vl)), SVE_VQ_BYTES); if (!efi_sve_state) goto fail; @@ -849,6 +913,7 @@ u64 read_zcr_features(void) void __init sve_setup(void) { + struct vl_info *info = &vl_info[ARM64_VEC_SVE]; u64 zcr; DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); unsigned long b; @@ -861,49 +926,52 @@ void __init sve_setup(void) * so sve_vq_map must have at least SVE_VQ_MIN set. * If something went wrong, at least try to patch it up: */ - if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map))) - set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map); + if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map))) + set_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map); zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1); - sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1); + info->max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1); /* * Sanity-check that the max VL we determined through CPU features * corresponds properly to sve_vq_map. If not, do our best: */ - if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl))) - sve_max_vl = find_supported_vector_length(sve_max_vl); + if (WARN_ON(info->max_vl != find_supported_vector_length(ARM64_VEC_SVE, + info->max_vl))) + info->max_vl = find_supported_vector_length(ARM64_VEC_SVE, + info->max_vl); /* * For the default VL, pick the maximum supported value <= 64. * VL == 64 is guaranteed not to grow the signal frame. */ - set_sve_default_vl(find_supported_vector_length(64)); + set_sve_default_vl(find_supported_vector_length(ARM64_VEC_SVE, 64)); - bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map, + bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map, SVE_VQ_MAX); b = find_last_bit(tmp_map, SVE_VQ_MAX); if (b >= SVE_VQ_MAX) /* No non-virtualisable VLs found */ - sve_max_virtualisable_vl = SVE_VQ_MAX; + info->max_virtualisable_vl = SVE_VQ_MAX; else if (WARN_ON(b == SVE_VQ_MAX - 1)) /* No virtualisable VLs? This is architecturally forbidden. */ - sve_max_virtualisable_vl = SVE_VQ_MIN; + info->max_virtualisable_vl = SVE_VQ_MIN; else /* b + 1 < SVE_VQ_MAX */ - sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1)); + info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1)); - if (sve_max_virtualisable_vl > sve_max_vl) - sve_max_virtualisable_vl = sve_max_vl; + if (info->max_virtualisable_vl > info->max_vl) + info->max_virtualisable_vl = info->max_vl; - pr_info("SVE: maximum available vector length %u bytes per vector\n", - sve_max_vl); - pr_info("SVE: default vector length %u bytes per vector\n", - get_sve_default_vl()); + pr_info("%s: maximum available vector length %u bytes per vector\n", + info->name, info->max_vl); + pr_info("%s: default vector length %u bytes per vector\n", + info->name, get_sve_default_vl()); /* KVM decides whether to support mismatched systems. Just warn here: */ - if (sve_max_virtualisable_vl < sve_max_vl) - pr_warn("SVE: unvirtualisable vector lengths present\n"); + if (sve_max_virtualisable_vl() < sve_max_vl()) + pr_warn("%s: unvirtualisable vector lengths present\n", + info->name); sve_efi_setup(); } @@ -958,9 +1026,9 @@ void do_sve_acc(unsigned int esr, struct pt_regs *regs) */ if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { unsigned long vq_minus_one = - sve_vq_from_vl(current->thread.sve_vl) - 1; + sve_vq_from_vl(task_get_sve_vl(current)) - 1; sve_set_vq(vq_minus_one); - sve_flush_live(vq_minus_one); + sve_flush_live(true, vq_minus_one); fpsimd_bind_task_to_cpu(); } else { fpsimd_to_sve(current); @@ -1030,10 +1098,43 @@ void fpsimd_thread_switch(struct task_struct *next) __put_cpu_fpsimd_context(); } -void fpsimd_flush_thread(void) +static void fpsimd_flush_thread_vl(enum vec_type type) { int vl, supported_vl; + /* + * Reset the task vector length as required. This is where we + * ensure that all user tasks have a valid vector length + * configured: no kernel task can become a user task without + * an exec and hence a call to this function. By the time the + * first call to this function is made, all early hardware + * probing is complete, so __sve_default_vl should be valid. + * If a bug causes this to go wrong, we make some noise and + * try to fudge thread.sve_vl to a safe value here. + */ + vl = task_get_vl_onexec(current, type); + if (!vl) + vl = get_default_vl(type); + + if (WARN_ON(!sve_vl_valid(vl))) + vl = vl_info[type].min_vl; + + supported_vl = find_supported_vector_length(type, vl); + if (WARN_ON(supported_vl != vl)) + vl = supported_vl; + + task_set_vl(current, type, vl); + + /* + * If the task is not set to inherit, ensure that the vector + * length will be reset by a subsequent exec: + */ + if (!test_thread_flag(vec_vl_inherit_flag(type))) + task_set_vl_onexec(current, type, 0); +} + +void fpsimd_flush_thread(void) +{ if (!system_supports_fpsimd()) return; @@ -1046,36 +1147,7 @@ void fpsimd_flush_thread(void) if (system_supports_sve()) { clear_thread_flag(TIF_SVE); sve_free(current); - - /* - * Reset the task vector length as required. - * This is where we ensure that all user tasks have a valid - * vector length configured: no kernel task can become a user - * task without an exec and hence a call to this function. - * By the time the first call to this function is made, all - * early hardware probing is complete, so __sve_default_vl - * should be valid. - * If a bug causes this to go wrong, we make some noise and - * try to fudge thread.sve_vl to a safe value here. - */ - vl = current->thread.sve_vl_onexec ? - current->thread.sve_vl_onexec : get_sve_default_vl(); - - if (WARN_ON(!sve_vl_valid(vl))) - vl = SVE_VL_MIN; - - supported_vl = find_supported_vector_length(vl); - if (WARN_ON(supported_vl != vl)) - vl = supported_vl; - - current->thread.sve_vl = vl; - - /* - * If the task is not set to inherit, ensure that the vector - * length will be reset by a subsequent exec: - */ - if (!test_thread_flag(TIF_SVE_VL_INHERIT)) - current->thread.sve_vl_onexec = 0; + fpsimd_flush_thread_vl(ARM64_VEC_SVE); } put_cpu_fpsimd_context(); @@ -1120,7 +1192,7 @@ static void fpsimd_bind_task_to_cpu(void) WARN_ON(!system_supports_fpsimd()); last->st = ¤t->thread.uw.fpsimd_state; last->sve_state = current->thread.sve_state; - last->sve_vl = current->thread.sve_vl; + last->sve_vl = task_get_sve_vl(current); current->thread.fpsimd_cpu = smp_processor_id(); if (system_supports_sve()) { @@ -1151,7 +1223,8 @@ void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state, /* * Load the userland FPSIMD state of 'current' from memory, but only if the * FPSIMD state already held in the registers is /not/ the most recent FPSIMD - * state of 'current' + * state of 'current'. This is called when we are preparing to return to + * userspace to ensure that userspace sees a good register state. */ void fpsimd_restore_current_state(void) { @@ -1182,7 +1255,9 @@ void fpsimd_restore_current_state(void) /* * Load an updated userland FPSIMD state for 'current' from memory and set the * flag that indicates that the FPSIMD register contents are the most recent - * FPSIMD state of 'current' + * FPSIMD state of 'current'. This is used by the signal code to restore the + * register state when returning from a signal handler in FPSIMD only cases, + * any SVE context will be discarded. */ void fpsimd_update_current_state(struct user_fpsimd_state const *state) { @@ -1353,8 +1428,9 @@ void __efi_fpsimd_begin(void) __this_cpu_write(efi_sve_state_used, true); - sve_save_state(sve_state + sve_ffr_offset(sve_max_vl), - &this_cpu_ptr(&efi_fpsimd_state)->fpsr); + sve_save_state(sve_state + sve_ffr_offset(sve_max_vl()), + &this_cpu_ptr(&efi_fpsimd_state)->fpsr, + true); } else { fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state)); } @@ -1378,9 +1454,10 @@ void __efi_fpsimd_end(void) likely(__this_cpu_read(efi_sve_state_used))) { char const *sve_state = this_cpu_ptr(efi_sve_state); - sve_load_state(sve_state + sve_ffr_offset(sve_max_vl), + sve_set_vq(sve_vq_from_vl(sve_get_vl()) - 1); + sve_load_state(sve_state + sve_ffr_offset(sve_max_vl()), &this_cpu_ptr(&efi_fpsimd_state)->fpsr, - sve_vq_from_vl(sve_get_vl()) - 1); + true); __this_cpu_write(efi_sve_state_used, false); } else { diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index 7f467bd9db..4506c4a90a 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -236,11 +236,6 @@ void arch_ftrace_update_code(int command) command |= FTRACE_MAY_SLEEP; ftrace_modify_all_code(command); } - -int __init ftrace_dyn_arch_init(void) -{ - return 0; -} #endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_FUNCTION_GRAPH_TRACER @@ -249,8 +244,6 @@ int __init ftrace_dyn_arch_init(void) * on the way back to parent. For this purpose, this function is called * in _mcount() or ftrace_caller() to replace return address (*parent) on * the call stack to return_to_handler. - * - * Note that @frame_pointer is used only for sanity check later. */ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent, unsigned long frame_pointer) @@ -268,8 +261,10 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent, */ old = *parent; - if (!function_graph_enter(old, self_addr, frame_pointer, NULL)) + if (!function_graph_enter(old, self_addr, frame_pointer, + (void *)frame_pointer)) { *parent = return_hooker; + } } #ifdef CONFIG_DYNAMIC_FTRACE diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 17962452e3..6a98f1a38c 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -412,7 +412,7 @@ SYM_FUNC_END(__create_page_tables) scs_load \tsk adr_l \tmp1, __per_cpu_offset - ldr w\tmp2, [\tsk, #TSK_CPU] + ldr w\tmp2, [\tsk, #TSK_TI_CPU] ldr \tmp1, [\tmp1, \tmp2, lsl #3] set_this_cpu_offset \tmp1 .endm diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S index 81c0186a5e..0e1d9c3c6a 100644 --- a/arch/arm64/kernel/hibernate-asm.S +++ b/arch/arm64/kernel/hibernate-asm.S @@ -15,26 +15,6 @@ #include #include -/* - * To prevent the possibility of old and new partial table walks being visible - * in the tlb, switch the ttbr to a zero page when we invalidate the old - * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i - * Even switching to our copied tables will cause a changed output address at - * each stage of the walk. - */ -.macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2 - phys_to_ttbr \tmp, \zero_page - msr ttbr1_el1, \tmp - isb - tlbi vmalle1 - dsb nsh - phys_to_ttbr \tmp, \page_table - offset_ttbr1 \tmp, \tmp2 - msr ttbr1_el1, \tmp - isb -.endm - - /* * Resume from hibernate * @@ -112,56 +92,4 @@ alternative_insn "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE hvc #0 3: ret SYM_CODE_END(swsusp_arch_suspend_exit) - -/* - * Restore the hyp stub. - * This must be done before the hibernate page is unmapped by _cpu_resume(), - * but happens before any of the hyp-stub's code is cleaned to PoC. - * - * x24: The physical address of __hyp_stub_vectors - */ -SYM_CODE_START_LOCAL(el1_sync) - msr vbar_el2, x24 - eret -SYM_CODE_END(el1_sync) - -.macro invalid_vector label -SYM_CODE_START_LOCAL(\label) - b \label -SYM_CODE_END(\label) -.endm - - invalid_vector el2_sync_invalid - invalid_vector el2_irq_invalid - invalid_vector el2_fiq_invalid - invalid_vector el2_error_invalid - invalid_vector el1_sync_invalid - invalid_vector el1_irq_invalid - invalid_vector el1_fiq_invalid - invalid_vector el1_error_invalid - -/* el2 vectors - switch el2 here while we restore the memory image. */ - .align 11 -SYM_CODE_START(hibernate_el2_vectors) - ventry el2_sync_invalid // Synchronous EL2t - ventry el2_irq_invalid // IRQ EL2t - ventry el2_fiq_invalid // FIQ EL2t - ventry el2_error_invalid // Error EL2t - - ventry el2_sync_invalid // Synchronous EL2h - ventry el2_irq_invalid // IRQ EL2h - ventry el2_fiq_invalid // FIQ EL2h - ventry el2_error_invalid // Error EL2h - - ventry el1_sync // Synchronous 64-bit EL1 - ventry el1_irq_invalid // IRQ 64-bit EL1 - ventry el1_fiq_invalid // FIQ 64-bit EL1 - ventry el1_error_invalid // Error 64-bit EL1 - - ventry el1_sync_invalid // Synchronous 32-bit EL1 - ventry el1_irq_invalid // IRQ 32-bit EL1 - ventry el1_fiq_invalid // FIQ 32-bit EL1 - ventry el1_error_invalid // Error 32-bit EL1 -SYM_CODE_END(hibernate_el2_vectors) - .popsection diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 46a0b4d6e2..6328308be2 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -7,10 +7,6 @@ * Ubuntu project, hibernation support for mach-dove * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu) * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.) - * https://lkml.org/lkml/2010/6/18/4 - * https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html - * https://patchwork.kernel.org/patch/96442/ - * * Copyright (C) 2006 Rafael J. Wysocki */ #define pr_fmt(x) "hibernate: " x @@ -49,10 +45,7 @@ extern int in_suspend; /* Do we need to reset el2? */ -#define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) - -/* temporary el2 vectors in the __hibernate_exit_text section. */ -extern char hibernate_el2_vectors[]; +#define el2_reset_needed() (is_hyp_nvhe()) /* hyp-stub vectors, used to restore el2 during resume from hibernate. */ extern char __hyp_stub_vectors[]; @@ -215,26 +208,7 @@ static int create_safe_exec_page(void *src_start, size_t length, if (rc) return rc; - /* - * Load our new page tables. A strict BBM approach requires that we - * ensure that TLBs are free of any entries that may overlap with the - * global mappings we are about to install. - * - * For a real hibernate/resume cycle TTBR0 currently points to a zero - * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI - * runtime services), while for a userspace-driven test_resume cycle it - * points to userspace page tables (and we must point it at a zero page - * ourselves). - * - * We change T0SZ as part of installing the idmap. This is undone by - * cpu_uninstall_idmap() in __cpu_suspend_exit(). - */ - cpu_set_reserved_ttbr0(); - local_flush_tlb_all(); - __cpu_set_tcr_t0sz(t0sz); - write_sysreg(trans_ttbr0, ttbr0_el1); - isb(); - + cpu_install_ttbr0(trans_ttbr0, t0sz); *phys_dst_addr = virt_to_phys(page); return 0; @@ -434,6 +408,7 @@ int swsusp_arch_resume(void) void *zero_page; size_t exit_size; pgd_t *tmp_pg_dir; + phys_addr_t el2_vectors; void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *, void *, phys_addr_t, phys_addr_t); struct trans_pgd_info trans_info = { @@ -461,6 +436,14 @@ int swsusp_arch_resume(void) return -ENOMEM; } + if (el2_reset_needed()) { + rc = trans_pgd_copy_el2_vectors(&trans_info, &el2_vectors); + if (rc) { + pr_err("Failed to setup el2 vectors\n"); + return rc; + } + } + exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start; /* * Copy swsusp_arch_suspend_exit() to a safe page. This will generate @@ -473,26 +456,14 @@ int swsusp_arch_resume(void) return rc; } - /* - * The hibernate exit text contains a set of el2 vectors, that will - * be executed at el2 with the mmu off in order to reload hyp-stub. - */ - dcache_clean_inval_poc((unsigned long)hibernate_exit, - (unsigned long)hibernate_exit + exit_size); - /* * KASLR will cause the el2 vectors to be in a different location in * the resumed kernel. Load hibernate's temporary copy into el2. * * We can skip this step if we booted at EL1, or are running with VHE. */ - if (el2_reset_needed()) { - phys_addr_t el2_vectors = (phys_addr_t)hibernate_exit; - el2_vectors += hibernate_el2_vectors - - __hibernate_exit_text_start; /* offset */ - + if (el2_reset_needed()) __hyp_set_vectors(el2_vectors); - } hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, resume_hdr.reenter_kernel, restore_pblist, diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index e03e60f948..55a1ced8eb 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -106,7 +106,9 @@ KVM_NVHE_ALIAS(__stop___kvm_ex_table); KVM_NVHE_ALIAS(kvm_arm_hyp_percpu_base); /* PMU available static key */ +#ifdef CONFIG_HW_PERF_EVENTS KVM_NVHE_ALIAS(kvm_arm_pmu_available); +#endif /* Position-independent library routines */ KVM_NVHE_ALIAS_HYP(clear_page, __pi_clear_page); diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 213d56c14f..e16b248699 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -21,12 +21,8 @@ #include #include #include - -#include "cpu-reset.h" - -/* Global variables for the arm64_relocate_new_kernel routine. */ -extern const unsigned char arm64_relocate_new_kernel[]; -extern const unsigned long arm64_relocate_new_kernel_size; +#include +#include /** * kexec_image_info - For debugging output. @@ -43,7 +39,9 @@ static void _kexec_image_info(const char *func, int line, pr_debug(" start: %lx\n", kimage->start); pr_debug(" head: %lx\n", kimage->head); pr_debug(" nr_segments: %lu\n", kimage->nr_segments); + pr_debug(" dtb_mem: %pa\n", &kimage->arch.dtb_mem); pr_debug(" kern_reloc: %pa\n", &kimage->arch.kern_reloc); + pr_debug(" el2_vectors: %pa\n", &kimage->arch.el2_vectors); for (i = 0; i < kimage->nr_segments; i++) { pr_debug(" segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n", @@ -60,29 +58,6 @@ void machine_kexec_cleanup(struct kimage *kimage) /* Empty routine needed to avoid build errors. */ } -int machine_kexec_post_load(struct kimage *kimage) -{ - void *reloc_code = page_to_virt(kimage->control_code_page); - - memcpy(reloc_code, arm64_relocate_new_kernel, - arm64_relocate_new_kernel_size); - kimage->arch.kern_reloc = __pa(reloc_code); - kexec_image_info(kimage); - - /* - * For execution with the MMU off, reloc_code needs to be cleaned to the - * PoC and invalidated from the I-cache. - */ - dcache_clean_inval_poc((unsigned long)reloc_code, - (unsigned long)reloc_code + - arm64_relocate_new_kernel_size); - icache_inval_pou((uintptr_t)reloc_code, - (uintptr_t)reloc_code + - arm64_relocate_new_kernel_size); - - return 0; -} - /** * machine_kexec_prepare - Prepare for a kexec reboot. * @@ -100,45 +75,6 @@ int machine_kexec_prepare(struct kimage *kimage) return 0; } -/** - * kexec_list_flush - Helper to flush the kimage list and source pages to PoC. - */ -static void kexec_list_flush(struct kimage *kimage) -{ - kimage_entry_t *entry; - - for (entry = &kimage->head; ; entry++) { - unsigned int flag; - unsigned long addr; - - /* flush the list entries. */ - dcache_clean_inval_poc((unsigned long)entry, - (unsigned long)entry + - sizeof(kimage_entry_t)); - - flag = *entry & IND_FLAGS; - if (flag == IND_DONE) - break; - - addr = (unsigned long)phys_to_virt(*entry & PAGE_MASK); - - switch (flag) { - case IND_INDIRECTION: - /* Set entry point just before the new list page. */ - entry = (kimage_entry_t *)addr - 1; - break; - case IND_SOURCE: - /* flush the source pages. */ - dcache_clean_inval_poc(addr, addr + PAGE_SIZE); - break; - case IND_DESTINATION: - break; - default: - BUG(); - } - } -} - /** * kexec_segment_flush - Helper to flush the kimage segments to PoC. */ @@ -163,6 +99,77 @@ static void kexec_segment_flush(const struct kimage *kimage) } } +/* Allocates pages for kexec page table */ +static void *kexec_page_alloc(void *arg) +{ + struct kimage *kimage = (struct kimage *)arg; + struct page *page = kimage_alloc_control_pages(kimage, 0); + void *vaddr = NULL; + + if (!page) + return NULL; + + vaddr = page_address(page); + memset(vaddr, 0, PAGE_SIZE); + + return vaddr; +} + +int machine_kexec_post_load(struct kimage *kimage) +{ + int rc; + pgd_t *trans_pgd; + void *reloc_code = page_to_virt(kimage->control_code_page); + long reloc_size; + struct trans_pgd_info info = { + .trans_alloc_page = kexec_page_alloc, + .trans_alloc_arg = kimage, + }; + + /* If in place, relocation is not used, only flush next kernel */ + if (kimage->head & IND_DONE) { + kexec_segment_flush(kimage); + kexec_image_info(kimage); + return 0; + } + + kimage->arch.el2_vectors = 0; + if (is_hyp_nvhe()) { + rc = trans_pgd_copy_el2_vectors(&info, + &kimage->arch.el2_vectors); + if (rc) + return rc; + } + + /* Create a copy of the linear map */ + trans_pgd = kexec_page_alloc(kimage); + if (!trans_pgd) + return -ENOMEM; + rc = trans_pgd_create_copy(&info, &trans_pgd, PAGE_OFFSET, PAGE_END); + if (rc) + return rc; + kimage->arch.ttbr1 = __pa(trans_pgd); + kimage->arch.zero_page = __pa_symbol(empty_zero_page); + + reloc_size = __relocate_new_kernel_end - __relocate_new_kernel_start; + memcpy(reloc_code, __relocate_new_kernel_start, reloc_size); + kimage->arch.kern_reloc = __pa(reloc_code); + rc = trans_pgd_idmap_page(&info, &kimage->arch.ttbr0, + &kimage->arch.t0sz, reloc_code); + if (rc) + return rc; + kimage->arch.phys_offset = virt_to_phys(kimage) - (long)kimage; + + /* Flush the reloc_code in preparation for its execution. */ + dcache_clean_inval_poc((unsigned long)reloc_code, + (unsigned long)reloc_code + reloc_size); + icache_inval_pou((uintptr_t)reloc_code, + (uintptr_t)reloc_code + reloc_size); + kexec_image_info(kimage); + + return 0; +} + /** * machine_kexec - Do the kexec reboot. * @@ -180,31 +187,35 @@ void machine_kexec(struct kimage *kimage) WARN(in_kexec_crash && (stuck_cpus || smp_crash_stop_failed()), "Some CPUs may be stale, kdump will be unreliable.\n"); - /* Flush the kimage list and its buffers. */ - kexec_list_flush(kimage); - - /* Flush the new image if already in place. */ - if ((kimage != kexec_crash_image) && (kimage->head & IND_DONE)) - kexec_segment_flush(kimage); - pr_info("Bye!\n"); local_daif_mask(); /* - * cpu_soft_restart will shutdown the MMU, disable data caches, then - * transfer control to the kern_reloc which contains a copy of - * the arm64_relocate_new_kernel routine. arm64_relocate_new_kernel - * uses physical addressing to relocate the new image to its final - * position and transfers control to the image entry point when the - * relocation is complete. + * Both restart and kernel_reloc will shutdown the MMU, disable data + * caches. However, restart will start new kernel or purgatory directly, + * kernel_reloc contains the body of arm64_relocate_new_kernel * In kexec case, kimage->start points to purgatory assuming that * kernel entry and dtb address are embedded in purgatory by * userspace (kexec-tools). * In kexec_file case, the kernel starts directly without purgatory. */ - cpu_soft_restart(kimage->arch.kern_reloc, kimage->head, kimage->start, - kimage->arch.dtb_mem); + if (kimage->head & IND_DONE) { + typeof(cpu_soft_restart) *restart; + + cpu_install_idmap(); + restart = (void *)__pa_symbol(function_nocfi(cpu_soft_restart)); + restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem, + 0, 0); + } else { + void (*kernel_reloc)(struct kimage *kimage); + + if (is_hyp_nvhe()) + __hyp_set_vectors(kimage->arch.el2_vectors); + cpu_install_ttbr0(kimage->arch.ttbr0, kimage->arch.t0sz); + kernel_reloc = (void *)kimage->arch.kern_reloc; + kernel_reloc(kimage); + } BUG(); /* Should never get here. */ } @@ -261,8 +272,6 @@ void arch_kexec_protect_crashkres(void) { int i; - kexec_segment_flush(kexec_crash_image); - for (i = 0; i < kexec_crash_image->nr_segments; i++) set_memory_valid( __phys_to_virt(kexec_crash_image->segment[i].mem), diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index e5e801bc53..f418ebc65f 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -26,9 +26,12 @@ static DEFINE_PER_CPU_READ_MOSTLY(u64, mte_tcf_preferred); #ifdef CONFIG_KASAN_HW_TAGS -/* Whether the MTE asynchronous mode is enabled. */ -DEFINE_STATIC_KEY_FALSE(mte_async_mode); -EXPORT_SYMBOL_GPL(mte_async_mode); +/* + * The asynchronous and asymmetric MTE modes have the same behavior for + * store operations. This flag is set when either of these modes is enabled. + */ +DEFINE_STATIC_KEY_FALSE(mte_async_or_asymm_mode); +EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode); #endif static void mte_sync_page_tags(struct page *page, pte_t old_pte, @@ -116,7 +119,7 @@ void mte_enable_kernel_sync(void) * Make sure we enter this function when no PE has set * async mode previously. */ - WARN_ONCE(system_uses_mte_async_mode(), + WARN_ONCE(system_uses_mte_async_or_asymm_mode(), "MTE async mode enabled system wide!"); __mte_enable_kernel("synchronous", SCTLR_ELx_TCF_SYNC); @@ -134,8 +137,34 @@ void mte_enable_kernel_async(void) * mode in between sync and async, this strategy needs * to be reviewed. */ - if (!system_uses_mte_async_mode()) - static_branch_enable(&mte_async_mode); + if (!system_uses_mte_async_or_asymm_mode()) + static_branch_enable(&mte_async_or_asymm_mode); +} + +void mte_enable_kernel_asymm(void) +{ + if (cpus_have_cap(ARM64_MTE_ASYMM)) { + __mte_enable_kernel("asymmetric", SCTLR_ELx_TCF_ASYMM); + + /* + * MTE asymm mode behaves as async mode for store + * operations. The mode is set system wide by the + * first PE that executes this function. + * + * Note: If in future KASAN acquires a runtime switching + * mode in between sync and async, this strategy needs + * to be reviewed. + */ + if (!system_uses_mte_async_or_asymm_mode()) + static_branch_enable(&mte_async_or_asymm_mode); + } else { + /* + * If the CPU does not support MTE asymmetric mode the + * kernel falls back on synchronous mode which is the + * default for kasan=on. + */ + mte_enable_kernel_sync(); + } } #endif @@ -179,6 +208,30 @@ static void mte_update_sctlr_user(struct task_struct *task) task->thread.sctlr_user = sctlr; } +static void mte_update_gcr_excl(struct task_struct *task) +{ + /* + * SYS_GCR_EL1 will be set to current->thread.mte_ctrl value by + * mte_set_user_gcr() in kernel_exit, but only if KASAN is enabled. + */ + if (kasan_hw_tags_enabled()) + return; + + write_sysreg_s( + ((task->thread.mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) & + SYS_GCR_EL1_EXCL_MASK) | SYS_GCR_EL1_RRND, + SYS_GCR_EL1); +} + +void __init kasan_hw_tags_enable(struct alt_instr *alt, __le32 *origptr, + __le32 *updptr, int nr_inst) +{ + BUG_ON(nr_inst != 1); /* Branch -> NOP */ + + if (kasan_hw_tags_enabled()) + *updptr = cpu_to_le32(aarch64_insn_gen_nop()); +} + void mte_thread_init_user(void) { if (!system_supports_mte()) @@ -198,6 +251,7 @@ void mte_thread_switch(struct task_struct *next) return; mte_update_sctlr_user(next); + mte_update_gcr_excl(next); /* * Check if an async tag exception occurred at EL1. @@ -243,6 +297,7 @@ long set_mte_ctrl(struct task_struct *task, unsigned long arg) if (task == current) { preempt_disable(); mte_update_sctlr_user(task); + mte_update_gcr_excl(task); update_sctlr_el1(task->thread.sctlr_user); preempt_enable(); } diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c index 86d9f20131..65b196e3ca 100644 --- a/arch/arm64/kernel/perf_callchain.c +++ b/arch/arm64/kernel/perf_callchain.c @@ -5,10 +5,10 @@ * Copyright (C) 2015 ARM Limited */ #include +#include #include #include -#include struct frame_tail { struct frame_tail __user *fp; @@ -102,9 +102,7 @@ compat_user_backtrace(struct compat_frame_tail __user *tail, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - if (guest_cbs && guest_cbs->is_in_guest()) { + if (perf_guest_state()) { /* We don't support guest os callchain now */ return; } @@ -134,50 +132,38 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, } } -/* - * Gets called by walk_stackframe() for every stackframe. This will be called - * whist unwinding the stackframe and is like a subroutine return so we use - * the PC. - */ static bool callchain_trace(void *data, unsigned long pc) { struct perf_callchain_entry_ctx *entry = data; - perf_callchain_store(entry, pc); - return true; + return perf_callchain_store(entry, pc) == 0; } void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - struct stackframe frame; - - if (guest_cbs && guest_cbs->is_in_guest()) { + if (perf_guest_state()) { /* We don't support guest os callchain now */ return; } - start_backtrace(&frame, regs->regs[29], regs->pc); - walk_stackframe(current, &frame, callchain_trace, entry); + arch_stack_walk(callchain_trace, entry, current, regs); } unsigned long perf_instruction_pointer(struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - if (guest_cbs && guest_cbs->is_in_guest()) - return guest_cbs->get_guest_ip(); + if (perf_guest_state()) + return perf_guest_get_ip(); return instruction_pointer(regs); } unsigned long perf_misc_flags(struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + unsigned int guest_state = perf_guest_state(); int misc = 0; - if (guest_cbs && guest_cbs->is_in_guest()) { - if (guest_cbs->is_user_mode()) + if (guest_state) { + if (guest_state & PERF_GUEST_USER) misc |= PERF_RECORD_MISC_GUEST_USER; else misc |= PERF_RECORD_MISC_GUEST_KERNEL; diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index b404446952..cab678ed66 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -285,15 +285,24 @@ static const struct attribute_group armv8_pmuv3_events_attr_group = { PMU_FORMAT_ATTR(event, "config:0-15"); PMU_FORMAT_ATTR(long, "config1:0"); +PMU_FORMAT_ATTR(rdpmc, "config1:1"); + +static int sysctl_perf_user_access __read_mostly; static inline bool armv8pmu_event_is_64bit(struct perf_event *event) { return event->attr.config1 & 0x1; } +static inline bool armv8pmu_event_want_user_access(struct perf_event *event) +{ + return event->attr.config1 & 0x2; +} + static struct attribute *armv8_pmuv3_format_attrs[] = { &format_attr_event.attr, &format_attr_long.attr, + &format_attr_rdpmc.attr, NULL, }; @@ -362,7 +371,7 @@ static const struct attribute_group armv8_pmuv3_caps_attr_group = { */ #define ARMV8_IDX_CYCLE_COUNTER 0 #define ARMV8_IDX_COUNTER0 1 - +#define ARMV8_IDX_CYCLE_COUNTER_USER 32 /* * We unconditionally enable ARMv8.5-PMU long event counter support @@ -374,18 +383,22 @@ static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu) return (cpu_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_5); } +static inline bool armv8pmu_event_has_user_read(struct perf_event *event) +{ + return event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT; +} + /* * We must chain two programmable counters for 64 bit events, * except when we have allocated the 64bit cycle counter (for CPU - * cycles event). This must be called only when the event has - * a counter allocated. + * cycles event) or when user space counter access is enabled. */ static inline bool armv8pmu_event_is_chained(struct perf_event *event) { int idx = event->hw.idx; struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - return !WARN_ON(idx < 0) && + return !armv8pmu_event_has_user_read(event) && armv8pmu_event_is_64bit(event) && !armv8pmu_has_long_event(cpu_pmu) && (idx != ARMV8_IDX_CYCLE_COUNTER); @@ -718,6 +731,28 @@ static inline u32 armv8pmu_getreset_flags(void) return value; } +static void armv8pmu_disable_user_access(void) +{ + write_sysreg(0, pmuserenr_el0); +} + +static void armv8pmu_enable_user_access(struct arm_pmu *cpu_pmu) +{ + int i; + struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); + + /* Clear any unused counters to avoid leaking their contents */ + for_each_clear_bit(i, cpuc->used_mask, cpu_pmu->num_events) { + if (i == ARMV8_IDX_CYCLE_COUNTER) + write_sysreg(0, pmccntr_el0); + else + armv8pmu_write_evcntr(i, 0); + } + + write_sysreg(0, pmuserenr_el0); + write_sysreg(ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR, pmuserenr_el0); +} + static void armv8pmu_enable_event(struct perf_event *event) { /* @@ -761,6 +796,14 @@ static void armv8pmu_disable_event(struct perf_event *event) static void armv8pmu_start(struct arm_pmu *cpu_pmu) { + struct perf_event_context *task_ctx = + this_cpu_ptr(cpu_pmu->pmu.pmu_cpu_context)->task_ctx; + + if (sysctl_perf_user_access && task_ctx && task_ctx->nr_user) + armv8pmu_enable_user_access(cpu_pmu); + else + armv8pmu_disable_user_access(); + /* Enable all counters */ armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E); } @@ -878,13 +921,16 @@ static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) { if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask)) return ARMV8_IDX_CYCLE_COUNTER; + else if (armv8pmu_event_is_64bit(event) && + armv8pmu_event_want_user_access(event) && + !armv8pmu_has_long_event(cpu_pmu)) + return -EAGAIN; } /* * Otherwise use events counters */ - if (armv8pmu_event_is_64bit(event) && - !armv8pmu_has_long_event(cpu_pmu)) + if (armv8pmu_event_is_chained(event)) return armv8pmu_get_chain_idx(cpuc, cpu_pmu); else return armv8pmu_get_single_idx(cpuc, cpu_pmu); @@ -900,6 +946,22 @@ static void armv8pmu_clear_event_idx(struct pmu_hw_events *cpuc, clear_bit(idx - 1, cpuc->used_mask); } +static int armv8pmu_user_event_idx(struct perf_event *event) +{ + if (!sysctl_perf_user_access || !armv8pmu_event_has_user_read(event)) + return 0; + + /* + * We remap the cycle counter index to 32 to + * match the offset applied to the rest of + * the counter indices. + */ + if (event->hw.idx == ARMV8_IDX_CYCLE_COUNTER) + return ARMV8_IDX_CYCLE_COUNTER_USER; + + return event->hw.idx; +} + /* * Add an event filter to a given event. */ @@ -996,6 +1058,25 @@ static int __armv8_pmuv3_map_event(struct perf_event *event, if (armv8pmu_event_is_64bit(event)) event->hw.flags |= ARMPMU_EVT_64BIT; + /* + * User events must be allocated into a single counter, and so + * must not be chained. + * + * Most 64-bit events require long counter support, but 64-bit + * CPU_CYCLES events can be placed into the dedicated cycle + * counter when this is free. + */ + if (armv8pmu_event_want_user_access(event)) { + if (!(event->attach_state & PERF_ATTACH_TASK)) + return -EINVAL; + if (armv8pmu_event_is_64bit(event) && + (hw_event_id != ARMV8_PMUV3_PERFCTR_CPU_CYCLES) && + !armv8pmu_has_long_event(armpmu)) + return -EOPNOTSUPP; + + event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; + } + /* Only expose micro/arch events supported by this PMU */ if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS) && test_bit(hw_event_id, armpmu->pmceid_bitmap)) { @@ -1104,6 +1185,43 @@ static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu) return probe.present ? 0 : -ENODEV; } +static void armv8pmu_disable_user_access_ipi(void *unused) +{ + armv8pmu_disable_user_access(); +} + +static int armv8pmu_proc_user_access_handler(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write || sysctl_perf_user_access) + return ret; + + on_each_cpu(armv8pmu_disable_user_access_ipi, NULL, 1); + return 0; +} + +static struct ctl_table armv8_pmu_sysctl_table[] = { + { + .procname = "perf_user_access", + .data = &sysctl_perf_user_access, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = armv8pmu_proc_user_access_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { } +}; + +static void armv8_pmu_register_sysctl_table(void) +{ + static u32 tbl_registered = 0; + + if (!cmpxchg_relaxed(&tbl_registered, 0, 1)) + register_sysctl("kernel", armv8_pmu_sysctl_table); +} + static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name, int (*map_event)(struct perf_event *event), const struct attribute_group *events, @@ -1127,6 +1245,8 @@ static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name, cpu_pmu->set_event_filter = armv8pmu_set_event_filter; cpu_pmu->filter_match = armv8pmu_filter_match; + cpu_pmu->pmu.event_idx = armv8pmu_user_event_idx; + cpu_pmu->name = name; cpu_pmu->map_event = map_event; cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = events ? @@ -1136,6 +1256,7 @@ static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name, cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = caps ? caps : &armv8_pmuv3_caps_attr_group; + armv8_pmu_register_sysctl_table(); return 0; } @@ -1145,17 +1266,32 @@ static int armv8_pmu_init_nogroups(struct arm_pmu *cpu_pmu, char *name, return armv8_pmu_init(cpu_pmu, name, map_event, NULL, NULL, NULL); } -static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_pmuv3", - armv8_pmuv3_map_event); +#define PMUV3_INIT_SIMPLE(name) \ +static int name##_pmu_init(struct arm_pmu *cpu_pmu) \ +{ \ + return armv8_pmu_init_nogroups(cpu_pmu, #name, armv8_pmuv3_map_event);\ } -static int armv8_a34_pmu_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a34", - armv8_pmuv3_map_event); -} +PMUV3_INIT_SIMPLE(armv8_pmuv3) + +PMUV3_INIT_SIMPLE(armv8_cortex_a34) +PMUV3_INIT_SIMPLE(armv8_cortex_a55) +PMUV3_INIT_SIMPLE(armv8_cortex_a65) +PMUV3_INIT_SIMPLE(armv8_cortex_a75) +PMUV3_INIT_SIMPLE(armv8_cortex_a76) +PMUV3_INIT_SIMPLE(armv8_cortex_a77) +PMUV3_INIT_SIMPLE(armv8_cortex_a78) +PMUV3_INIT_SIMPLE(armv9_cortex_a510) +PMUV3_INIT_SIMPLE(armv9_cortex_a710) +PMUV3_INIT_SIMPLE(armv8_cortex_x1) +PMUV3_INIT_SIMPLE(armv9_cortex_x2) +PMUV3_INIT_SIMPLE(armv8_neoverse_e1) +PMUV3_INIT_SIMPLE(armv8_neoverse_n1) +PMUV3_INIT_SIMPLE(armv9_neoverse_n2) +PMUV3_INIT_SIMPLE(armv8_neoverse_v1) + +PMUV3_INIT_SIMPLE(armv8_nvidia_carmel) +PMUV3_INIT_SIMPLE(armv8_nvidia_denver) static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu) { @@ -1169,24 +1305,12 @@ static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu) armv8_a53_map_event); } -static int armv8_a55_pmu_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a55", - armv8_pmuv3_map_event); -} - static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu) { return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a57", armv8_a57_map_event); } -static int armv8_a65_pmu_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a65", - armv8_pmuv3_map_event); -} - static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu) { return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a72", @@ -1199,42 +1323,6 @@ static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu) armv8_a73_map_event); } -static int armv8_a75_pmu_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a75", - armv8_pmuv3_map_event); -} - -static int armv8_a76_pmu_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a76", - armv8_pmuv3_map_event); -} - -static int armv8_a77_pmu_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a77", - armv8_pmuv3_map_event); -} - -static int armv8_a78_pmu_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a78", - armv8_pmuv3_map_event); -} - -static int armv8_e1_pmu_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_neoverse_e1", - armv8_pmuv3_map_event); -} - -static int armv8_n1_pmu_init(struct arm_pmu *cpu_pmu) -{ - return armv8_pmu_init_nogroups(cpu_pmu, "armv8_neoverse_n1", - armv8_pmuv3_map_event); -} - static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu) { return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cavium_thunder", @@ -1248,23 +1336,31 @@ static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu) } static const struct of_device_id armv8_pmu_of_device_ids[] = { - {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init}, - {.compatible = "arm,cortex-a34-pmu", .data = armv8_a34_pmu_init}, + {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_pmu_init}, + {.compatible = "arm,cortex-a34-pmu", .data = armv8_cortex_a34_pmu_init}, {.compatible = "arm,cortex-a35-pmu", .data = armv8_a35_pmu_init}, {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init}, - {.compatible = "arm,cortex-a55-pmu", .data = armv8_a55_pmu_init}, + {.compatible = "arm,cortex-a55-pmu", .data = armv8_cortex_a55_pmu_init}, {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init}, - {.compatible = "arm,cortex-a65-pmu", .data = armv8_a65_pmu_init}, + {.compatible = "arm,cortex-a65-pmu", .data = armv8_cortex_a65_pmu_init}, {.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init}, {.compatible = "arm,cortex-a73-pmu", .data = armv8_a73_pmu_init}, - {.compatible = "arm,cortex-a75-pmu", .data = armv8_a75_pmu_init}, - {.compatible = "arm,cortex-a76-pmu", .data = armv8_a76_pmu_init}, - {.compatible = "arm,cortex-a77-pmu", .data = armv8_a77_pmu_init}, - {.compatible = "arm,cortex-a78-pmu", .data = armv8_a78_pmu_init}, - {.compatible = "arm,neoverse-e1-pmu", .data = armv8_e1_pmu_init}, - {.compatible = "arm,neoverse-n1-pmu", .data = armv8_n1_pmu_init}, + {.compatible = "arm,cortex-a75-pmu", .data = armv8_cortex_a75_pmu_init}, + {.compatible = "arm,cortex-a76-pmu", .data = armv8_cortex_a76_pmu_init}, + {.compatible = "arm,cortex-a77-pmu", .data = armv8_cortex_a77_pmu_init}, + {.compatible = "arm,cortex-a78-pmu", .data = armv8_cortex_a78_pmu_init}, + {.compatible = "arm,cortex-a510-pmu", .data = armv9_cortex_a510_pmu_init}, + {.compatible = "arm,cortex-a710-pmu", .data = armv9_cortex_a710_pmu_init}, + {.compatible = "arm,cortex-x1-pmu", .data = armv8_cortex_x1_pmu_init}, + {.compatible = "arm,cortex-x2-pmu", .data = armv9_cortex_x2_pmu_init}, + {.compatible = "arm,neoverse-e1-pmu", .data = armv8_neoverse_e1_pmu_init}, + {.compatible = "arm,neoverse-n1-pmu", .data = armv8_neoverse_n1_pmu_init}, + {.compatible = "arm,neoverse-n2-pmu", .data = armv9_neoverse_n2_pmu_init}, + {.compatible = "arm,neoverse-v1-pmu", .data = armv8_neoverse_v1_pmu_init}, {.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init}, {.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init}, + {.compatible = "nvidia,carmel-pmu", .data = armv8_nvidia_carmel_pmu_init}, + {.compatible = "nvidia,denver-pmu", .data = armv8_nvidia_denver_pmu_init}, {}, }; @@ -1287,7 +1383,7 @@ static int __init armv8_pmu_driver_init(void) if (acpi_disabled) return platform_driver_register(&armv8_pmu_driver); else - return arm_pmu_acpi_probe(armv8_pmuv3_init); + return arm_pmu_acpi_probe(armv8_pmuv3_pmu_init); } device_initcall(armv8_pmu_driver_init) @@ -1301,6 +1397,14 @@ void arch_perf_update_userpage(struct perf_event *event, userpg->cap_user_time = 0; userpg->cap_user_time_zero = 0; userpg->cap_user_time_short = 0; + userpg->cap_user_rdpmc = armv8pmu_event_has_user_read(event); + + if (userpg->cap_user_rdpmc) { + if (event->hw.flags & ARMPMU_EVT_64BIT) + userpg->pmc_width = 64; + else + userpg->pmc_width = 32; + } do { rd = sched_clock_read_begin(&seq); diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 6dbcc89f66..d9dfa82c1f 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -7,6 +7,9 @@ * Copyright (C) 2013 Linaro Limited. * Author: Sandeepa Prabhu */ + +#define pr_fmt(fmt) "kprobes: " fmt + #include #include #include @@ -218,7 +221,7 @@ static int __kprobes reenter_kprobe(struct kprobe *p, break; case KPROBE_HIT_SS: case KPROBE_REENTER: - pr_warn("Unrecoverable kprobe detected.\n"); + pr_warn("Failed to recover from reentered kprobes.\n"); dump_kprobe(p); BUG(); break; @@ -398,18 +401,17 @@ int __init arch_populate_kprobe_blacklist(void) void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs) { - return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, - (void *)kernel_stack_pointer(regs)); + return (void *)kretprobe_trampoline_handler(regs, (void *)regs->regs[29]); } void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { ri->ret_addr = (kprobe_opcode_t *)regs->regs[30]; - ri->fp = (void *)kernel_stack_pointer(regs); + ri->fp = (void *)regs->regs[29]; /* replace return addr (x30) with trampoline */ - regs->regs[30] = (long)&kretprobe_trampoline; + regs->regs[30] = (long)&__kretprobe_trampoline; } int __kprobes arch_trampoline_kprobe(struct kprobe *p) diff --git a/arch/arm64/kernel/probes/kprobes_trampoline.S b/arch/arm64/kernel/probes/kprobes_trampoline.S index 288a84e253..9a6499bed5 100644 --- a/arch/arm64/kernel/probes/kprobes_trampoline.S +++ b/arch/arm64/kernel/probes/kprobes_trampoline.S @@ -61,11 +61,14 @@ ldp x28, x29, [sp, #S_X28] .endm -SYM_CODE_START(kretprobe_trampoline) +SYM_CODE_START(__kretprobe_trampoline) sub sp, sp, #PT_REGS_SIZE save_all_base_regs + /* Setup a frame pointer. */ + add x29, sp, #S_FP + mov x0, sp bl trampoline_probe_handler /* @@ -74,9 +77,10 @@ SYM_CODE_START(kretprobe_trampoline) */ mov lr, x0 + /* The frame pointer (x29) is restored with other registers. */ restore_all_base_regs add sp, sp, #PT_REGS_SIZE ret -SYM_CODE_END(kretprobe_trampoline) +SYM_CODE_END(__kretprobe_trampoline) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 23efabcb00..5369e649fa 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -482,7 +483,8 @@ void update_sctlr_el1(u64 sctlr) /* * Thread switching. */ -__notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, +__notrace_funcgraph __sched +struct task_struct *__switch_to(struct task_struct *prev, struct task_struct *next) { struct task_struct *last; @@ -520,32 +522,37 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, return last; } -unsigned long get_wchan(struct task_struct *p) +struct wchan_info { + unsigned long pc; + int count; +}; + +static bool get_wchan_cb(void *arg, unsigned long pc) { - struct stackframe frame; - unsigned long stack_page, ret = 0; - int count = 0; - if (!p || p == current || task_is_running(p)) + struct wchan_info *wchan_info = arg; + + if (!in_sched_functions(pc)) { + wchan_info->pc = pc; + return false; + } + return wchan_info->count++ < 16; +} + +unsigned long __get_wchan(struct task_struct *p) +{ + struct wchan_info wchan_info = { + .pc = 0, + .count = 0, + }; + + if (!try_get_task_stack(p)) return 0; - stack_page = (unsigned long)try_get_task_stack(p); - if (!stack_page) - return 0; + arch_stack_walk(get_wchan_cb, &wchan_info, p, NULL); - start_backtrace(&frame, thread_saved_fp(p), thread_saved_pc(p)); - - do { - if (unwind_frame(p, &frame)) - goto out; - if (!in_sched_functions(frame.pc)) { - ret = frame.pc; - goto out; - } - } while (count++ < 16); - -out: put_task_stack(p); - return ret; + + return wchan_info.pc; } unsigned long arch_align_stack(unsigned long sp) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index e26196a33c..39dbdfdc38 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -725,10 +725,10 @@ static void sve_init_header_from_task(struct user_sve_header *header, if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT)) header->flags |= SVE_PT_VL_INHERIT; - header->vl = target->thread.sve_vl; + header->vl = task_get_sve_vl(target); vq = sve_vq_from_vl(header->vl); - header->max_vl = sve_max_vl; + header->max_vl = sve_max_vl(); header->size = SVE_PT_SIZE(vq, header->flags); header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), SVE_PT_REGS_SVE); @@ -812,15 +812,15 @@ static int sve_set(struct task_struct *target, /* * Apart from SVE_PT_REGS_MASK, all SVE_PT_* flags are consumed by - * sve_set_vector_length(), which will also validate them for us: + * vec_set_vector_length(), which will also validate them for us: */ - ret = sve_set_vector_length(target, header.vl, + ret = vec_set_vector_length(target, ARM64_VEC_SVE, header.vl, ((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16); if (ret) goto out; /* Actual VL set may be less than the user asked for: */ - vq = sve_vq_from_vl(target->thread.sve_vl); + vq = sve_vq_from_vl(task_get_sve_vl(target)); /* Registers: FPSIMD-only case */ @@ -1839,7 +1839,7 @@ static void tracehook_report_syscall(struct pt_regs *regs, int syscall_trace_enter(struct pt_regs *regs) { - unsigned long flags = READ_ONCE(current_thread_info()->flags); + unsigned long flags = read_thread_flags(); if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) { tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER); @@ -1862,7 +1862,7 @@ int syscall_trace_enter(struct pt_regs *regs) void syscall_trace_exit(struct pt_regs *regs) { - unsigned long flags = READ_ONCE(current_thread_info()->flags); + unsigned long flags = read_thread_flags(); audit_syscall_exit(regs); diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S index b78ea5de97..f0a3df9e18 100644 --- a/arch/arm64/kernel/relocate_kernel.S +++ b/arch/arm64/kernel/relocate_kernel.S @@ -4,6 +4,8 @@ * * Copyright (C) Linaro. * Copyright (C) Huawei Futurewei Technologies. + * Copyright (C) 2021, Microsoft Corporation. + * Pasha Tatashin */ #include @@ -13,7 +15,16 @@ #include #include #include +#include +.macro turn_off_mmu tmp1, tmp2 + mov_q \tmp1, INIT_SCTLR_EL1_MMU_OFF + pre_disable_mmu_workaround + msr sctlr_el1, \tmp1 + isb +.endm + +.section ".kexec_relocate.text", "ax" /* * arm64_relocate_new_kernel - Put a 2nd stage image in place and boot it. * @@ -27,33 +38,24 @@ */ SYM_CODE_START(arm64_relocate_new_kernel) /* Setup the list loop variables. */ - mov x18, x2 /* x18 = dtb address */ - mov x17, x1 /* x17 = kimage_start */ - mov x16, x0 /* x16 = kimage_head */ - mov x14, xzr /* x14 = entry ptr */ - mov x13, xzr /* x13 = copy dest */ - /* Check if the new image needs relocation. */ - tbnz x16, IND_DONE_BIT, .Ldone + ldr x18, [x0, #KIMAGE_ARCH_ZERO_PAGE] /* x18 = zero page for BBM */ + ldr x17, [x0, #KIMAGE_ARCH_TTBR1] /* x17 = linear map copy */ + ldr x16, [x0, #KIMAGE_HEAD] /* x16 = kimage_head */ + ldr x22, [x0, #KIMAGE_ARCH_PHYS_OFFSET] /* x22 phys_offset */ raw_dcache_line_size x15, x1 /* x15 = dcache line size */ + break_before_make_ttbr_switch x18, x17, x1, x2 /* set linear map */ .Lloop: and x12, x16, PAGE_MASK /* x12 = addr */ - + sub x12, x12, x22 /* Convert x12 to virt */ /* Test the entry flags. */ .Ltest_source: tbz x16, IND_SOURCE_BIT, .Ltest_indirection /* Invalidate dest page to PoC. */ - mov x2, x13 - add x20, x2, #PAGE_SIZE - sub x1, x15, #1 - bic x2, x2, x1 -2: dc ivac, x2 - add x2, x2, x15 - cmp x2, x20 - b.lo 2b - dsb sy - + mov x19, x13 copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8 + add x1, x19, #PAGE_SIZE + dcache_by_myline_op civac, sy, x19, x1, x15, x20 b .Lnext .Ltest_indirection: tbz x16, IND_INDIRECTION_BIT, .Ltest_destination @@ -65,31 +67,26 @@ SYM_CODE_START(arm64_relocate_new_kernel) .Lnext: ldr x16, [x14], #8 /* entry = *ptr++ */ tbz x16, IND_DONE_BIT, .Lloop /* while (!(entry & DONE)) */ -.Ldone: /* wait for writes from copy_page to finish */ dsb nsh ic iallu dsb nsh isb + ldr x4, [x0, #KIMAGE_START] /* relocation start */ + ldr x1, [x0, #KIMAGE_ARCH_EL2_VECTORS] /* relocation start */ + ldr x0, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */ + turn_off_mmu x12, x13 /* Start new image. */ - mov x0, x18 - mov x1, xzr + cbz x1, .Lel1 + mov x1, x4 /* relocation start */ + mov x2, x0 /* dtb address */ + mov x3, xzr + mov x4, xzr + mov x0, #HVC_SOFT_RESTART + hvc #0 /* Jumps from el2 */ +.Lel1: mov x2, xzr mov x3, xzr - br x17 - + br x4 /* Jumps from el1 */ SYM_CODE_END(arm64_relocate_new_kernel) - -.align 3 /* To keep the 64-bit values below naturally aligned. */ - -.Lcopy_end: -.org KEXEC_CONTROL_PAGE_SIZE - -/* - * arm64_relocate_new_kernel_size - Number of bytes to copy to the - * control_code_page. - */ -.globl arm64_relocate_new_kernel_size -arm64_relocate_new_kernel_size: - .quad .Lcopy_end - arm64_relocate_new_kernel diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c index a6d1875565..68330017d0 100644 --- a/arch/arm64/kernel/return_address.c +++ b/arch/arm64/kernel/return_address.c @@ -9,9 +9,9 @@ #include #include #include +#include #include -#include struct return_address_data { unsigned int level; @@ -35,15 +35,11 @@ NOKPROBE_SYMBOL(save_return_addr); void *return_address(unsigned int level) { struct return_address_data data; - struct stackframe frame; data.level = level + 2; data.addr = NULL; - start_backtrace(&frame, - (unsigned long)__builtin_frame_address(0), - (unsigned long)return_address); - walk_stackframe(current, &frame, save_return_addr, &data); + arch_stack_walk(save_return_addr, &data, current, NULL); if (!data.level) return data.addr; diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c index 47f77d1234..d20620a1c5 100644 --- a/arch/arm64/kernel/sdei.c +++ b/arch/arm64/kernel/sdei.c @@ -202,7 +202,7 @@ unsigned long sdei_arch_get_entry_point(int conduit) * dropped to EL1 because we don't support VHE, then we can't support * SDEI. */ - if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) { + if (is_hyp_nvhe()) { pr_err("Not supported on this hardware/boot configuration\n"); goto out_err; } diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index be5f85b0a2..f70573928f 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -189,11 +189,16 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys) if (!dt_virt || !early_init_dt_scan(dt_virt)) { pr_crit("\n" - "Error: invalid device tree blob at physical address %pa (virtual address 0x%p)\n" + "Error: invalid device tree blob at physical address %pa (virtual address 0x%px)\n" "The dtb must be 8-byte aligned and must not exceed 2 MB in size\n" "\nPlease check your bootloader.", &dt_phys, dt_virt); + /* + * Note that in this _really_ early stage we cannot even BUG() + * or oops, so the least terrible thing to do is cpu_relax(), + * or else we could end-up printing non-initialized data, etc. + */ while (true) cpu_relax(); } @@ -232,12 +237,14 @@ static void __init request_standard_resources(void) if (memblock_is_nomap(region)) { res->name = "reserved"; res->flags = IORESOURCE_MEM; + res->start = __pfn_to_phys(memblock_region_reserved_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_reserved_end_pfn(region)) - 1; } else { res->name = "System RAM"; res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; } - res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); - res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; request_resource(&iomem_resource, res); diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index c287b9407f..d8aaf4b6f4 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -227,7 +227,7 @@ static int preserve_sve_context(struct sve_context __user *ctx) { int err = 0; u16 reserved[ARRAY_SIZE(ctx->__reserved)]; - unsigned int vl = current->thread.sve_vl; + unsigned int vl = task_get_sve_vl(current); unsigned int vq = 0; if (test_thread_flag(TIF_SVE)) @@ -266,7 +266,7 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) if (__copy_from_user(&sve, user->sve, sizeof(sve))) return -EFAULT; - if (sve.vl != current->thread.sve_vl) + if (sve.vl != task_get_sve_vl(current)) return -EINVAL; if (sve.head.size <= sizeof(*user->sve)) { @@ -594,10 +594,10 @@ static int setup_sigframe_layout(struct rt_sigframe_user_layout *user, unsigned int vq = 0; if (add_all || test_thread_flag(TIF_SVE)) { - int vl = sve_max_vl; + int vl = sve_max_vl(); if (!add_all) - vl = current->thread.sve_vl; + vl = task_get_sve_vl(current); vq = sve_vq_from_vl(vl); } @@ -948,7 +948,7 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) } local_daif_mask(); - thread_flags = READ_ONCE(current_thread_info()->flags); + thread_flags = read_thread_flags(); } while (thread_flags & _TIF_WORK_MASK); } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 6f6ff072ac..27df5c1e6b 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -466,33 +466,6 @@ void __init smp_prepare_boot_cpu(void) kasan_init_hw_tags(); } -static u64 __init of_get_cpu_mpidr(struct device_node *dn) -{ - const __be32 *cell; - u64 hwid; - - /* - * A cpu node with missing "reg" property is - * considered invalid to build a cpu_logical_map - * entry. - */ - cell = of_get_property(dn, "reg", NULL); - if (!cell) { - pr_err("%pOF: missing reg property\n", dn); - return INVALID_HWID; - } - - hwid = of_read_number(cell, of_n_addr_cells(dn)); - /* - * Non affinity bits must be set to 0 in the DT - */ - if (hwid & ~MPIDR_HWID_BITMASK) { - pr_err("%pOF: invalid reg property\n", dn); - return INVALID_HWID; - } - return hwid; -} - /* * Duplicate MPIDRs are a recipe for disaster. Scan all initialized * entries and check for duplicates. If any is found just ignore the @@ -656,9 +629,9 @@ static void __init of_parse_and_init_cpus(void) struct device_node *dn; for_each_of_cpu_node(dn) { - u64 hwid = of_get_cpu_mpidr(dn); + u64 hwid = of_get_cpu_hwid(dn, 0); - if (hwid == INVALID_HWID) + if (hwid & ~MPIDR_HWID_BITMASK) goto next; if (is_mpidr_duplicate(cpu_count, hwid)) { @@ -1128,5 +1101,6 @@ bool cpus_are_stuck_in_kernel(void) { bool smp_spin_tables = (num_possible_cpus() > 1 && !have_cpu_die()); - return !!cpus_stuck_in_kernel || smp_spin_tables; + return !!cpus_stuck_in_kernel || smp_spin_tables || + is_protected_kvm_enabled(); } diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index 3b8dc538a4..e4103e0856 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -33,13 +33,13 @@ */ -notrace void start_backtrace(struct stackframe *frame, unsigned long fp, - unsigned long pc) +static notrace void start_backtrace(struct stackframe *frame, unsigned long fp, + unsigned long pc) { frame->fp = fp; frame->pc = pc; -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - frame->graph = 0; +#ifdef CONFIG_KRETPROBES + frame->kr_cur = NULL; #endif /* @@ -64,7 +64,8 @@ NOKPROBE_SYMBOL(start_backtrace); * records (e.g. a cycle), determined based on the location and fp value of A * and the location (but not the fp value) of B. */ -int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) +static int notrace unwind_frame(struct task_struct *tsk, + struct stackframe *frame) { unsigned long fp = frame->fp; struct stack_info info; @@ -114,31 +115,37 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) frame->prev_fp = fp; frame->prev_type = info.type; + frame->pc = ptrauth_strip_insn_pac(frame->pc); + #ifdef CONFIG_FUNCTION_GRAPH_TRACER if (tsk->ret_stack && - (ptrauth_strip_insn_pac(frame->pc) == (unsigned long)return_to_handler)) { - struct ftrace_ret_stack *ret_stack; + (frame->pc == (unsigned long)return_to_handler)) { + unsigned long orig_pc; /* * This is a case where function graph tracer has * modified a return address (LR) in a stack frame * to hook a function return. * So replace it to an original value. */ - ret_stack = ftrace_graph_get_ret_stack(tsk, frame->graph++); - if (WARN_ON_ONCE(!ret_stack)) + orig_pc = ftrace_graph_ret_addr(tsk, NULL, frame->pc, + (void *)frame->fp); + if (WARN_ON_ONCE(frame->pc == orig_pc)) return -EINVAL; - frame->pc = ret_stack->ret; + frame->pc = orig_pc; } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ - - frame->pc = ptrauth_strip_insn_pac(frame->pc); +#ifdef CONFIG_KRETPROBES + if (is_kretprobe_trampoline(frame->pc)) + frame->pc = kretprobe_find_ret_addr(tsk, (void *)frame->fp, &frame->kr_cur); +#endif return 0; } NOKPROBE_SYMBOL(unwind_frame); -void notrace walk_stackframe(struct task_struct *tsk, struct stackframe *frame, - bool (*fn)(void *, unsigned long), void *data) +static void notrace walk_stackframe(struct task_struct *tsk, + struct stackframe *frame, + bool (*fn)(void *, unsigned long), void *data) { while (1) { int ret; @@ -152,24 +159,20 @@ void notrace walk_stackframe(struct task_struct *tsk, struct stackframe *frame, } NOKPROBE_SYMBOL(walk_stackframe); -static void dump_backtrace_entry(unsigned long where, const char *loglvl) +static bool dump_backtrace_entry(void *arg, unsigned long where) { + char *loglvl = arg; printk("%s %pSb\n", loglvl, (void *)where); + return true; } void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, const char *loglvl) { - struct stackframe frame; - int skip = 0; - pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); - if (regs) { - if (user_mode(regs)) - return; - skip = 1; - } + if (regs && user_mode(regs)) + return; if (!tsk) tsk = current; @@ -177,36 +180,8 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, if (!try_get_task_stack(tsk)) return; - if (tsk == current) { - start_backtrace(&frame, - (unsigned long)__builtin_frame_address(0), - (unsigned long)dump_backtrace); - } else { - /* - * task blocked in __switch_to - */ - start_backtrace(&frame, - thread_saved_fp(tsk), - thread_saved_pc(tsk)); - } - printk("%sCall trace:\n", loglvl); - do { - /* skip until specified stack frame */ - if (!skip) { - dump_backtrace_entry(frame.pc, loglvl); - } else if (frame.fp == regs->regs[29]) { - skip = 0; - /* - * Mostly, this is the case where this function is - * called in panic/abort. As exception handler's - * stack frame does not contain the corresponding pc - * at which an exception has taken place, use regs->pc - * instead. - */ - dump_backtrace_entry(regs->pc, loglvl); - } - } while (!unwind_frame(tsk, &frame)); + arch_stack_walk(dump_backtrace_entry, (void *)loglvl, tsk, regs); put_task_stack(tsk); } @@ -217,8 +192,6 @@ void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) barrier(); } -#ifdef CONFIG_STACKTRACE - noinline notrace void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, struct task_struct *task, struct pt_regs *regs) @@ -237,5 +210,3 @@ noinline notrace void arch_stack_walk(stack_trace_consume_fn consume_entry, walk_stackframe(task, &frame, consume_entry, cookie); } - -#endif diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index 50a0f1a38e..c938603b3b 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -81,7 +81,7 @@ void syscall_trace_exit(struct pt_regs *regs); static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, const syscall_fn_t syscall_table[]) { - unsigned long flags = current_thread_info()->flags; + unsigned long flags = read_thread_flags(); regs->orig_x0 = regs->regs[0]; regs->syscallno = scno; @@ -148,7 +148,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, */ if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) { local_daif_mask(); - flags = current_thread_info()->flags; + flags = read_thread_flags(); if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP)) return; local_daif_restore(DAIF_PROCCTX); diff --git a/arch/arm64/kernel/time.c b/arch/arm64/kernel/time.c index eebbc8d712..b5855eb743 100644 --- a/arch/arm64/kernel/time.c +++ b/arch/arm64/kernel/time.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -29,25 +30,25 @@ #include #include -#include #include +static bool profile_pc_cb(void *arg, unsigned long pc) +{ + unsigned long *prof_pc = arg; + + if (in_lock_functions(pc)) + return true; + *prof_pc = pc; + return false; +} + unsigned long profile_pc(struct pt_regs *regs) { - struct stackframe frame; + unsigned long prof_pc = 0; - if (!in_lock_functions(regs->pc)) - return regs->pc; + arch_stack_walk(profile_pc_cb, &prof_pc, current, regs); - start_backtrace(&frame, regs->regs[29], regs->pc); - - do { - int ret = unwind_frame(NULL, &frame); - if (ret < 0) - return 0; - } while (in_lock_functions(frame.pc)); - - return frame.pc; + return prof_pc; } EXPORT_SYMBOL(profile_pc); diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 4dd14a6620..9ab78ad826 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -103,6 +103,8 @@ int __init parse_acpi_topology(void) cpu_topology[cpu].thread_id = -1; cpu_topology[cpu].core_id = topology_id; } + topology_id = find_acpi_cpu_topology_cluster(cpu); + cpu_topology[cpu].cluster_id = topology_id; topology_id = find_acpi_cpu_topology_package(cpu); cpu_topology[cpu].package_id = topology_id; diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index fe0cd05688..70fc42470f 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -235,7 +235,7 @@ void die(const char *str, struct pt_regs *regs, int err) raw_spin_unlock_irqrestore(&die_lock, flags); if (ret != NOTIFY_STOP) - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } static void arm64_show_signal(int signo, const char *str) @@ -400,11 +400,11 @@ static int call_undef_hook(struct pt_regs *regs) unsigned long flags; u32 instr; int (*fn)(struct pt_regs *regs, u32 instr) = NULL; - void __user *pc = (void __user *)instruction_pointer(regs); + unsigned long pc = instruction_pointer(regs); if (!user_mode(regs)) { __le32 instr_le; - if (get_kernel_nofault(instr_le, (__force __le32 *)pc)) + if (get_kernel_nofault(instr_le, (__le32 *)pc)) goto exit; instr = le32_to_cpu(instr_le); } else if (compat_thumb_mode(regs)) { @@ -527,14 +527,9 @@ NOKPROBE_SYMBOL(do_ptrauth_fault); "1: " insn ", %1\n" \ " mov %w0, #0\n" \ "2:\n" \ - " .pushsection .fixup,\"ax\"\n" \ - " .align 2\n" \ - "3: mov %w0, %w2\n" \ - " b 2b\n" \ - " .popsection\n" \ - _ASM_EXTABLE(1b, 3b) \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \ : "=r" (res) \ - : "r" (address), "i" (-EFAULT)); \ + : "r" (address)); \ uaccess_ttbr0_disable(); \ } @@ -653,6 +648,12 @@ static const struct sys64_hook sys64_hooks[] = { .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT, .handler = cntvct_read_handler, }, + { + /* Trap read access to CNTVCTSS_EL0 */ + .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, + .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCTSS, + .handler = cntvct_read_handler, + }, { /* Trap read access to CNTFRQ_EL0 */ .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, @@ -729,6 +730,11 @@ static const struct sys64_hook cp15_64_hooks[] = { .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT, .handler = compat_cntvct_read_handler, }, + { + .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK, + .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCTSS, + .handler = compat_cntvct_read_handler, + }, {}, }; diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index 945e6bb326..172452f79e 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -23,15 +23,19 @@ btildflags-$(CONFIG_ARM64_BTI_KERNEL) += -z force-bti # potential future proofing if we end up with internal calls to the exported # routines, as x86 does (see 6f121e548f83 ("x86, vdso: Reimplement vdso.so # preparation in build-time C")). -ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \ +ldflags-y := -shared -soname=linux-vdso.so.1 --hash-style=sysv \ -Bsymbolic --build-id=sha1 -n $(btildflags-y) -T ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18 ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO +# -Wmissing-prototypes and -Wmissing-declarations are removed from +# the CFLAGS of vgettimeofday.c to make possible to build the +# kernel with CONFIG_WERROR enabled. CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS) \ - $(CC_FLAGS_LTO) + $(CC_FLAGS_LTO) -Wmissing-prototypes -Wmissing-declarations KASAN_SANITIZE := n +KCSAN_SANITIZE := n UBSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y KCOV_INSTRUMENT := n diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile index 3514269ac7..6c01b63ff5 100644 --- a/arch/arm64/kernel/vdso32/Makefile +++ b/arch/arm64/kernel/vdso32/Makefile @@ -26,8 +26,6 @@ cc32-option = $(call try-run,\ $(CC_COMPAT) $(1) -c -x c /dev/null -o "$$TMP",$(1),$(2)) cc32-disable-warning = $(call try-run,\ $(CC_COMPAT) -W$(strip $(1)) -c -x c /dev/null -o "$$TMP",-Wno-$(strip $(1))) -cc32-as-instr = $(call try-run,\ - printf "%b\n" "$(1)" | $(CC_COMPAT) $(VDSO_AFLAGS) -c -x assembler -o "$$TMP" -,$(2),$(3)) # We cannot use the global flags to compile the vDSO files, the main reason # being that the 32-bit compiler may be older than the main (64-bit) compiler @@ -61,13 +59,7 @@ endif # From arm vDSO Makefile VDSO_CAFLAGS += -fPIC -fno-builtin -fno-stack-protector VDSO_CAFLAGS += -DDISABLE_BRANCH_PROFILING - - -# Try to compile for ARMv8. If the compiler is too old and doesn't support it, -# fall back to v7. There is no easy way to check for what architecture the code -# is being compiled, so define a macro specifying that (see arch/arm/Makefile). -VDSO_CAFLAGS += $(call cc32-option,-march=armv8-a -D__LINUX_ARM_ARCH__=8,\ - -march=armv7-a -D__LINUX_ARM_ARCH__=7) +VDSO_CAFLAGS += -march=armv8-a VDSO_CFLAGS := $(VDSO_CAFLAGS) VDSO_CFLAGS += -DENABLE_COMPAT_VDSO=1 @@ -107,16 +99,10 @@ endif VDSO_AFLAGS := $(VDSO_CAFLAGS) VDSO_AFLAGS += -D__ASSEMBLY__ -# Check for binutils support for dmb ishld -dmbinstr := $(call cc32-as-instr,dmb ishld,-DCONFIG_AS_DMB_ISHLD=1) - -VDSO_CFLAGS += $(dmbinstr) -VDSO_AFLAGS += $(dmbinstr) - # From arm vDSO Makefile VDSO_LDFLAGS += -Bsymbolic --no-undefined -soname=linux-vdso.so.1 VDSO_LDFLAGS += -z max-page-size=4096 -z common-page-size=4096 -VDSO_LDFLAGS += -nostdlib -shared --hash-style=sysv --build-id=sha1 +VDSO_LDFLAGS += -shared --hash-style=sysv --build-id=sha1 # Borrow vdsomunge.c from the arm vDSO diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 184abd7c42..edaf0faf76 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -57,12 +57,13 @@ #define SBSS_ALIGN 0 #endif -#define RO_EXCEPTION_TABLE_ALIGN 8 +#define RO_EXCEPTION_TABLE_ALIGN 4 #define RUNTIME_DISCARD_EXIT #include #include #include +#include #include #include @@ -100,6 +101,16 @@ jiffies = jiffies_64; #define HIBERNATE_TEXT #endif +#ifdef CONFIG_KEXEC_CORE +#define KEXEC_TEXT \ + . = ALIGN(SZ_4K); \ + __relocate_new_kernel_start = .; \ + *(.kexec_relocate.text) \ + __relocate_new_kernel_end = .; +#else +#define KEXEC_TEXT +#endif + #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 #define TRAMP_TEXT \ . = ALIGN(PAGE_SIZE); \ @@ -160,8 +171,8 @@ SECTIONS HYPERVISOR_TEXT IDMAP_TEXT HIBERNATE_TEXT + KEXEC_TEXT TRAMP_TEXT - *(.fixup) *(.gnu.warning) . = ALIGN(16); *(.got) /* Global offset table */ @@ -348,3 +359,10 @@ ASSERT(swapper_pg_dir - reserved_pg_dir == RESERVED_SWAPPER_OFFSET, ASSERT(swapper_pg_dir - tramp_pg_dir == TRAMP_SWAPPER_OFFSET, "TRAMP_SWAPPER_OFFSET is wrong!") #endif + +#ifdef CONFIG_KEXEC_CORE +/* kexec relocation code should fit into one KEXEC_CONTROL_PAGE_SIZE */ +ASSERT(__relocate_new_kernel_end - (__relocate_new_kernel_start & ~(SZ_4K - 1)) + <= SZ_4K, "kexec relocation code is too big or misaligned") +ASSERT(KEXEC_CONTROL_PAGE_SIZE >= SZ_4K, "KEXEC_CONTROL_PAGE_SIZE is broken") +#endif diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index d7eec0b437..8a5fbbf084 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -4,6 +4,7 @@ # source "virt/lib/Kconfig" +source "virt/kvm/Kconfig" menuconfig VIRTUALIZATION bool "Virtualization" @@ -19,7 +20,7 @@ if VIRTUALIZATION menuconfig KVM bool "Kernel-based Virtual Machine (KVM) support" - depends on OF + depends on HAVE_KVM select MMU_NOTIFIER select PREEMPT_NOTIFIERS select HAVE_KVM_CPU_RELAX_INTERCEPT @@ -38,17 +39,16 @@ menuconfig KVM select HAVE_KVM_IRQ_BYPASS select HAVE_KVM_VCPU_RUN_PID_CHANGE select SCHED_INFO + select GUEST_PERF_EVENTS if PERF_EVENTS + select INTERVAL_TREE help Support hosting virtualized guest machines. If unsure, say N. -if KVM - -source "virt/kvm/Kconfig" - config NVHE_EL2_DEBUG bool "Debug mode for non-VHE EL2 object" + depends on KVM help Say Y here to enable the debug mode for the non-VHE KVM EL2 object. Failure reports will BUG() in the hypervisor. This is intended for @@ -56,6 +56,4 @@ config NVHE_EL2_DEBUG If unsure, say N. -endif # KVM - endif # VIRTUALIZATION diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 989bb5dad2..91861fd8b8 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -5,17 +5,15 @@ ccflags-y += -I $(srctree)/$(src) -KVM=../../../virt/kvm +include $(srctree)/virt/kvm/Makefile.kvm obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM) += hyp/ -kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \ - $(KVM)/vfio.o $(KVM)/irqchip.o $(KVM)/binary_stats.o \ - arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \ +kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \ inject_fault.o va_layout.o handle_exit.o \ guest.o debug.o reset.o sys_regs.o \ - vgic-sys-reg-v3.o fpsimd.o pmu.o \ + vgic-sys-reg-v3.o fpsimd.o pmu.o pkvm.o \ arch_timer.o trng.o\ vgic/vgic.o vgic/vgic-init.o \ vgic/vgic-irqfd.o vgic/vgic-v2.o \ @@ -25,3 +23,19 @@ kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \ vgic/vgic-its.o vgic/vgic-debug.o kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o + +always-y := hyp_constants.h hyp-constants.s + +define rule_gen_hyp_constants + $(call filechk,offsets,__HYP_CONSTANTS_H__) +endef + +CFLAGS_hyp-constants.o = -I $(srctree)/$(src)/hyp/include +$(obj)/hyp-constants.s: $(src)/hyp/hyp-constants.c FORCE + $(call if_changed_dep,cc_s_c) + +$(obj)/hyp_constants.h: $(obj)/hyp-constants.s FORCE + $(call if_changed_rule,gen_hyp_constants) + +obj-kvm := $(addprefix $(obj)/, $(kvm-y)) +$(obj-kvm): $(obj)/hyp_constants.h diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 3df67c1274..6e542e2eae 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -467,7 +467,7 @@ static void timer_save_state(struct arch_timer_context *ctx) } /* - * Schedule the background timer before calling kvm_vcpu_block, so that this + * Schedule the background timer before calling kvm_vcpu_halt, so that this * thread is removed from its waitqueue and made runnable when there's a timer * interrupt to handle. */ @@ -649,7 +649,6 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) { struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct timer_map map; - struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); if (unlikely(!timer->enabled)) return; @@ -672,7 +671,7 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) if (map.emul_ptimer) soft_timer_cancel(&map.emul_ptimer->hrtimer); - if (rcuwait_active(wait)) + if (kvm_vcpu_is_blocking(vcpu)) kvm_timer_blocking(vcpu); /* @@ -750,7 +749,7 @@ int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu) /* Make the updates of cntvoff for all vtimer contexts atomic */ static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff) { - int i; + unsigned long i; struct kvm *kvm = vcpu->kvm; struct kvm_vcpu *tmp; @@ -1189,8 +1188,8 @@ void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu) static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu) { - int vtimer_irq, ptimer_irq; - int i, ret; + int vtimer_irq, ptimer_irq, ret; + unsigned long i; vtimer_irq = vcpu_vtimer(vcpu)->irq.irq; ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu)); @@ -1297,7 +1296,7 @@ void kvm_timer_init_vhe(void) static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq) { struct kvm_vcpu *vcpu; - int i; + unsigned long i; kvm_for_each_vcpu(i, vcpu, kvm) { vcpu_vtimer(vcpu)->irq.irq = vtimer_irq; diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 0b2f684cd8..4dca6ffd03 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -146,7 +146,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (ret) return ret; - ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP); + ret = kvm_share_hyp(kvm, kvm + 1); if (ret) goto out_free_stage2_pgd; @@ -175,19 +175,13 @@ vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) */ void kvm_arch_destroy_vm(struct kvm *kvm) { - int i; - bitmap_free(kvm->arch.pmu_filter); kvm_vgic_destroy(kvm); - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - if (kvm->vcpus[i]) { - kvm_vcpu_destroy(kvm->vcpus[i]); - kvm->vcpus[i] = NULL; - } - } - atomic_set(&kvm->online_vcpus, 0); + kvm_destroy_vcpus(kvm); + + kvm_unshare_hyp(kvm, kvm + 1); } int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) @@ -223,7 +217,14 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = 1; break; case KVM_CAP_NR_VCPUS: - r = num_online_cpus(); + /* + * ARM64 treats KVM_CAP_NR_CPUS differently from all other + * architectures, as it does not always bound it to + * KVM_CAP_MAX_VCPUS. It should not matter much because + * this is just an advisory value. + */ + r = min_t(unsigned int, num_online_cpus(), + kvm_arm_default_max_vcpus()); break; case KVM_CAP_MAX_VCPUS: case KVM_CAP_MAX_VCPU_ID: @@ -291,18 +292,12 @@ long kvm_arch_dev_ioctl(struct file *filp, struct kvm *kvm_arch_alloc_vm(void) { - if (!has_vhe()) - return kzalloc(sizeof(struct kvm), GFP_KERNEL); + size_t sz = sizeof(struct kvm); - return vzalloc(sizeof(struct kvm)); -} - -void kvm_arch_free_vm(struct kvm *kvm) -{ if (!has_vhe()) - kfree(kvm); - else - vfree(kvm); + return kzalloc(sz, GFP_KERNEL_ACCOUNT); + + return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO); } int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) @@ -341,7 +336,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) if (err) return err; - return create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP); + return kvm_share_hyp(vcpu, vcpu + 1); } void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) @@ -350,7 +345,7 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) { - if (vcpu->arch.has_run_once && unlikely(!irqchip_in_kernel(vcpu->kvm))) + if (vcpu_has_run_once(vcpu) && unlikely(!irqchip_in_kernel(vcpu->kvm))) static_branch_dec(&userspace_irqchip_in_use); kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); @@ -367,27 +362,12 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) { - /* - * If we're about to block (most likely because we've just hit a - * WFI), we need to sync back the state of the GIC CPU interface - * so that we have the latest PMR and group enables. This ensures - * that kvm_arch_vcpu_runnable has up-to-date data to decide - * whether we have pending interrupts. - * - * For the same reason, we want to tell GICv4 that we need - * doorbells to be signalled, should an interrupt become pending. - */ - preempt_disable(); - kvm_vgic_vmcr_sync(vcpu); - vgic_v4_put(vcpu, true); - preempt_enable(); + } void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) { - preempt_disable(); - vgic_v4_load(vcpu); - preempt_enable(); + } void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) @@ -502,6 +482,13 @@ bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) return vcpu_mode_priv(vcpu); } +#ifdef CONFIG_GUEST_PERF_EVENTS +unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) +{ + return *vcpu_pc(vcpu); +} +#endif + /* Just ensure a guest exit from a particular CPU */ static void exit_vm_noop(void *info) { @@ -583,18 +570,33 @@ static void update_vmid(struct kvm_vmid *vmid) spin_unlock(&kvm_vmid_lock); } -static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) +static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.target >= 0; +} + +/* + * Handle both the initialisation that is being done when the vcpu is + * run for the first time, as well as the updates that must be + * performed each time we get a new thread dealing with this vcpu. + */ +int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) { struct kvm *kvm = vcpu->kvm; - int ret = 0; + int ret; - if (likely(vcpu->arch.has_run_once)) - return 0; + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; if (!kvm_arm_vcpu_is_finalized(vcpu)) return -EPERM; - vcpu->arch.has_run_once = true; + ret = kvm_arch_vcpu_run_map_fp(vcpu); + if (ret) + return ret; + + if (likely(vcpu_has_run_once(vcpu))) + return 0; kvm_arm_vcpu_init_debug(vcpu); @@ -606,12 +608,6 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) ret = kvm_vgic_map_resources(kvm); if (ret) return ret; - } else { - /* - * Tell the rest of the code that there are userspace irqchip - * VMs in the wild. - */ - static_branch_inc(&userspace_irqchip_in_use); } ret = kvm_timer_enable(vcpu); @@ -619,6 +615,24 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) return ret; ret = kvm_arm_pmu_v3_enable(vcpu); + if (ret) + return ret; + + if (!irqchip_in_kernel(kvm)) { + /* + * Tell the rest of the code that there are userspace irqchip + * VMs in the wild. + */ + static_branch_inc(&userspace_irqchip_in_use); + } + + /* + * Initialize traps for protected VMs. + * NOTE: Move to run in EL2 directly, rather than via a hypercall, once + * the code is in place for first run initialization at EL2. + */ + if (kvm_vm_is_protected(kvm)) + kvm_call_hyp_nvhe(__pkvm_vcpu_init_traps, vcpu); return ret; } @@ -630,7 +644,7 @@ bool kvm_arch_intc_initialized(struct kvm *kvm) void kvm_arm_halt_guest(struct kvm *kvm) { - int i; + unsigned long i; struct kvm_vcpu *vcpu; kvm_for_each_vcpu(i, vcpu, kvm) @@ -640,12 +654,12 @@ void kvm_arm_halt_guest(struct kvm *kvm) void kvm_arm_resume_guest(struct kvm *kvm) { - int i; + unsigned long i; struct kvm_vcpu *vcpu; kvm_for_each_vcpu(i, vcpu, kvm) { vcpu->arch.pause = false; - rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu)); + __kvm_vcpu_wake_up(vcpu); } } @@ -670,9 +684,37 @@ static void vcpu_req_sleep(struct kvm_vcpu *vcpu) smp_rmb(); } -static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu) +/** + * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior + * @vcpu: The VCPU pointer + * + * Suspend execution of a vCPU until a valid wake event is detected, i.e. until + * the vCPU is runnable. The vCPU may or may not be scheduled out, depending + * on when a wake event arrives, e.g. there may already be a pending wake event. + */ +void kvm_vcpu_wfi(struct kvm_vcpu *vcpu) { - return vcpu->arch.target >= 0; + /* + * Sync back the state of the GIC CPU interface so that we have + * the latest PMR and group enables. This ensures that + * kvm_arch_vcpu_runnable has up-to-date data to decide whether + * we have pending interrupts, e.g. when determining if the + * vCPU should block. + * + * For the same reason, we want to tell GICv4 that we need + * doorbells to be signalled, should an interrupt become pending. + */ + preempt_disable(); + kvm_vgic_vmcr_sync(vcpu); + vgic_v4_put(vcpu, true); + preempt_enable(); + + kvm_vcpu_halt(vcpu); + kvm_clear_request(KVM_REQ_UNHALT, vcpu); + + preempt_disable(); + vgic_v4_load(vcpu); + preempt_enable(); } static void check_vcpu_requests(struct kvm_vcpu *vcpu) @@ -788,13 +830,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) struct kvm_run *run = vcpu->run; int ret; - if (unlikely(!kvm_vcpu_initialized(vcpu))) - return -ENOEXEC; - - ret = kvm_vcpu_first_run_init(vcpu); - if (ret) - return ret; - if (run->exit_reason == KVM_EXIT_MMIO) { ret = kvm_handle_mmio_return(vcpu); if (ret) @@ -858,6 +893,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) } kvm_arm_setup_debug(vcpu); + kvm_arch_vcpu_ctxflush_fp(vcpu); /************************************************************** * Enter the guest @@ -1129,7 +1165,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, * need to invalidate the I-cache though, as FWB does *not* * imply CTR_EL0.DIC. */ - if (vcpu->arch.has_run_once) { + if (vcpu_has_run_once(vcpu)) { if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) stage2_unmap_vm(vcpu->kvm); else @@ -1402,12 +1438,9 @@ long kvm_arch_vm_ioctl(struct file *filp, return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr); } case KVM_ARM_PREFERRED_TARGET: { - int err; struct kvm_vcpu_init init; - err = kvm_vcpu_preferred_target(&init); - if (err) - return err; + kvm_vcpu_preferred_target(&init); if (copy_to_user(argp, &init, sizeof(init))) return -EFAULT; @@ -1591,25 +1624,33 @@ static void cpu_set_hyp_vector(void) kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot); } -static void cpu_hyp_reinit(void) +static void cpu_hyp_init_context(void) { kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt); - cpu_hyp_reset(); + if (!is_kernel_in_hyp_mode()) + cpu_init_hyp_mode(); +} + +static void cpu_hyp_init_features(void) +{ + cpu_set_hyp_vector(); + kvm_arm_init_debug(); if (is_kernel_in_hyp_mode()) kvm_timer_init_vhe(); - else - cpu_init_hyp_mode(); - - cpu_set_hyp_vector(); - - kvm_arm_init_debug(); if (vgic_present) kvm_vgic_init_cpu_hardware(); } +static void cpu_hyp_reinit(void) +{ + cpu_hyp_reset(); + cpu_hyp_init_context(); + cpu_hyp_init_features(); +} + static void _kvm_arch_hardware_enable(void *discard) { if (!__this_cpu_read(kvm_arm_hardware_enabled)) { @@ -1773,7 +1814,8 @@ static int init_subsystems(void) if (err) goto out; - kvm_perf_init(); + kvm_register_perf_callbacks(NULL); + kvm_sys_reg_table_init(); out: @@ -1800,10 +1842,17 @@ static int do_pkvm_init(u32 hyp_va_bits) int ret; preempt_disable(); - hyp_install_host_vector(); + cpu_hyp_init_context(); ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size, num_possible_cpus(), kern_hyp_va(per_cpu_base), hyp_va_bits); + cpu_hyp_init_features(); + + /* + * The stub hypercalls are now disabled, so set our local flag to + * prevent a later re-init attempt in kvm_arch_hardware_enable(). + */ + __this_cpu_write(kvm_arm_hardware_enabled, 1); preempt_enable(); return ret; @@ -1814,8 +1863,13 @@ static int kvm_hyp_init_protection(u32 hyp_va_bits) void *addr = phys_to_virt(hyp_mem_base); int ret; + kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); + kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1); + kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1); kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); + kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1); ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP); if (ret) @@ -2021,7 +2075,7 @@ static int finalize_hyp_mode(void) struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr) { struct kvm_vcpu *vcpu; - int i; + unsigned long i; mpidr &= MPIDR_HWID_BITMASK; kvm_for_each_vcpu(i, vcpu, kvm) { @@ -2084,6 +2138,11 @@ int kvm_arch_init(void *opaque) return -ENODEV; } + if (kvm_get_mode() == KVM_MODE_NONE) { + kvm_info("KVM disabled from command line\n"); + return -ENODEV; + } + in_hyp_mode = is_kernel_in_hyp_mode(); if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) || @@ -2144,7 +2203,7 @@ int kvm_arch_init(void *opaque) /* NOP: Compiling as a module not supported */ void kvm_arch_exit(void) { - kvm_perf_teardown(); + kvm_unregister_perf_callbacks(); } static int __init early_kvm_mode_cfg(char *arg) @@ -2157,8 +2216,15 @@ static int __init early_kvm_mode_cfg(char *arg) return 0; } - if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) + if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) { + kvm_mode = KVM_MODE_DEFAULT; return 0; + } + + if (strcmp(arg, "none") == 0) { + kvm_mode = KVM_MODE_NONE; + return 0; + } return -EINVAL; } diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index 5621020b28..2f48fd362a 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -7,7 +7,6 @@ */ #include #include -#include #include #include #include @@ -15,6 +14,19 @@ #include #include +void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu) +{ + struct task_struct *p = vcpu->arch.parent_task; + struct user_fpsimd_state *fpsimd; + + if (!is_protected_kvm_enabled() || !p) + return; + + fpsimd = &p->thread.uw.fpsimd_state; + kvm_unshare_hyp(fpsimd, fpsimd + 1); + put_task_struct(p); +} + /* * Called on entry to KVM_RUN unless this vcpu previously ran at least * once and the most recent prior KVM_RUN for this vcpu was called from @@ -28,36 +40,29 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) { int ret; - struct thread_info *ti = ¤t->thread_info; struct user_fpsimd_state *fpsimd = ¤t->thread.uw.fpsimd_state; + kvm_vcpu_unshare_task_fp(vcpu); + + /* Make sure the host task fpsimd state is visible to hyp: */ + ret = kvm_share_hyp(fpsimd, fpsimd + 1); + if (ret) + return ret; + + vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd); + /* - * Make sure the host task thread flags and fpsimd state are - * visible to hyp: + * We need to keep current's task_struct pinned until its data has been + * unshared with the hypervisor to make sure it is not re-used by the + * kernel and donated to someone else while already shared -- see + * kvm_vcpu_unshare_task_fp() for the matching put_task_struct(). */ - ret = create_hyp_mappings(ti, ti + 1, PAGE_HYP); - if (ret) - goto error; - - ret = create_hyp_mappings(fpsimd, fpsimd + 1, PAGE_HYP); - if (ret) - goto error; - - if (vcpu->arch.sve_state) { - void *sve_end; - - sve_end = vcpu->arch.sve_state + vcpu_sve_state_size(vcpu); - - ret = create_hyp_mappings(vcpu->arch.sve_state, sve_end, - PAGE_HYP); - if (ret) - goto error; + if (is_protected_kvm_enabled()) { + get_task_struct(current); + vcpu->arch.parent_task = current; } - vcpu->arch.host_thread_info = kern_hyp_va(ti); - vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd); -error: - return ret; + return 0; } /* @@ -66,26 +71,27 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) * * Here, we just set the correct metadata to indicate that the FPSIMD * state in the cpu regs (if any) belongs to current on the host. - * - * TIF_SVE is backed up here, since it may get clobbered with guest state. - * This flag is restored by kvm_arch_vcpu_put_fp(vcpu). */ void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu) { BUG_ON(!current->mm); + BUG_ON(test_thread_flag(TIF_SVE)); - vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED | - KVM_ARM64_HOST_SVE_IN_USE | - KVM_ARM64_HOST_SVE_ENABLED); + vcpu->arch.flags &= ~KVM_ARM64_FP_ENABLED; vcpu->arch.flags |= KVM_ARM64_FP_HOST; - if (test_thread_flag(TIF_SVE)) - vcpu->arch.flags |= KVM_ARM64_HOST_SVE_IN_USE; - if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN) vcpu->arch.flags |= KVM_ARM64_HOST_SVE_ENABLED; } +void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu) +{ + if (test_thread_flag(TIF_FOREIGN_FPSTATE)) + vcpu->arch.flags |= KVM_ARM64_FP_FOREIGN_FPSTATE; + else + vcpu->arch.flags &= ~KVM_ARM64_FP_FOREIGN_FPSTATE; +} + /* * If the guest FPSIMD state was loaded, update the host's context * tracking data mark the CPU FPSIMD regs as dirty and belonging to vcpu @@ -115,13 +121,11 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) { unsigned long flags; - bool host_has_sve = system_supports_sve(); - bool guest_has_sve = vcpu_has_sve(vcpu); local_irq_save(flags); if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) { - if (guest_has_sve) { + if (vcpu_has_sve(vcpu)) { __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); /* Restore the VL that was saved when bound to the CPU */ @@ -131,7 +135,7 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) } fpsimd_save_and_flush_cpu_state(); - } else if (has_vhe() && host_has_sve) { + } else if (has_vhe() && system_supports_sve()) { /* * The FPSIMD/SVE state in the CPU has not been touched, and we * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been @@ -145,8 +149,7 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0); } - update_thread_flag(TIF_SVE, - vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE); + update_thread_flag(TIF_SVE, 0); local_irq_restore(flags); } diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 5ce26bedf2..e116c77677 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -869,13 +869,10 @@ u32 __attribute_const__ kvm_target_cpu(void) return KVM_ARM_TARGET_GENERIC_V8; } -int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) +void kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) { u32 target = kvm_target_cpu(); - if (target < 0) - return -ENODEV; - memset(init, 0, sizeof(*init)); /* @@ -885,8 +882,6 @@ int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) * target type. */ init->target = (__u32)target; - - return 0; } int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index a5ab521509..e3140abd2e 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -82,7 +82,7 @@ static int handle_no_fpsimd(struct kvm_vcpu *vcpu) * * WFE: Yield the CPU and come back to this vcpu when the scheduler * decides to. - * WFI: Simply call kvm_vcpu_block(), which will halt execution of + * WFI: Simply call kvm_vcpu_halt(), which will halt execution of * world-switches and schedule other host processes until there is an * incoming IRQ or FIQ to the VM. */ @@ -95,8 +95,7 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu) } else { trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false); vcpu->stat.wfi_exit_stat++; - kvm_vcpu_block(vcpu); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); + kvm_vcpu_wfi(vcpu); } kvm_incr_pc(vcpu); @@ -140,9 +139,12 @@ static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu) return 1; } +/* + * Guest access to SVE registers should be routed to this handler only + * when the system doesn't support SVE. + */ static int handle_sve(struct kvm_vcpu *vcpu) { - /* Until SVE is supported for guests: */ kvm_inject_undefined(vcpu); return 1; } diff --git a/arch/arm64/kvm/hyp/Makefile b/arch/arm64/kvm/hyp/Makefile index b726332eec..687598e41b 100644 --- a/arch/arm64/kvm/hyp/Makefile +++ b/arch/arm64/kvm/hyp/Makefile @@ -10,4 +10,4 @@ subdir-ccflags-y := -I$(incdir) \ -DDISABLE_BRANCH_PROFILING \ $(DISABLE_STACKLEAK_PLUGIN) -obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o reserved_mem.o +obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S index 3c63592977..61e6f3ba7b 100644 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ b/arch/arm64/kvm/hyp/fpsimd.S @@ -21,11 +21,7 @@ SYM_FUNC_START(__fpsimd_restore_state) SYM_FUNC_END(__fpsimd_restore_state) SYM_FUNC_START(__sve_restore_state) - __sve_load 0, x1, 2 + mov x2, #1 + sve_load 0, x1, x2, 3 ret SYM_FUNC_END(__sve_restore_state) - -SYM_FUNC_START(__sve_save_state) - sve_save 0, x1, 2 - ret -SYM_FUNC_END(__sve_save_state) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index ecd41844ed..701cfb9649 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -8,6 +8,7 @@ #define __ARM64_KVM_HYP_SWITCH_H__ #include +#include #include #include @@ -28,10 +29,13 @@ #include #include #include -#include -extern struct exception_table_entry __start___kvm_ex_table; -extern struct exception_table_entry __stop___kvm_ex_table; +struct kvm_exception_table_entry { + int insn, fixup; +}; + +extern struct kvm_exception_table_entry __start___kvm_ex_table; +extern struct kvm_exception_table_entry __stop___kvm_ex_table; /* Check whether the FP regs were dirtied while in the host-side run loop: */ static inline bool update_fp_enabled(struct kvm_vcpu *vcpu) @@ -44,7 +48,7 @@ static inline bool update_fp_enabled(struct kvm_vcpu *vcpu) * trap the accesses. */ if (!system_supports_fpsimd() || - vcpu->arch.host_thread_info->flags & _TIF_FOREIGN_FPSTATE) + vcpu->arch.flags & KVM_ARM64_FP_FOREIGN_FPSTATE) vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED | KVM_ARM64_FP_HOST); @@ -133,88 +137,9 @@ static inline void ___deactivate_traps(struct kvm_vcpu *vcpu) } } -static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar) -{ - u64 par, tmp; - - /* - * Resolve the IPA the hard way using the guest VA. - * - * Stage-1 translation already validated the memory access - * rights. As such, we can use the EL1 translation regime, and - * don't have to distinguish between EL0 and EL1 access. - * - * We do need to save/restore PAR_EL1 though, as we haven't - * saved the guest context yet, and we may return early... - */ - par = read_sysreg_par(); - if (!__kvm_at("s1e1r", far)) - tmp = read_sysreg_par(); - else - tmp = SYS_PAR_EL1_F; /* back to the guest */ - write_sysreg(par, par_el1); - - if (unlikely(tmp & SYS_PAR_EL1_F)) - return false; /* Translation failed, back to guest */ - - /* Convert PAR to HPFAR format */ - *hpfar = PAR_TO_HPFAR(tmp); - return true; -} - -static inline bool __get_fault_info(u64 esr, struct kvm_vcpu_fault_info *fault) -{ - u64 hpfar, far; - - far = read_sysreg_el2(SYS_FAR); - - /* - * The HPFAR can be invalid if the stage 2 fault did not - * happen during a stage 1 page table walk (the ESR_EL2.S1PTW - * bit is clear) and one of the two following cases are true: - * 1. The fault was due to a permission fault - * 2. The processor carries errata 834220 - * - * Therefore, for all non S1PTW faults where we either have a - * permission fault or the errata workaround is enabled, we - * resolve the IPA using the AT instruction. - */ - if (!(esr & ESR_ELx_S1PTW) && - (cpus_have_final_cap(ARM64_WORKAROUND_834220) || - (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) { - if (!__translate_far_to_hpfar(far, &hpfar)) - return false; - } else { - hpfar = read_sysreg(hpfar_el2); - } - - fault->far_el2 = far; - fault->hpfar_el2 = hpfar; - return true; -} - static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) { - u8 ec; - u64 esr; - - esr = vcpu->arch.fault.esr_el2; - ec = ESR_ELx_EC(esr); - - if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW) - return true; - - return __get_fault_info(esr, &vcpu->arch.fault); -} - -static inline void __hyp_sve_save_host(struct kvm_vcpu *vcpu) -{ - struct thread_struct *thread; - - thread = container_of(vcpu->arch.host_fpsimd_state, struct thread_struct, - uw.fpsimd_state); - - __sve_save_state(sve_pffr(thread), &vcpu->arch.host_fpsimd_state->fpsr); + return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault); } static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) @@ -225,28 +150,23 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) write_sysreg_el1(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR); } -/* Check for an FPSIMD/SVE trap and handle as appropriate */ -static inline bool __hyp_handle_fpsimd(struct kvm_vcpu *vcpu) +/* + * We trap the first access to the FP/SIMD to save the host context and + * restore the guest context lazily. + * If FP/SIMD is not implemented, handle the trap and inject an undefined + * instruction exception to the guest. Similarly for trapped SVE accesses. + */ +static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) { - bool sve_guest, sve_host; + bool sve_guest; u8 esr_ec; u64 reg; if (!system_supports_fpsimd()) return false; - if (system_supports_sve()) { - sve_guest = vcpu_has_sve(vcpu); - sve_host = vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE; - } else { - sve_guest = false; - sve_host = false; - } - + sve_guest = vcpu_has_sve(vcpu); esr_ec = kvm_vcpu_trap_get_class(vcpu); - if (esr_ec != ESR_ELx_EC_FP_ASIMD && - esr_ec != ESR_ELx_EC_SVE) - return false; /* Don't handle SVE traps for non-SVE vcpus here: */ if (!sve_guest && esr_ec != ESR_ELx_EC_FP_ASIMD) @@ -269,11 +189,7 @@ static inline bool __hyp_handle_fpsimd(struct kvm_vcpu *vcpu) isb(); if (vcpu->arch.flags & KVM_ARM64_FP_HOST) { - if (sve_host) - __hyp_sve_save_host(vcpu); - else - __fpsimd_save_state(vcpu->arch.host_fpsimd_state); - + __fpsimd_save_state(vcpu->arch.host_fpsimd_state); vcpu->arch.flags &= ~KVM_ARM64_FP_HOST; } @@ -348,14 +264,6 @@ static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu) static inline bool esr_is_ptrauth_trap(u32 esr) { - u32 ec = ESR_ELx_EC(esr); - - if (ec == ESR_ELx_EC_PAC) - return true; - - if (ec != ESR_ELx_EC_SYS64) - return false; - switch (esr_sys64_to_sysreg(esr)) { case SYS_APIAKEYLO_EL1: case SYS_APIAKEYHI_EL1: @@ -384,13 +292,12 @@ static inline bool esr_is_ptrauth_trap(u32 esr) DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); -static inline bool __hyp_handle_ptrauth(struct kvm_vcpu *vcpu) +static bool kvm_hyp_handle_ptrauth(struct kvm_vcpu *vcpu, u64 *exit_code) { struct kvm_cpu_context *ctxt; u64 val; - if (!vcpu_has_ptrauth(vcpu) || - !esr_is_ptrauth_trap(kvm_vcpu_get_esr(vcpu))) + if (!vcpu_has_ptrauth(vcpu)) return false; ctxt = this_cpu_ptr(&kvm_hyp_ctxt); @@ -409,6 +316,110 @@ static inline bool __hyp_handle_ptrauth(struct kvm_vcpu *vcpu) return true; } +static bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) && + handle_tx2_tvm(vcpu)) + return true; + + if (static_branch_unlikely(&vgic_v3_cpuif_trap) && + __vgic_v3_perform_cpuif_access(vcpu) == 1) + return true; + + if (esr_is_ptrauth_trap(kvm_vcpu_get_esr(vcpu))) + return kvm_hyp_handle_ptrauth(vcpu, exit_code); + + return false; +} + +static bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (static_branch_unlikely(&vgic_v3_cpuif_trap) && + __vgic_v3_perform_cpuif_access(vcpu) == 1) + return true; + + return false; +} + +static bool kvm_hyp_handle_iabt_low(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (!__populate_fault_info(vcpu)) + return true; + + return false; +} + +static bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (!__populate_fault_info(vcpu)) + return true; + + if (static_branch_unlikely(&vgic_v2_cpuif_trap)) { + bool valid; + + valid = kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT && + kvm_vcpu_dabt_isvalid(vcpu) && + !kvm_vcpu_abt_issea(vcpu) && + !kvm_vcpu_abt_iss1tw(vcpu); + + if (valid) { + int ret = __vgic_v2_perform_cpuif_access(vcpu); + + if (ret == 1) + return true; + + /* Promote an illegal access to an SError.*/ + if (ret == -1) + *exit_code = ARM_EXCEPTION_EL1_SERROR; + } + } + + return false; +} + +typedef bool (*exit_handler_fn)(struct kvm_vcpu *, u64 *); + +static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu); + +static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code); + +/* + * Allow the hypervisor to handle the exit with an exit handler if it has one. + * + * Returns true if the hypervisor handled the exit, and control should go back + * to the guest, or false if it hasn't. + */ +static inline bool kvm_hyp_handle_exit(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + const exit_handler_fn *handlers = kvm_get_exit_handler_array(vcpu); + exit_handler_fn fn; + + fn = handlers[kvm_vcpu_trap_get_class(vcpu)]; + + if (fn) + return fn(vcpu, exit_code); + + return false; +} + +static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + /* + * Check for the conditions of Cortex-A510's #2077057. When these occur + * SPSR_EL2 can't be trusted, but isn't needed either as it is + * unchanged from the value in vcpu_gp_regs(vcpu)->pstate. + * Are we single-stepping the guest, and took a PAC exception from the + * active-not-pending state? + */ + if (cpus_have_final_cap(ARM64_WORKAROUND_2077057) && + vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && + *vcpu_cpsr(vcpu) & DBG_SPSR_SS && + ESR_ELx_EC(read_sysreg_el2(SYS_ESR)) == ESR_ELx_EC_PAC) + write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); + + vcpu->arch.ctxt.regs.pstate = read_sysreg_el2(SYS_SPSR); +} + /* * Return true when we were able to fixup the guest exit and should return to * the guest, false when we should restore the host state and return to the @@ -420,7 +431,13 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) * Save PSTATE early so that we can evaluate the vcpu mode * early on. */ - vcpu->arch.ctxt.regs.pstate = read_sysreg_el2(SYS_SPSR); + synchronize_vcpu_pstate(vcpu, exit_code); + + /* + * Check whether we want to repaint the state one way or + * another. + */ + early_exit_filter(vcpu, exit_code); if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR); @@ -450,59 +467,9 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) if (*exit_code != ARM_EXCEPTION_TRAP) goto exit; - if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) && - kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 && - handle_tx2_tvm(vcpu)) + /* Check if there's an exit handler and allow it to handle the exit. */ + if (kvm_hyp_handle_exit(vcpu, exit_code)) goto guest; - - /* - * We trap the first access to the FP/SIMD to save the host context - * and restore the guest context lazily. - * If FP/SIMD is not implemented, handle the trap and inject an - * undefined instruction exception to the guest. - * Similarly for trapped SVE accesses. - */ - if (__hyp_handle_fpsimd(vcpu)) - goto guest; - - if (__hyp_handle_ptrauth(vcpu)) - goto guest; - - if (!__populate_fault_info(vcpu)) - goto guest; - - if (static_branch_unlikely(&vgic_v2_cpuif_trap)) { - bool valid; - - valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW && - kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT && - kvm_vcpu_dabt_isvalid(vcpu) && - !kvm_vcpu_abt_issea(vcpu) && - !kvm_vcpu_abt_iss1tw(vcpu); - - if (valid) { - int ret = __vgic_v2_perform_cpuif_access(vcpu); - - if (ret == 1) - goto guest; - - /* Promote an illegal access to an SError.*/ - if (ret == -1) - *exit_code = ARM_EXCEPTION_EL1_SERROR; - - goto exit; - } - } - - if (static_branch_unlikely(&vgic_v3_cpuif_trap) && - (kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 || - kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) { - int ret = __vgic_v3_perform_cpuif_access(vcpu); - - if (ret == 1) - goto guest; - } - exit: /* Return to the host kernel and handle the exit */ return false; @@ -517,7 +484,7 @@ static inline void __kvm_unexpected_el2_exception(void) { extern char __guest_exit_panic[]; unsigned long addr, fixup; - struct exception_table_entry *entry, *end; + struct kvm_exception_table_entry *entry, *end; unsigned long elr_el2 = read_sysreg(elr_el2); entry = &__start___kvm_ex_table; diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index b58c910bab..80e99836ea 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -24,6 +24,11 @@ enum pkvm_page_state { PKVM_PAGE_OWNED = 0ULL, PKVM_PAGE_SHARED_OWNED = KVM_PGTABLE_PROT_SW0, PKVM_PAGE_SHARED_BORROWED = KVM_PGTABLE_PROT_SW1, + __PKVM_PAGE_RESERVED = KVM_PGTABLE_PROT_SW0 | + KVM_PGTABLE_PROT_SW1, + + /* Meta-states which aren't encoded directly in the PTE's SW bits */ + PKVM_NOPAGE, }; #define PKVM_PAGE_STATE_PROT_MASK (KVM_PGTABLE_PROT_SW0 | KVM_PGTABLE_PROT_SW1) @@ -50,6 +55,7 @@ extern const u8 pkvm_hyp_id; int __pkvm_prot_finalize(void); int __pkvm_host_share_hyp(u64 pfn); +int __pkvm_host_unshare_hyp(u64 pfn); bool addr_is_memory(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot); diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index c9a8f53521..2d08510c6c 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -10,13 +10,8 @@ #include #include -#define HYP_MEMBLOCK_REGIONS 128 -extern struct memblock_region kvm_nvhe_sym(hyp_memory)[]; -extern unsigned int kvm_nvhe_sym(hyp_memblock_nr); extern struct kvm_pgtable pkvm_pgtable; extern hyp_spinlock_t pkvm_pgd_lock; -extern struct hyp_pool hpool; -extern u64 __io_map_base; int hyp_create_idmap(u32 hyp_va_bits); int hyp_map_vectors(void); @@ -39,58 +34,4 @@ static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size, *end = ALIGN(*end, PAGE_SIZE); } -static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages) -{ - unsigned long total = 0, i; - - /* Provision the worst case scenario */ - for (i = 0; i < KVM_PGTABLE_MAX_LEVELS; i++) { - nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE); - total += nr_pages; - } - - return total; -} - -static inline unsigned long __hyp_pgtable_total_pages(void) -{ - unsigned long res = 0, i; - - /* Cover all of memory with page-granularity */ - for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) { - struct memblock_region *reg = &kvm_nvhe_sym(hyp_memory)[i]; - res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT); - } - - return res; -} - -static inline unsigned long hyp_s1_pgtable_pages(void) -{ - unsigned long res; - - res = __hyp_pgtable_total_pages(); - - /* Allow 1 GiB for private mappings */ - res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); - - return res; -} - -static inline unsigned long host_s2_pgtable_pages(void) -{ - unsigned long res; - - /* - * Include an extra 16 pages to safely upper-bound the worst case of - * concatenated pgds. - */ - res = __hyp_pgtable_total_pages() + 16; - - /* Allow 1 GiB for MMIO mappings */ - res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); - - return res; -} - #endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/trap_handler.h b/arch/arm64/kvm/hyp/include/nvhe/trap_handler.h index 1e6d995968..45a84f0ade 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/trap_handler.h +++ b/arch/arm64/kvm/hyp/include/nvhe/trap_handler.h @@ -15,4 +15,6 @@ #define DECLARE_REG(type, name, ctxt, reg) \ type name = (type)cpu_reg(ctxt, (reg)) +void __pkvm_vcpu_init_traps(struct kvm_vcpu *vcpu); + #endif /* __ARM64_KVM_NVHE_TRAP_HANDLER_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 8d741f7137..24b2c2425b 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -14,7 +14,7 @@ lib-objs := $(addprefix ../../../lib/, $(lib-objs)) obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \ hyp-main.o hyp-smp.o psci-relay.o early_alloc.o stub.o page_alloc.o \ - cache.o setup.o mm.o mem_protect.o + cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o obj-y += $(lib-objs) @@ -89,6 +89,7 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_CFI) # cause crashes. Just disable it. GCOV_PROFILE := n KASAN_SANITIZE := n +KCSAN_SANITIZE := n UBSAN_SANITIZE := n KCOV_INSTRUMENT := n diff --git a/arch/arm64/kvm/hyp/nvhe/early_alloc.c b/arch/arm64/kvm/hyp/nvhe/early_alloc.c index 1306c430ab..00de04153c 100644 --- a/arch/arm64/kvm/hyp/nvhe/early_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/early_alloc.c @@ -43,6 +43,9 @@ void *hyp_early_alloc_page(void *arg) return hyp_early_alloc_contig(1); } +static void hyp_early_alloc_get_page(void *addr) { } +static void hyp_early_alloc_put_page(void *addr) { } + void hyp_early_alloc_init(void *virt, unsigned long size) { base = cur = (unsigned long)virt; @@ -51,4 +54,6 @@ void hyp_early_alloc_init(void *virt, unsigned long size) hyp_early_alloc_mm_ops.zalloc_page = hyp_early_alloc_page; hyp_early_alloc_mm_ops.phys_to_virt = hyp_phys_to_virt; hyp_early_alloc_mm_ops.virt_to_phys = hyp_virt_to_phys; + hyp_early_alloc_mm_ops.get_page = hyp_early_alloc_get_page; + hyp_early_alloc_mm_ops.put_page = hyp_early_alloc_put_page; } diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index d310d2b2c8..3d613e721a 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -110,17 +110,14 @@ SYM_FUNC_START(__hyp_do_panic) b __host_enter_for_panic SYM_FUNC_END(__hyp_do_panic) -.macro host_el1_sync_vect - .align 7 -.L__vect_start\@: - stp x0, x1, [sp, #-16]! - mrs x0, esr_el2 - ubfx x0, x0, #ESR_ELx_EC_SHIFT, #ESR_ELx_EC_WIDTH - cmp x0, #ESR_ELx_EC_HVC64 - b.ne __host_exit - +SYM_FUNC_START(__host_hvc) ldp x0, x1, [sp] // Don't fixup the stack yet + /* No stub for you, sonny Jim */ +alternative_if ARM64_KVM_PROTECTED_MODE + b __host_exit +alternative_else_nop_endif + /* Check for a stub HVC call */ cmp x0, #HVC_STUB_HCALL_NR b.hs __host_exit @@ -137,6 +134,17 @@ SYM_FUNC_END(__hyp_do_panic) ldr x5, =__kvm_handle_stub_hvc hyp_pa x5, x6 br x5 +SYM_FUNC_END(__host_hvc) + +.macro host_el1_sync_vect + .align 7 +.L__vect_start\@: + stp x0, x1, [sp, #-16]! + mrs x0, esr_el2 + ubfx x0, x0, #ESR_ELx_EC_SHIFT, #ESR_ELx_EC_WIDTH + cmp x0, #ESR_ELx_EC_HVC64 + b.eq __host_hvc + b __host_exit .L__vect_end\@: .if ((.L__vect_end\@ - .L__vect_start\@) > 0x80) .error "host_el1_sync_vect larger than vector entry" diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 2da6aa8da8..5e2197db0d 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -4,7 +4,7 @@ * Author: Andrew Scull */ -#include +#include #include #include @@ -147,6 +147,13 @@ static void handle___pkvm_host_share_hyp(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = __pkvm_host_share_hyp(pfn); } +static void handle___pkvm_host_unshare_hyp(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(u64, pfn, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_host_unshare_hyp(pfn); +} + static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); @@ -160,41 +167,66 @@ static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt) { cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize(); } + +static void handle___pkvm_vcpu_init_traps(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); + + __pkvm_vcpu_init_traps(kern_hyp_va(vcpu)); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x static const hcall_t host_hcall[] = { - HANDLE_FUNC(__kvm_vcpu_run), + /* ___kvm_hyp_init */ + HANDLE_FUNC(__kvm_get_mdcr_el2), + HANDLE_FUNC(__pkvm_init), + HANDLE_FUNC(__pkvm_create_private_mapping), + HANDLE_FUNC(__pkvm_cpu_set_vector), + HANDLE_FUNC(__kvm_enable_ssbs), + HANDLE_FUNC(__vgic_v3_init_lrs), + HANDLE_FUNC(__vgic_v3_get_gic_config), + HANDLE_FUNC(__pkvm_prot_finalize), + + HANDLE_FUNC(__pkvm_host_share_hyp), + HANDLE_FUNC(__pkvm_host_unshare_hyp), HANDLE_FUNC(__kvm_adjust_pc), + HANDLE_FUNC(__kvm_vcpu_run), HANDLE_FUNC(__kvm_flush_vm_context), HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa), HANDLE_FUNC(__kvm_tlb_flush_vmid), HANDLE_FUNC(__kvm_flush_cpu_context), HANDLE_FUNC(__kvm_timer_set_cntvoff), - HANDLE_FUNC(__kvm_enable_ssbs), - HANDLE_FUNC(__vgic_v3_get_gic_config), HANDLE_FUNC(__vgic_v3_read_vmcr), HANDLE_FUNC(__vgic_v3_write_vmcr), - HANDLE_FUNC(__vgic_v3_init_lrs), - HANDLE_FUNC(__kvm_get_mdcr_el2), HANDLE_FUNC(__vgic_v3_save_aprs), HANDLE_FUNC(__vgic_v3_restore_aprs), - HANDLE_FUNC(__pkvm_init), - HANDLE_FUNC(__pkvm_cpu_set_vector), - HANDLE_FUNC(__pkvm_host_share_hyp), - HANDLE_FUNC(__pkvm_create_private_mapping), - HANDLE_FUNC(__pkvm_prot_finalize), + HANDLE_FUNC(__pkvm_vcpu_init_traps), }; static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(unsigned long, id, host_ctxt, 0); + unsigned long hcall_min = 0; hcall_t hfn; + /* + * If pKVM has been initialised then reject any calls to the + * early "privileged" hypercalls. Note that we cannot reject + * calls to __pkvm_prot_finalize for two reasons: (1) The static + * key used to determine initialisation must be toggled prior to + * finalisation and (2) finalisation is performed on a per-CPU + * basis. This is all fine, however, since __pkvm_prot_finalize + * returns -EPERM after the first call for a given CPU. + */ + if (static_branch_unlikely(&kvm_protected_mode_initialized)) + hcall_min = __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize; + id -= KVM_HOST_SMCCC_ID(0); - if (unlikely(id >= ARRAY_SIZE(host_hcall))) + if (unlikely(id < hcall_min || id >= ARRAY_SIZE(host_hcall))) goto inval; hfn = host_hcall[id]; diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 34eeb524b6..674f105643 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -9,9 +9,10 @@ #include #include #include +#include #include -#include +#include #include #include @@ -25,14 +26,28 @@ struct host_kvm host_kvm; static struct hyp_pool host_s2_pool; -/* - * Copies of the host's CPU features registers holding sanitized values. - */ -u64 id_aa64mmfr0_el1_sys_val; -u64 id_aa64mmfr1_el1_sys_val; - const u8 pkvm_hyp_id = 1; +static void host_lock_component(void) +{ + hyp_spin_lock(&host_kvm.lock); +} + +static void host_unlock_component(void) +{ + hyp_spin_unlock(&host_kvm.lock); +} + +static void hyp_lock_component(void) +{ + hyp_spin_lock(&pkvm_pgd_lock); +} + +static void hyp_unlock_component(void) +{ + hyp_spin_unlock(&pkvm_pgd_lock); +} + static void *host_s2_zalloc_pages_exact(size_t size) { void *addr = hyp_alloc_pages(&host_s2_pool, get_order(size)); @@ -109,19 +124,19 @@ int kvm_host_prepare_stage2(void *pgt_pool_base) prepare_host_vtcr(); hyp_spin_lock_init(&host_kvm.lock); + mmu->arch = &host_kvm.arch; ret = prepare_s2_pool(pgt_pool_base); if (ret) return ret; - ret = __kvm_pgtable_stage2_init(&host_kvm.pgt, &host_kvm.arch, + ret = __kvm_pgtable_stage2_init(&host_kvm.pgt, mmu, &host_kvm.mm_ops, KVM_HOST_S2_FLAGS, host_stage2_force_pte_cb); if (ret) return ret; mmu->pgd_phys = __hyp_pa(host_kvm.pgt.pgd); - mmu->arch = &host_kvm.arch; mmu->pgt = &host_kvm.pgt; WRITE_ONCE(mmu->vmid.vmid_gen, 0); WRITE_ONCE(mmu->vmid.vmid, 0); @@ -134,6 +149,9 @@ int __pkvm_prot_finalize(void) struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu; struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params); + if (params->hcr_el2 & HCR_VM) + return -EPERM; + params->vttbr = kvm_get_vttbr(mmu); params->vtcr = host_kvm.arch.vtcr; params->hcr_el2 |= HCR_VM; @@ -341,102 +359,14 @@ static int host_stage2_idmap(u64 addr) prot = is_memory ? PKVM_HOST_MEM_PROT : PKVM_HOST_MMIO_PROT; - hyp_spin_lock(&host_kvm.lock); + host_lock_component(); ret = host_stage2_adjust_range(addr, &range); if (ret) goto unlock; ret = host_stage2_idmap_locked(range.start, range.end - range.start, prot); unlock: - hyp_spin_unlock(&host_kvm.lock); - - return ret; -} - -static inline bool check_prot(enum kvm_pgtable_prot prot, - enum kvm_pgtable_prot required, - enum kvm_pgtable_prot denied) -{ - return (prot & (required | denied)) == required; -} - -int __pkvm_host_share_hyp(u64 pfn) -{ - phys_addr_t addr = hyp_pfn_to_phys(pfn); - enum kvm_pgtable_prot prot, cur; - void *virt = __hyp_va(addr); - enum pkvm_page_state state; - kvm_pte_t pte; - int ret; - - if (!addr_is_memory(addr)) - return -EINVAL; - - hyp_spin_lock(&host_kvm.lock); - hyp_spin_lock(&pkvm_pgd_lock); - - ret = kvm_pgtable_get_leaf(&host_kvm.pgt, addr, &pte, NULL); - if (ret) - goto unlock; - if (!pte) - goto map_shared; - - /* - * Check attributes in the host stage-2 PTE. We need the page to be: - * - mapped RWX as we're sharing memory; - * - not borrowed, as that implies absence of ownership. - * Otherwise, we can't let it got through - */ - cur = kvm_pgtable_stage2_pte_prot(pte); - prot = pkvm_mkstate(0, PKVM_PAGE_SHARED_BORROWED); - if (!check_prot(cur, PKVM_HOST_MEM_PROT, prot)) { - ret = -EPERM; - goto unlock; - } - - state = pkvm_getstate(cur); - if (state == PKVM_PAGE_OWNED) - goto map_shared; - - /* - * Tolerate double-sharing the same page, but this requires - * cross-checking the hypervisor stage-1. - */ - if (state != PKVM_PAGE_SHARED_OWNED) { - ret = -EPERM; - goto unlock; - } - - ret = kvm_pgtable_get_leaf(&pkvm_pgtable, (u64)virt, &pte, NULL); - if (ret) - goto unlock; - - /* - * If the page has been shared with the hypervisor, it must be - * already mapped as SHARED_BORROWED in its stage-1. - */ - cur = kvm_pgtable_hyp_pte_prot(pte); - prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_SHARED_BORROWED); - if (!check_prot(cur, prot, ~prot)) - ret = -EPERM; - goto unlock; - -map_shared: - /* - * If the page is not yet shared, adjust mappings in both page-tables - * while both locks are held. - */ - prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_SHARED_BORROWED); - ret = pkvm_create_mappings_locked(virt, virt + PAGE_SIZE, prot); - BUG_ON(ret); - - prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, PKVM_PAGE_SHARED_OWNED); - ret = host_stage2_idmap_locked(addr, PAGE_SIZE, prot); - BUG_ON(ret); - -unlock: - hyp_spin_unlock(&pkvm_pgd_lock); - hyp_spin_unlock(&host_kvm.lock); + host_unlock_component(); return ret; } @@ -454,3 +384,421 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) ret = host_stage2_idmap(addr); BUG_ON(ret && ret != -EAGAIN); } + +/* This corresponds to locking order */ +enum pkvm_component_id { + PKVM_ID_HOST, + PKVM_ID_HYP, +}; + +struct pkvm_mem_transition { + u64 nr_pages; + + struct { + enum pkvm_component_id id; + /* Address in the initiator's address space */ + u64 addr; + + union { + struct { + /* Address in the completer's address space */ + u64 completer_addr; + } host; + }; + } initiator; + + struct { + enum pkvm_component_id id; + } completer; +}; + +struct pkvm_mem_share { + const struct pkvm_mem_transition tx; + const enum kvm_pgtable_prot completer_prot; +}; + +struct check_walk_data { + enum pkvm_page_state desired; + enum pkvm_page_state (*get_page_state)(kvm_pte_t pte); +}; + +static int __check_page_state_visitor(u64 addr, u64 end, u32 level, + kvm_pte_t *ptep, + enum kvm_pgtable_walk_flags flag, + void * const arg) +{ + struct check_walk_data *d = arg; + kvm_pte_t pte = *ptep; + + if (kvm_pte_valid(pte) && !addr_is_memory(kvm_pte_to_phys(pte))) + return -EINVAL; + + return d->get_page_state(pte) == d->desired ? 0 : -EPERM; +} + +static int check_page_state_range(struct kvm_pgtable *pgt, u64 addr, u64 size, + struct check_walk_data *data) +{ + struct kvm_pgtable_walker walker = { + .cb = __check_page_state_visitor, + .arg = data, + .flags = KVM_PGTABLE_WALK_LEAF, + }; + + return kvm_pgtable_walk(pgt, addr, size, &walker); +} + +static enum pkvm_page_state host_get_page_state(kvm_pte_t pte) +{ + if (!kvm_pte_valid(pte) && pte) + return PKVM_NOPAGE; + + return pkvm_getstate(kvm_pgtable_stage2_pte_prot(pte)); +} + +static int __host_check_page_state_range(u64 addr, u64 size, + enum pkvm_page_state state) +{ + struct check_walk_data d = { + .desired = state, + .get_page_state = host_get_page_state, + }; + + hyp_assert_lock_held(&host_kvm.lock); + return check_page_state_range(&host_kvm.pgt, addr, size, &d); +} + +static int __host_set_page_state_range(u64 addr, u64 size, + enum pkvm_page_state state) +{ + enum kvm_pgtable_prot prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, state); + + return host_stage2_idmap_locked(addr, size, prot); +} + +static int host_request_owned_transition(u64 *completer_addr, + const struct pkvm_mem_transition *tx) +{ + u64 size = tx->nr_pages * PAGE_SIZE; + u64 addr = tx->initiator.addr; + + *completer_addr = tx->initiator.host.completer_addr; + return __host_check_page_state_range(addr, size, PKVM_PAGE_OWNED); +} + +static int host_request_unshare(u64 *completer_addr, + const struct pkvm_mem_transition *tx) +{ + u64 size = tx->nr_pages * PAGE_SIZE; + u64 addr = tx->initiator.addr; + + *completer_addr = tx->initiator.host.completer_addr; + return __host_check_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED); +} + +static int host_initiate_share(u64 *completer_addr, + const struct pkvm_mem_transition *tx) +{ + u64 size = tx->nr_pages * PAGE_SIZE; + u64 addr = tx->initiator.addr; + + *completer_addr = tx->initiator.host.completer_addr; + return __host_set_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED); +} + +static int host_initiate_unshare(u64 *completer_addr, + const struct pkvm_mem_transition *tx) +{ + u64 size = tx->nr_pages * PAGE_SIZE; + u64 addr = tx->initiator.addr; + + *completer_addr = tx->initiator.host.completer_addr; + return __host_set_page_state_range(addr, size, PKVM_PAGE_OWNED); +} + +static enum pkvm_page_state hyp_get_page_state(kvm_pte_t pte) +{ + if (!kvm_pte_valid(pte)) + return PKVM_NOPAGE; + + return pkvm_getstate(kvm_pgtable_stage2_pte_prot(pte)); +} + +static int __hyp_check_page_state_range(u64 addr, u64 size, + enum pkvm_page_state state) +{ + struct check_walk_data d = { + .desired = state, + .get_page_state = hyp_get_page_state, + }; + + hyp_assert_lock_held(&pkvm_pgd_lock); + return check_page_state_range(&pkvm_pgtable, addr, size, &d); +} + +static bool __hyp_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx) +{ + return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) || + tx->initiator.id != PKVM_ID_HOST); +} + +static int hyp_ack_share(u64 addr, const struct pkvm_mem_transition *tx, + enum kvm_pgtable_prot perms) +{ + u64 size = tx->nr_pages * PAGE_SIZE; + + if (perms != PAGE_HYP) + return -EPERM; + + if (__hyp_ack_skip_pgtable_check(tx)) + return 0; + + return __hyp_check_page_state_range(addr, size, PKVM_NOPAGE); +} + +static int hyp_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx) +{ + u64 size = tx->nr_pages * PAGE_SIZE; + + if (__hyp_ack_skip_pgtable_check(tx)) + return 0; + + return __hyp_check_page_state_range(addr, size, + PKVM_PAGE_SHARED_BORROWED); +} + +static int hyp_complete_share(u64 addr, const struct pkvm_mem_transition *tx, + enum kvm_pgtable_prot perms) +{ + void *start = (void *)addr, *end = start + (tx->nr_pages * PAGE_SIZE); + enum kvm_pgtable_prot prot; + + prot = pkvm_mkstate(perms, PKVM_PAGE_SHARED_BORROWED); + return pkvm_create_mappings_locked(start, end, prot); +} + +static int hyp_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx) +{ + u64 size = tx->nr_pages * PAGE_SIZE; + int ret = kvm_pgtable_hyp_unmap(&pkvm_pgtable, addr, size); + + return (ret != size) ? -EFAULT : 0; +} + +static int check_share(struct pkvm_mem_share *share) +{ + const struct pkvm_mem_transition *tx = &share->tx; + u64 completer_addr; + int ret; + + switch (tx->initiator.id) { + case PKVM_ID_HOST: + ret = host_request_owned_transition(&completer_addr, tx); + break; + default: + ret = -EINVAL; + } + + if (ret) + return ret; + + switch (tx->completer.id) { + case PKVM_ID_HYP: + ret = hyp_ack_share(completer_addr, tx, share->completer_prot); + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static int __do_share(struct pkvm_mem_share *share) +{ + const struct pkvm_mem_transition *tx = &share->tx; + u64 completer_addr; + int ret; + + switch (tx->initiator.id) { + case PKVM_ID_HOST: + ret = host_initiate_share(&completer_addr, tx); + break; + default: + ret = -EINVAL; + } + + if (ret) + return ret; + + switch (tx->completer.id) { + case PKVM_ID_HYP: + ret = hyp_complete_share(completer_addr, tx, share->completer_prot); + break; + default: + ret = -EINVAL; + } + + return ret; +} + +/* + * do_share(): + * + * The page owner grants access to another component with a given set + * of permissions. + * + * Initiator: OWNED => SHARED_OWNED + * Completer: NOPAGE => SHARED_BORROWED + */ +static int do_share(struct pkvm_mem_share *share) +{ + int ret; + + ret = check_share(share); + if (ret) + return ret; + + return WARN_ON(__do_share(share)); +} + +static int check_unshare(struct pkvm_mem_share *share) +{ + const struct pkvm_mem_transition *tx = &share->tx; + u64 completer_addr; + int ret; + + switch (tx->initiator.id) { + case PKVM_ID_HOST: + ret = host_request_unshare(&completer_addr, tx); + break; + default: + ret = -EINVAL; + } + + if (ret) + return ret; + + switch (tx->completer.id) { + case PKVM_ID_HYP: + ret = hyp_ack_unshare(completer_addr, tx); + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static int __do_unshare(struct pkvm_mem_share *share) +{ + const struct pkvm_mem_transition *tx = &share->tx; + u64 completer_addr; + int ret; + + switch (tx->initiator.id) { + case PKVM_ID_HOST: + ret = host_initiate_unshare(&completer_addr, tx); + break; + default: + ret = -EINVAL; + } + + if (ret) + return ret; + + switch (tx->completer.id) { + case PKVM_ID_HYP: + ret = hyp_complete_unshare(completer_addr, tx); + break; + default: + ret = -EINVAL; + } + + return ret; +} + +/* + * do_unshare(): + * + * The page owner revokes access from another component for a range of + * pages which were previously shared using do_share(). + * + * Initiator: SHARED_OWNED => OWNED + * Completer: SHARED_BORROWED => NOPAGE + */ +static int do_unshare(struct pkvm_mem_share *share) +{ + int ret; + + ret = check_unshare(share); + if (ret) + return ret; + + return WARN_ON(__do_unshare(share)); +} + +int __pkvm_host_share_hyp(u64 pfn) +{ + int ret; + u64 host_addr = hyp_pfn_to_phys(pfn); + u64 hyp_addr = (u64)__hyp_va(host_addr); + struct pkvm_mem_share share = { + .tx = { + .nr_pages = 1, + .initiator = { + .id = PKVM_ID_HOST, + .addr = host_addr, + .host = { + .completer_addr = hyp_addr, + }, + }, + .completer = { + .id = PKVM_ID_HYP, + }, + }, + .completer_prot = PAGE_HYP, + }; + + host_lock_component(); + hyp_lock_component(); + + ret = do_share(&share); + + hyp_unlock_component(); + host_unlock_component(); + + return ret; +} + +int __pkvm_host_unshare_hyp(u64 pfn) +{ + int ret; + u64 host_addr = hyp_pfn_to_phys(pfn); + u64 hyp_addr = (u64)__hyp_va(host_addr); + struct pkvm_mem_share share = { + .tx = { + .nr_pages = 1, + .initiator = { + .id = PKVM_ID_HOST, + .addr = host_addr, + .host = { + .completer_addr = hyp_addr, + }, + }, + .completer = { + .id = PKVM_ID_HYP, + }, + }, + .completer_prot = PAGE_HYP, + }; + + host_lock_component(); + hyp_lock_component(); + + ret = do_unshare(&share); + + hyp_unlock_component(); + host_unlock_component(); + + return ret; +} diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 5146fb1705..cdbe8e2464 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -18,11 +19,12 @@ struct kvm_pgtable pkvm_pgtable; hyp_spinlock_t pkvm_pgd_lock; -u64 __io_map_base; struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS]; unsigned int hyp_memblock_nr; +static u64 __io_map_base; + static int __pkvm_create_mappings(unsigned long start, unsigned long size, unsigned long phys, enum kvm_pgtable_prot prot) { diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c index 0bd7701ad1..543cad6c37 100644 --- a/arch/arm64/kvm/hyp/nvhe/page_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c @@ -241,7 +241,7 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, int i; hyp_spin_lock_init(&pool->lock); - pool->max_order = min(MAX_ORDER, get_order(nr_pages << PAGE_SHIFT)); + pool->max_order = min(MAX_ORDER, get_order((nr_pages + 1) << PAGE_SHIFT)); for (i = 0; i < pool->max_order; i++) INIT_LIST_HEAD(&pool->free_area[i]); pool->range_start = phys; diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 58ad9c5ba3..27af337f9f 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -8,15 +8,16 @@ #include #include #include +#include #include +#include #include #include #include #include #include -struct hyp_pool hpool; unsigned long hyp_nr_cpus; #define hyp_percpu_size ((unsigned long)__per_cpu_end - \ @@ -26,6 +27,7 @@ static void *vmemmap_base; static void *hyp_pgt_base; static void *host_s2_pgt_base; static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops; +static struct hyp_pool hpool; static int divide_memory_pool(void *virt, unsigned long size) { @@ -164,6 +166,7 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level, enum kvm_pgtable_walk_flags flag, void * const arg) { + struct kvm_pgtable_mm_ops *mm_ops = arg; enum kvm_pgtable_prot prot; enum pkvm_page_state state; kvm_pte_t pte = *ptep; @@ -172,6 +175,15 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level, if (!kvm_pte_valid(pte)) return 0; + /* + * Fix-up the refcount for the page-table pages as the early allocator + * was unable to access the hyp_vmemmap and so the buddy allocator has + * initialised the refcount to '1'. + */ + mm_ops->get_page(ptep); + if (flag != KVM_PGTABLE_WALK_LEAF) + return 0; + if (level != (KVM_PGTABLE_MAX_LEVELS - 1)) return -EINVAL; @@ -204,7 +216,8 @@ static int finalize_host_mappings(void) { struct kvm_pgtable_walker walker = { .cb = finalize_host_mappings_walker, - .flags = KVM_PGTABLE_WALK_LEAF, + .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, + .arg = pkvm_pgtable.mm_ops, }; int i, ret; @@ -239,19 +252,20 @@ void __noreturn __pkvm_init_finalise(void) if (ret) goto out; - ret = finalize_host_mappings(); - if (ret) - goto out; - pkvm_pgtable_mm_ops = (struct kvm_pgtable_mm_ops) { .zalloc_page = hyp_zalloc_hyp_page, .phys_to_virt = hyp_phys_to_virt, .virt_to_phys = hyp_virt_to_phys, .get_page = hpool_get_page, .put_page = hpool_put_page, + .page_count = hyp_page_count, }; pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops; + ret = finalize_host_mappings(); + if (ret) + goto out; + out: /* * We tail-called to here from handle___pkvm_init() and will not return, @@ -270,6 +284,8 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus, void (*fn)(phys_addr_t params_pa, void *finalize_fn_va); int ret; + BUG_ON(kvm_check_pvm_sysreg_table()); + if (!PAGE_ALIGNED(phys) || !PAGE_ALIGNED(size)) return -EINVAL; diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index a34b01cc8a..6410d21d86 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -25,8 +25,8 @@ #include #include #include -#include +#include #include /* Non-VHE specific context */ @@ -158,6 +158,98 @@ static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) write_sysreg(pmu->events_host, pmcntenset_el0); } +/** + * Handler for protected VM MSR, MRS or System instruction execution in AArch64. + * + * Returns true if the hypervisor has handled the exit, and control should go + * back to the guest, or false if it hasn't. + */ +static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + /* + * Make sure we handle the exit for workarounds and ptrauth + * before the pKVM handling, as the latter could decide to + * UNDEF. + */ + return (kvm_hyp_handle_sysreg(vcpu, exit_code) || + kvm_handle_pvm_sysreg(vcpu, exit_code)); +} + +/** + * Handler for protected floating-point and Advanced SIMD accesses. + * + * Returns true if the hypervisor has handled the exit, and control should go + * back to the guest, or false if it hasn't. + */ +static bool kvm_handle_pvm_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + /* Linux guests assume support for floating-point and Advanced SIMD. */ + BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_FP), + PVM_ID_AA64PFR0_ALLOW)); + BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_ASIMD), + PVM_ID_AA64PFR0_ALLOW)); + + return kvm_hyp_handle_fpsimd(vcpu, exit_code); +} + +static const exit_handler_fn hyp_exit_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = NULL, + [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32, + [ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg, + [ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd, + [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, + [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low, + [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low, + [ESR_ELx_EC_PAC] = kvm_hyp_handle_ptrauth, +}; + +static const exit_handler_fn pvm_exit_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = NULL, + [ESR_ELx_EC_SYS64] = kvm_handle_pvm_sys64, + [ESR_ELx_EC_SVE] = kvm_handle_pvm_restricted, + [ESR_ELx_EC_FP_ASIMD] = kvm_handle_pvm_fpsimd, + [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low, + [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low, + [ESR_ELx_EC_PAC] = kvm_hyp_handle_ptrauth, +}; + +static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu) +{ + if (unlikely(kvm_vm_is_protected(kern_hyp_va(vcpu->kvm)))) + return pvm_exit_handlers; + + return hyp_exit_handlers; +} + +/* + * Some guests (e.g., protected VMs) are not be allowed to run in AArch32. + * The ARMv8 architecture does not give the hypervisor a mechanism to prevent a + * guest from dropping to AArch32 EL0 if implemented by the CPU. If the + * hypervisor spots a guest in such a state ensure it is handled, and don't + * trust the host to spot or fix it. The check below is based on the one in + * kvm_arch_vcpu_ioctl_run(). + * + * Returns false if the guest ran in AArch32 when it shouldn't have, and + * thus should exit to the host, or true if a the guest run loop can continue. + */ +static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + struct kvm *kvm = kern_hyp_va(vcpu->kvm); + + if (kvm_vm_is_protected(kvm) && vcpu_mode_is_32bit(vcpu)) { + /* + * As we have caught the guest red-handed, decide that it isn't + * fit for purpose anymore by making the vcpu invalid. The VMM + * can try and fix it by re-initializing the vcpu with + * KVM_ARM_VCPU_INIT, however, this is likely not possible for + * protected VMs. + */ + vcpu->arch.target = -1; + *exit_code &= BIT(ARM_EXIT_WITH_SERROR_BIT); + *exit_code |= ARM_EXCEPTION_IL; + } +} + /* Switch to the guest for legacy non-VHE systems */ int __kvm_vcpu_run(struct kvm_vcpu *vcpu) { diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 4c77ff556f..2cb3867eb7 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -383,21 +383,6 @@ enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte) return prot; } -static bool hyp_pte_needs_update(kvm_pte_t old, kvm_pte_t new) -{ - /* - * Tolerate KVM recreating the exact same mapping, or changing software - * bits if the existing mapping was valid. - */ - if (old == new) - return false; - - if (!kvm_pte_valid(old)) - return true; - - return !WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW); -} - static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, struct hyp_map_data *data) { @@ -407,11 +392,16 @@ static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level, if (!kvm_block_mapping_supported(addr, end, phys, level)) return false; - new = kvm_init_valid_leaf_pte(phys, data->attr, level); - if (hyp_pte_needs_update(old, new)) - smp_store_release(ptep, new); - data->phys += granule; + new = kvm_init_valid_leaf_pte(phys, data->attr, level); + if (old == new) + return true; + if (!kvm_pte_valid(old)) + data->mm_ops->get_page(ptep); + else if (WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW)) + return false; + + smp_store_release(ptep, new); return true; } @@ -433,6 +423,7 @@ static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, return -ENOMEM; kvm_set_table_pte(ptep, childp, mm_ops); + mm_ops->get_page(ptep); return 0; } @@ -460,6 +451,69 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, return ret; } +struct hyp_unmap_data { + u64 unmapped; + struct kvm_pgtable_mm_ops *mm_ops; +}; + +static int hyp_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, + enum kvm_pgtable_walk_flags flag, void * const arg) +{ + kvm_pte_t pte = *ptep, *childp = NULL; + u64 granule = kvm_granule_size(level); + struct hyp_unmap_data *data = arg; + struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; + + if (!kvm_pte_valid(pte)) + return -EINVAL; + + if (kvm_pte_table(pte, level)) { + childp = kvm_pte_follow(pte, mm_ops); + + if (mm_ops->page_count(childp) != 1) + return 0; + + kvm_clear_pte(ptep); + dsb(ishst); + __tlbi_level(vae2is, __TLBI_VADDR(addr, 0), level); + } else { + if (end - addr < granule) + return -EINVAL; + + kvm_clear_pte(ptep); + dsb(ishst); + __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level); + data->unmapped += granule; + } + + dsb(ish); + isb(); + mm_ops->put_page(ptep); + + if (childp) + mm_ops->put_page(childp); + + return 0; +} + +u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) +{ + struct hyp_unmap_data unmap_data = { + .mm_ops = pgt->mm_ops, + }; + struct kvm_pgtable_walker walker = { + .cb = hyp_unmap_walker, + .arg = &unmap_data, + .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, + }; + + if (!pgt->mm_ops->page_count) + return 0; + + kvm_pgtable_walk(pgt, addr, size, &walker); + return unmap_data.unmapped; +} + int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, struct kvm_pgtable_mm_ops *mm_ops) { @@ -482,8 +536,16 @@ static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, enum kvm_pgtable_walk_flags flag, void * const arg) { struct kvm_pgtable_mm_ops *mm_ops = arg; + kvm_pte_t pte = *ptep; + + if (!kvm_pte_valid(pte)) + return 0; + + mm_ops->put_page(ptep); + + if (kvm_pte_table(pte, level)) + mm_ops->put_page(kvm_pte_follow(pte, mm_ops)); - mm_ops->put_page((void *)kvm_pte_follow(*ptep, mm_ops)); return 0; } @@ -491,7 +553,7 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt) { struct kvm_pgtable_walker walker = { .cb = hyp_free_walker, - .flags = KVM_PGTABLE_WALK_TABLE_POST, + .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, .arg = pgt->mm_ops, }; @@ -1110,13 +1172,13 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) } -int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_arch *arch, +int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, struct kvm_pgtable_mm_ops *mm_ops, enum kvm_pgtable_stage2_flags flags, kvm_pgtable_force_pte_cb_t force_pte_cb) { size_t pgd_sz; - u64 vtcr = arch->vtcr; + u64 vtcr = mmu->arch->vtcr; u32 ia_bits = VTCR_EL2_IPA(vtcr); u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; @@ -1129,7 +1191,7 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_arch *arch, pgt->ia_bits = ia_bits; pgt->start_level = start_level; pgt->mm_ops = mm_ops; - pgt->mmu = &arch->mmu; + pgt->mmu = mmu; pgt->flags = flags; pgt->force_pte_cb = force_pte_cb; diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index 39f8f7f922..4fb419f7b8 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -695,9 +695,7 @@ static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt) goto spurious; lr_val &= ~ICH_LR_STATE; - /* No active state for LPIs */ - if ((lr_val & ICH_LR_VIRTUAL_ID_MASK) <= VGIC_MAX_SPI) - lr_val |= ICH_LR_ACTIVE_BIT; + lr_val |= ICH_LR_ACTIVE_BIT; __gic_v3_set_lr(lr_val, lr); __vgic_v3_set_active_priority(lr_prio, vmcr, grp); vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); @@ -764,20 +762,18 @@ static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) /* Drop priority in any case */ act_prio = __vgic_v3_clear_highest_active_priority(); - /* If EOIing an LPI, no deactivate to be performed */ - if (vid >= VGIC_MIN_LPI) - return; - - /* EOImode == 1, nothing to be done here */ - if (vmcr & ICH_VMCR_EOIM_MASK) - return; - lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); if (lr == -1) { - __vgic_v3_bump_eoicount(); + /* Do not bump EOIcount for LPIs that aren't in the LRs */ + if (!(vid >= VGIC_MIN_LPI)) + __vgic_v3_bump_eoicount(); return; } + /* EOImode == 1 and not an LPI, nothing to be done here */ + if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI)) + return; + lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; /* If priorities or group do not match, the guest has fscked-up. */ @@ -988,7 +984,8 @@ static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) /* IDbits */ val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT; /* SEIS */ - val |= ((vtr >> 22) & 1) << ICC_CTLR_EL1_SEIS_SHIFT; + if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK) + val |= BIT(ICC_CTLR_EL1_SEIS_SHIFT); /* A3V */ val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT; /* EOImode */ diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index d88d3c143a..54af47005e 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -101,6 +101,26 @@ void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu) __deactivate_traps_common(vcpu); } +static const exit_handler_fn hyp_exit_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = NULL, + [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32, + [ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg, + [ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd, + [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, + [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low, + [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low, + [ESR_ELx_EC_PAC] = kvm_hyp_handle_ptrauth, +}; + +static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu) +{ + return hyp_exit_handlers; +} + +static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code) +{ +} + /* Switch to the guest for VHE systems running in EL2 */ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) { diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 69bd1732a2..bc2aba9532 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -210,13 +210,13 @@ static void stage2_flush_vm(struct kvm *kvm) { struct kvm_memslots *slots; struct kvm_memory_slot *memslot; - int idx; + int idx, bkt; idx = srcu_read_lock(&kvm->srcu); spin_lock(&kvm->mmu_lock); slots = kvm_memslots(kvm); - kvm_for_each_memslot(memslot, slots) + kvm_for_each_memslot(memslot, bkt, slots) stage2_flush_memslot(kvm, memslot); spin_unlock(&kvm->mmu_lock); @@ -239,6 +239,9 @@ void free_hyp_pgds(void) static bool kvm_host_owns_hyp_mappings(void) { + if (is_kernel_in_hyp_mode()) + return false; + if (static_branch_likely(&kvm_protected_mode_initialized)) return false; @@ -281,14 +284,117 @@ static phys_addr_t kvm_kaddr_to_phys(void *kaddr) } } -static int pkvm_share_hyp(phys_addr_t start, phys_addr_t end) +struct hyp_shared_pfn { + u64 pfn; + int count; + struct rb_node node; +}; + +static DEFINE_MUTEX(hyp_shared_pfns_lock); +static struct rb_root hyp_shared_pfns = RB_ROOT; + +static struct hyp_shared_pfn *find_shared_pfn(u64 pfn, struct rb_node ***node, + struct rb_node **parent) { - phys_addr_t addr; + struct hyp_shared_pfn *this; + + *node = &hyp_shared_pfns.rb_node; + *parent = NULL; + while (**node) { + this = container_of(**node, struct hyp_shared_pfn, node); + *parent = **node; + if (this->pfn < pfn) + *node = &((**node)->rb_left); + else if (this->pfn > pfn) + *node = &((**node)->rb_right); + else + return this; + } + + return NULL; +} + +static int share_pfn_hyp(u64 pfn) +{ + struct rb_node **node, *parent; + struct hyp_shared_pfn *this; + int ret = 0; + + mutex_lock(&hyp_shared_pfns_lock); + this = find_shared_pfn(pfn, &node, &parent); + if (this) { + this->count++; + goto unlock; + } + + this = kzalloc(sizeof(*this), GFP_KERNEL); + if (!this) { + ret = -ENOMEM; + goto unlock; + } + + this->pfn = pfn; + this->count = 1; + rb_link_node(&this->node, parent, node); + rb_insert_color(&this->node, &hyp_shared_pfns); + ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn, 1); +unlock: + mutex_unlock(&hyp_shared_pfns_lock); + + return ret; +} + +static int unshare_pfn_hyp(u64 pfn) +{ + struct rb_node **node, *parent; + struct hyp_shared_pfn *this; + int ret = 0; + + mutex_lock(&hyp_shared_pfns_lock); + this = find_shared_pfn(pfn, &node, &parent); + if (WARN_ON(!this)) { + ret = -ENOENT; + goto unlock; + } + + this->count--; + if (this->count) + goto unlock; + + rb_erase(&this->node, &hyp_shared_pfns); + kfree(this); + ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn, 1); +unlock: + mutex_unlock(&hyp_shared_pfns_lock); + + return ret; +} + +int kvm_share_hyp(void *from, void *to) +{ + phys_addr_t start, end, cur; + u64 pfn; int ret; - for (addr = ALIGN_DOWN(start, PAGE_SIZE); addr < end; addr += PAGE_SIZE) { - ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, - __phys_to_pfn(addr)); + if (is_kernel_in_hyp_mode()) + return 0; + + /* + * The share hcall maps things in the 'fixed-offset' region of the hyp + * VA space, so we can only share physically contiguous data-structures + * for now. + */ + if (is_vmalloc_or_module_addr(from) || is_vmalloc_or_module_addr(to)) + return -EINVAL; + + if (kvm_host_owns_hyp_mappings()) + return create_hyp_mappings(from, to, PAGE_HYP); + + start = ALIGN_DOWN(__pa(from), PAGE_SIZE); + end = PAGE_ALIGN(__pa(to)); + for (cur = start; cur < end; cur += PAGE_SIZE) { + pfn = __phys_to_pfn(cur); + ret = share_pfn_hyp(pfn); if (ret) return ret; } @@ -296,6 +402,22 @@ static int pkvm_share_hyp(phys_addr_t start, phys_addr_t end) return 0; } +void kvm_unshare_hyp(void *from, void *to) +{ + phys_addr_t start, end, cur; + u64 pfn; + + if (is_kernel_in_hyp_mode() || kvm_host_owns_hyp_mappings() || !from) + return; + + start = ALIGN_DOWN(__pa(from), PAGE_SIZE); + end = PAGE_ALIGN(__pa(to)); + for (cur = start; cur < end; cur += PAGE_SIZE) { + pfn = __phys_to_pfn(cur); + WARN_ON(unshare_pfn_hyp(pfn)); + } +} + /** * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode * @from: The virtual kernel start address of the range @@ -316,12 +438,8 @@ int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot) if (is_kernel_in_hyp_mode()) return 0; - if (!kvm_host_owns_hyp_mappings()) { - if (WARN_ON(prot != PAGE_HYP)) - return -EPERM; - return pkvm_share_hyp(kvm_kaddr_to_phys(from), - kvm_kaddr_to_phys(to)); - } + if (!kvm_host_owns_hyp_mappings()) + return -EPERM; start = start & PAGE_MASK; end = PAGE_ALIGN(end); @@ -407,6 +525,9 @@ int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, unsigned long addr; int ret; + if (is_protected_kvm_enabled()) + return -EPERM; + *kaddr = ioremap(phys_addr, size); if (!*kaddr) return -ENOMEM; @@ -512,11 +633,12 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) return -EINVAL; } - pgt = kzalloc(sizeof(*pgt), GFP_KERNEL); + pgt = kzalloc(sizeof(*pgt), GFP_KERNEL_ACCOUNT); if (!pgt) return -ENOMEM; - err = kvm_pgtable_stage2_init(pgt, &kvm->arch, &kvm_s2_mm_ops); + mmu->arch = &kvm->arch; + err = kvm_pgtable_stage2_init(pgt, mmu, &kvm_s2_mm_ops); if (err) goto out_free_pgtable; @@ -529,7 +651,6 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) for_each_possible_cpu(cpu) *per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1; - mmu->arch = &kvm->arch; mmu->pgt = pgt; mmu->pgd_phys = __pa(pgt->pgd); WRITE_ONCE(mmu->vmid.vmid_gen, 0); @@ -595,14 +716,14 @@ void stage2_unmap_vm(struct kvm *kvm) { struct kvm_memslots *slots; struct kvm_memory_slot *memslot; - int idx; + int idx, bkt; idx = srcu_read_lock(&kvm->srcu); mmap_read_lock(current->mm); spin_lock(&kvm->mmu_lock); slots = kvm_memslots(kvm); - kvm_for_each_memslot(memslot, slots) + kvm_for_each_memslot(memslot, bkt, slots) stage2_unmap_memslot(kvm, memslot); spin_unlock(&kvm->mmu_lock); @@ -650,6 +771,9 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, KVM_PGTABLE_PROT_R | (writable ? KVM_PGTABLE_PROT_W : 0); + if (is_protected_kvm_enabled()) + return -EPERM; + size += offset_in_page(guest_ipa); guest_ipa &= PAGE_MASK; @@ -1463,7 +1587,6 @@ int kvm_mmu_init(u32 *hyp_va_bits) } void kvm_arch_commit_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) @@ -1473,25 +1596,24 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, * allocated dirty_bitmap[], dirty pages will be tracked while the * memory slot is write protected. */ - if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { + if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) { /* * If we're with initial-all-set, we don't need to write * protect any pages because they're all reported as dirty. * Huge pages and normal pages will be write protect gradually. */ if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) { - kvm_mmu_wp_memory_region(kvm, mem->slot); + kvm_mmu_wp_memory_region(kvm, new->id); } } } int kvm_arch_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, enum kvm_mr_change change) { - hva_t hva = mem->userspace_addr; - hva_t reg_end = hva + mem->memory_size; + hva_t hva, reg_end; int ret = 0; if (change != KVM_MR_CREATE && change != KVM_MR_MOVE && @@ -1502,9 +1624,12 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, * Prevent userspace from creating a memory region outside of the IPA * space addressable by the KVM guest IPA space. */ - if ((memslot->base_gfn + memslot->npages) > (kvm_phys_size(kvm) >> PAGE_SHIFT)) + if ((new->base_gfn + new->npages) > (kvm_phys_size(kvm) >> PAGE_SHIFT)) return -EFAULT; + hva = new->userspace_addr; + reg_end = hva + (new->npages << PAGE_SHIFT); + mmap_read_lock(current->mm); /* * A memory region could potentially cover multiple VMAs, and any holes @@ -1536,7 +1661,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, if (vma->vm_flags & VM_PFNMAP) { /* IO region dirty page logging not allowed */ - if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES) { + if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) { ret = -EINVAL; break; } diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 2af3c37445..fbcfd4ec6f 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -14,6 +14,8 @@ #include #include +DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available); + static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx); static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx); static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc); @@ -28,6 +30,7 @@ static u32 kvm_pmu_event_mask(struct kvm *kvm) case ID_AA64DFR0_PMUVER_8_1: case ID_AA64DFR0_PMUVER_8_4: case ID_AA64DFR0_PMUVER_8_5: + case ID_AA64DFR0_PMUVER_8_7: return GENMASK(15, 0); default: /* Shouldn't be here, just for sanity */ WARN_ONCE(1, "Unknown PMU version %d\n", kvm->arch.pmuver); @@ -900,7 +903,7 @@ static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu) */ static bool pmu_irq_is_valid(struct kvm *kvm, int irq) { - int i; + unsigned long i; struct kvm_vcpu *vcpu; kvm_for_each_vcpu(i, vcpu, kvm) { @@ -978,7 +981,7 @@ int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) mutex_lock(&vcpu->kvm->lock); if (!vcpu->kvm->arch.pmu_filter) { - vcpu->kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL); + vcpu->kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT); if (!vcpu->kvm->arch.pmu_filter) { mutex_unlock(&vcpu->kvm->lock); return -ENOMEM; diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 44efe12dfc..5918095c90 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -46,8 +46,7 @@ static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) * specification (ARM DEN 0022A). This means all suspend states * for KVM will preserve the register state. */ - kvm_vcpu_block(vcpu); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); + kvm_vcpu_wfi(vcpu); return PSCI_RET_SUCCESS; } @@ -109,7 +108,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) /* * Make sure the reset request is observed if the change to - * power_state is observed. + * power_off is observed. */ smp_wmb(); @@ -121,8 +120,8 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) { - int i, matching_cpus = 0; - unsigned long mpidr; + int matching_cpus = 0; + unsigned long i, mpidr; unsigned long target_affinity; unsigned long target_affinity_mask; unsigned long lowest_affinity_level; @@ -164,7 +163,7 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type) { - int i; + unsigned long i; struct kvm_vcpu *tmp; /* diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 5ce36b0a33..ecc40c8cd6 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -46,22 +46,22 @@ unsigned int kvm_sve_max_vl; int kvm_arm_init_sve(void) { if (system_supports_sve()) { - kvm_sve_max_vl = sve_max_virtualisable_vl; + kvm_sve_max_vl = sve_max_virtualisable_vl(); /* * The get_sve_reg()/set_sve_reg() ioctl interface will need * to be extended with multiple register slice support in * order to support vector lengths greater than - * SVE_VL_ARCH_MAX: + * VL_ARCH_MAX: */ - if (WARN_ON(kvm_sve_max_vl > SVE_VL_ARCH_MAX)) - kvm_sve_max_vl = SVE_VL_ARCH_MAX; + if (WARN_ON(kvm_sve_max_vl > VL_ARCH_MAX)) + kvm_sve_max_vl = VL_ARCH_MAX; /* * Don't even try to make use of vector lengths that * aren't available on all CPUs, for now: */ - if (kvm_sve_max_vl < sve_max_vl) + if (kvm_sve_max_vl < sve_max_vl()) pr_warn("KVM: SVE vector length for guests limited to %u bytes\n", kvm_sve_max_vl); } @@ -94,22 +94,31 @@ static int kvm_vcpu_finalize_sve(struct kvm_vcpu *vcpu) { void *buf; unsigned int vl; + size_t reg_sz; + int ret; vl = vcpu->arch.sve_max_vl; /* * Responsibility for these properties is shared between - * kvm_arm_init_arch_resources(), kvm_vcpu_enable_sve() and + * kvm_arm_init_sve(), kvm_vcpu_enable_sve() and * set_sve_vls(). Double-check here just to be sure: */ - if (WARN_ON(!sve_vl_valid(vl) || vl > sve_max_virtualisable_vl || - vl > SVE_VL_ARCH_MAX)) + if (WARN_ON(!sve_vl_valid(vl) || vl > sve_max_virtualisable_vl() || + vl > VL_ARCH_MAX)) return -EIO; - buf = kzalloc(SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl)), GFP_KERNEL); + reg_sz = vcpu_sve_state_size(vcpu); + buf = kzalloc(reg_sz, GFP_KERNEL_ACCOUNT); if (!buf) return -ENOMEM; + ret = kvm_share_hyp(buf, buf + reg_sz); + if (ret) { + kfree(buf); + return ret; + } + vcpu->arch.sve_state = buf; vcpu->arch.flags |= KVM_ARM64_VCPU_SVE_FINALIZED; return 0; @@ -141,7 +150,13 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu) void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu) { - kfree(vcpu->arch.sve_state); + void *sve_state = vcpu->arch.sve_state; + + kvm_vcpu_unshare_task_fp(vcpu); + kvm_unshare_hyp(vcpu, vcpu + 1); + if (sve_state) + kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu)); + kfree(sve_state); } static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu) @@ -170,7 +185,7 @@ static bool vcpu_allowed_register_width(struct kvm_vcpu *vcpu) { struct kvm_vcpu *tmp; bool is32bit; - int i; + unsigned long i; is32bit = vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT); if (!cpus_have_const_cap(ARM64_HAS_32BIT_EL1) && is32bit) @@ -193,10 +208,9 @@ static bool vcpu_allowed_register_width(struct kvm_vcpu *vcpu) * kvm_reset_vcpu - sets core registers and sys_regs to reset value * @vcpu: The VCPU pointer * - * This function finds the right table above and sets the registers on - * the virtual CPU struct to their architecturally defined reset - * values, except for registers whose reset is deferred until - * kvm_arm_vcpu_finalize(). + * This function sets the registers on the virtual CPU struct to their + * architecturally defined reset values, except for registers whose reset is + * deferred until kvm_arm_vcpu_finalize(). * * Note: This function can be called from two paths: The KVM_ARM_VCPU_INIT * ioctl or as part of handling a request issued by another VCPU in the PSCI diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 7c18e429b4..4dc2fba316 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1064,7 +1064,12 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r, bool raz) { u32 id = reg_to_encoding(r); - u64 val = raz ? 0 : read_sanitised_ftr_reg(id); + u64 val; + + if (raz) + return 0; + + val = read_sanitised_ftr_reg(id); switch (id) { case SYS_ID_AA64PFR0_EL1: @@ -1075,16 +1080,15 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2); val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3); val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3); + if (irqchip_in_kernel(vcpu->kvm) && + vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_GIC); + val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_GIC), 1); + } break; case SYS_ID_AA64PFR1_EL1: - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE); - if (kvm_has_mte(vcpu->kvm)) { - u64 pfr, mte; - - pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); - mte = cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR1_MTE_SHIFT); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR1_MTE), mte); - } + if (!kvm_has_mte(vcpu->kvm)) + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE); break; case SYS_ID_AA64ISAR1_EL1: if (!vcpu_has_ptrauth(vcpu)) @@ -1268,18 +1272,21 @@ static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, return __set_id_reg(vcpu, rd, uaddr, raz); } -static int get_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, - const struct kvm_one_reg *reg, void __user *uaddr) -{ - return __get_id_reg(vcpu, rd, uaddr, true); -} - static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, const struct kvm_one_reg *reg, void __user *uaddr) { return __set_id_reg(vcpu, rd, uaddr, true); } +static int get_raz_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr) +{ + const u64 id = sys_reg_to_index(rd); + const u64 val = 0; + + return reg_to_user(uaddr, &val, id); +} + static int set_wi_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, const struct kvm_one_reg *reg, void __user *uaddr) { @@ -1388,7 +1395,7 @@ static unsigned int mte_visibility(const struct kvm_vcpu *vcpu, #define ID_UNALLOCATED(crm, op2) { \ Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \ .access = access_raz_id_reg, \ - .get_user = get_raz_id_reg, \ + .get_user = get_raz_reg, \ .set_user = set_raz_id_reg, \ } @@ -1400,7 +1407,7 @@ static unsigned int mte_visibility(const struct kvm_vcpu *vcpu, #define ID_HIDDEN(name) { \ SYS_DESC(SYS_##name), \ .access = access_raz_id_reg, \ - .get_user = get_raz_id_reg, \ + .get_user = get_raz_reg, \ .set_user = set_raz_id_reg, \ } @@ -1642,7 +1649,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { * previously (and pointlessly) advertised in the past... */ { PMU_SYS_REG(SYS_PMSWINC_EL0), - .get_user = get_raz_id_reg, .set_user = set_wi_reg, + .get_user = get_raz_reg, .set_user = set_wi_reg, .access = access_pmswinc, .reset = NULL }, { PMU_SYS_REG(SYS_PMSELR_EL0), .access = access_pmselr, .reset = reset_pmselr, .reg = PMSELR_EL0 }, diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 340c51d876..fc00304fe7 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -70,8 +70,9 @@ void kvm_vgic_early_init(struct kvm *kvm) */ int kvm_vgic_create(struct kvm *kvm, u32 type) { - int i, ret; struct kvm_vcpu *vcpu; + unsigned long i; + int ret; if (irqchip_in_kernel(kvm)) return -EEXIST; @@ -91,7 +92,7 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) return ret; kvm_for_each_vcpu(i, vcpu, kvm) { - if (vcpu->arch.has_run_once) + if (vcpu_has_run_once(vcpu)) goto out_unlock; } ret = 0; @@ -134,7 +135,7 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis) struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); int i; - dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL); + dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL_ACCOUNT); if (!dist->spis) return -ENOMEM; @@ -255,7 +256,8 @@ int vgic_init(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; struct kvm_vcpu *vcpu; - int ret = 0, i, idx; + int ret = 0, i; + unsigned long idx; if (vgic_initialized(kvm)) return 0; @@ -308,7 +310,7 @@ int vgic_init(struct kvm *kvm) goto out; } - kvm_for_each_vcpu(i, vcpu, kvm) + kvm_for_each_vcpu(idx, vcpu, kvm) kvm_vgic_vcpu_enable(vcpu); ret = kvm_vgic_setup_default_irq_routing(kvm); @@ -370,7 +372,7 @@ void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu) static void __kvm_vgic_destroy(struct kvm *kvm) { struct kvm_vcpu *vcpu; - int i; + unsigned long i; vgic_debug_destroy(kvm); diff --git a/arch/arm64/kvm/vgic/vgic-irqfd.c b/arch/arm64/kvm/vgic/vgic-irqfd.c index 79f8899b23..475059bace 100644 --- a/arch/arm64/kvm/vgic/vgic-irqfd.c +++ b/arch/arm64/kvm/vgic/vgic-irqfd.c @@ -139,7 +139,7 @@ int kvm_vgic_setup_default_irq_routing(struct kvm *kvm) u32 nr = dist->nr_spis; int i, ret; - entries = kcalloc(nr, sizeof(*entries), GFP_KERNEL); + entries = kcalloc(nr, sizeof(*entries), GFP_KERNEL_ACCOUNT); if (!entries) return -ENOMEM; diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 61728c543e..089fc2ffcb 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -48,7 +48,7 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, if (irq) return irq; - irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL); + irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL_ACCOUNT); if (!irq) return ERR_PTR(-ENOMEM); @@ -332,7 +332,7 @@ int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr) * we must be careful not to overrun the array. */ irq_count = READ_ONCE(dist->lpi_list_count); - intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL); + intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL_ACCOUNT); if (!intids) return -ENOMEM; @@ -985,7 +985,7 @@ static int vgic_its_alloc_collection(struct vgic_its *its, if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL)) return E_ITS_MAPC_COLLECTION_OOR; - collection = kzalloc(sizeof(*collection), GFP_KERNEL); + collection = kzalloc(sizeof(*collection), GFP_KERNEL_ACCOUNT); if (!collection) return -ENOMEM; @@ -1029,7 +1029,7 @@ static struct its_ite *vgic_its_alloc_ite(struct its_device *device, { struct its_ite *ite; - ite = kzalloc(sizeof(*ite), GFP_KERNEL); + ite = kzalloc(sizeof(*ite), GFP_KERNEL_ACCOUNT); if (!ite) return ERR_PTR(-ENOMEM); @@ -1150,7 +1150,7 @@ static struct its_device *vgic_its_alloc_device(struct vgic_its *its, { struct its_device *device; - device = kzalloc(sizeof(*device), GFP_KERNEL); + device = kzalloc(sizeof(*device), GFP_KERNEL_ACCOUNT); if (!device) return ERR_PTR(-ENOMEM); @@ -1847,7 +1847,7 @@ void vgic_lpi_translation_cache_init(struct kvm *kvm) struct vgic_translation_cache_entry *cte; /* An allocation failure is not fatal */ - cte = kzalloc(sizeof(*cte), GFP_KERNEL); + cte = kzalloc(sizeof(*cte), GFP_KERNEL_ACCOUNT); if (WARN_ON(!cte)) break; @@ -1888,7 +1888,7 @@ static int vgic_its_create(struct kvm_device *dev, u32 type) if (type != KVM_DEV_TYPE_ARM_VGIC_ITS) return -ENODEV; - its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL); + its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT); if (!its) return -ENOMEM; @@ -2710,8 +2710,8 @@ static int vgic_its_set_attr(struct kvm_device *dev, if (copy_from_user(&addr, uaddr, sizeof(addr))) return -EFAULT; - ret = vgic_check_ioaddr(dev->kvm, &its->vgic_its_base, - addr, SZ_64K); + ret = vgic_check_iorange(dev->kvm, its->vgic_its_base, + addr, SZ_64K, KVM_VGIC_V3_ITS_SIZE); if (ret) return ret; diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c index 7740995de9..c6d52a1fd9 100644 --- a/arch/arm64/kvm/vgic/vgic-kvm-device.c +++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c @@ -14,17 +14,21 @@ /* common helpers */ -int vgic_check_ioaddr(struct kvm *kvm, phys_addr_t *ioaddr, - phys_addr_t addr, phys_addr_t alignment) +int vgic_check_iorange(struct kvm *kvm, phys_addr_t ioaddr, + phys_addr_t addr, phys_addr_t alignment, + phys_addr_t size) { - if (addr & ~kvm_phys_mask(kvm)) - return -E2BIG; + if (!IS_VGIC_ADDR_UNDEF(ioaddr)) + return -EEXIST; - if (!IS_ALIGNED(addr, alignment)) + if (!IS_ALIGNED(addr, alignment) || !IS_ALIGNED(size, alignment)) return -EINVAL; - if (!IS_VGIC_ADDR_UNDEF(*ioaddr)) - return -EEXIST; + if (addr + size < addr) + return -EINVAL; + + if (addr & ~kvm_phys_mask(kvm) || addr + size > kvm_phys_size(kvm)) + return -E2BIG; return 0; } @@ -57,7 +61,7 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write) { int r = 0; struct vgic_dist *vgic = &kvm->arch.vgic; - phys_addr_t *addr_ptr, alignment; + phys_addr_t *addr_ptr, alignment, size; u64 undef_value = VGIC_ADDR_UNDEF; mutex_lock(&kvm->lock); @@ -66,16 +70,19 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write) r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V2); addr_ptr = &vgic->vgic_dist_base; alignment = SZ_4K; + size = KVM_VGIC_V2_DIST_SIZE; break; case KVM_VGIC_V2_ADDR_TYPE_CPU: r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V2); addr_ptr = &vgic->vgic_cpu_base; alignment = SZ_4K; + size = KVM_VGIC_V2_CPU_SIZE; break; case KVM_VGIC_V3_ADDR_TYPE_DIST: r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V3); addr_ptr = &vgic->vgic_dist_base; alignment = SZ_64K; + size = KVM_VGIC_V3_DIST_SIZE; break; case KVM_VGIC_V3_ADDR_TYPE_REDIST: { struct vgic_redist_region *rdreg; @@ -140,7 +147,7 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write) goto out; if (write) { - r = vgic_check_ioaddr(kvm, addr_ptr, *addr, alignment); + r = vgic_check_iorange(kvm, *addr_ptr, *addr, alignment, size); if (!r) *addr_ptr = *addr; } else { @@ -318,7 +325,7 @@ void unlock_all_vcpus(struct kvm *kvm) bool lock_all_vcpus(struct kvm *kvm) { struct kvm_vcpu *tmp_vcpu; - int c; + unsigned long c; /* * Any time a vcpu is run, vcpu_load is called which tries to grab the diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v2.c b/arch/arm64/kvm/vgic/vgic-mmio-v2.c index 5f9014ae59..12e4c223e6 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v2.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v2.c @@ -113,9 +113,8 @@ static void vgic_mmio_write_sgir(struct kvm_vcpu *source_vcpu, int intid = val & 0xf; int targets = (val >> 16) & 0xff; int mode = (val >> 24) & 0x03; - int c; struct kvm_vcpu *vcpu; - unsigned long flags; + unsigned long flags, c; switch (mode) { case 0x0: /* as specified by targets */ diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index a09cdc0b95..58e40b4874 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -754,7 +754,8 @@ static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu) static int vgic_register_all_redist_iodevs(struct kvm *kvm) { struct kvm_vcpu *vcpu; - int c, ret = 0; + unsigned long c; + int ret = 0; kvm_for_each_vcpu(c, vcpu, kvm) { ret = vgic_register_redist_iodev(vcpu); @@ -763,10 +764,12 @@ static int vgic_register_all_redist_iodevs(struct kvm *kvm) } if (ret) { - /* The current c failed, so we start with the previous one. */ + /* The current c failed, so iterate over the previous ones. */ + int i; + mutex_lock(&kvm->slots_lock); - for (c--; c >= 0; c--) { - vcpu = kvm_get_vcpu(kvm, c); + for (i = 0; i < c; i++) { + vcpu = kvm_get_vcpu(kvm, i); vgic_unregister_redist_iodev(vcpu); } mutex_unlock(&kvm->slots_lock); @@ -796,7 +799,9 @@ static int vgic_v3_alloc_redist_region(struct kvm *kvm, uint32_t index, struct vgic_dist *d = &kvm->arch.vgic; struct vgic_redist_region *rdreg; struct list_head *rd_regions = &d->rd_regions; - size_t size = count * KVM_VGIC_V3_REDIST_SIZE; + int nr_vcpus = atomic_read(&kvm->online_vcpus); + size_t size = count ? count * KVM_VGIC_V3_REDIST_SIZE + : nr_vcpus * KVM_VGIC_V3_REDIST_SIZE; int ret; /* cross the end of memory ? */ @@ -834,13 +839,13 @@ static int vgic_v3_alloc_redist_region(struct kvm *kvm, uint32_t index, if (vgic_v3_rdist_overlap(kvm, base, size)) return -EINVAL; - rdreg = kzalloc(sizeof(*rdreg), GFP_KERNEL); + rdreg = kzalloc(sizeof(*rdreg), GFP_KERNEL_ACCOUNT); if (!rdreg) return -ENOMEM; rdreg->base = VGIC_ADDR_UNDEF; - ret = vgic_check_ioaddr(kvm, &rdreg->base, base, SZ_64K); + ret = vgic_check_iorange(kvm, rdreg->base, base, SZ_64K, size); if (ret) goto free; @@ -993,10 +998,10 @@ void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1) struct kvm_vcpu *c_vcpu; u16 target_cpus; u64 mpidr; - int sgi, c; + int sgi; int vcpu_id = vcpu->vcpu_id; bool broadcast; - unsigned long flags; + unsigned long c, flags; sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT; broadcast = reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT); diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c index f972992682..49837d3a3e 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.c +++ b/arch/arm64/kvm/vgic/vgic-mmio.c @@ -1052,7 +1052,7 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, return 0; } -struct kvm_io_device_ops kvm_io_gic_ops = { +const struct kvm_io_device_ops kvm_io_gic_ops = { .read = dispatch_mmio_read, .write = dispatch_mmio_write, }; diff --git a/arch/arm64/kvm/vgic/vgic-mmio.h b/arch/arm64/kvm/vgic/vgic-mmio.h index fefcca2b14..3fa696f198 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.h +++ b/arch/arm64/kvm/vgic/vgic-mmio.h @@ -34,7 +34,7 @@ struct vgic_register_region { }; }; -extern struct kvm_io_device_ops kvm_io_gic_ops; +extern const struct kvm_io_device_ops kvm_io_gic_ops; #define VGIC_ACCESS_8bit 1 #define VGIC_ACCESS_32bit 2 diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c index 95a18cec14..645648349c 100644 --- a/arch/arm64/kvm/vgic/vgic-v2.c +++ b/arch/arm64/kvm/vgic/vgic-v2.c @@ -293,12 +293,12 @@ int vgic_v2_map_resources(struct kvm *kvm) if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) || IS_VGIC_ADDR_UNDEF(dist->vgic_cpu_base)) { - kvm_err("Need to set vgic cpu and dist addresses first\n"); + kvm_debug("Need to set vgic cpu and dist addresses first\n"); return -ENXIO; } if (!vgic_v2_check_base(dist->vgic_dist_base, dist->vgic_cpu_base)) { - kvm_err("VGIC CPU and dist frames overlap\n"); + kvm_debug("VGIC CPU and dist frames overlap\n"); return -EINVAL; } @@ -345,6 +345,11 @@ int vgic_v2_probe(const struct gic_kvm_info *info) int ret; u32 vtr; + if (is_protected_kvm_enabled()) { + kvm_err("GICv2 not supported in protected mode\n"); + return -ENXIO; + } + if (!info->vctrl.start) { kvm_err("GICH not present in the firmware table\n"); return -ENXIO; diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 21a6207fb2..b549af8b1d 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -15,6 +15,7 @@ static bool group0_trap; static bool group1_trap; static bool common_trap; +static bool dir_trap; static bool gicv4_enable; void vgic_v3_set_underflow(struct kvm_vcpu *vcpu) @@ -296,6 +297,8 @@ void vgic_v3_enable(struct kvm_vcpu *vcpu) vgic_v3->vgic_hcr |= ICH_HCR_TALL1; if (common_trap) vgic_v3->vgic_hcr |= ICH_HCR_TC; + if (dir_trap) + vgic_v3->vgic_hcr |= ICH_HCR_TDIR; } int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq) @@ -483,8 +486,10 @@ bool vgic_v3_check_base(struct kvm *kvm) return false; list_for_each_entry(rdreg, &d->rd_regions, list) { - if (rdreg->base + vgic_v3_rd_region_size(kvm, rdreg) < - rdreg->base) + size_t sz = vgic_v3_rd_region_size(kvm, rdreg); + + if (vgic_check_iorange(kvm, VGIC_ADDR_UNDEF, + rdreg->base, SZ_64K, sz)) return false; } @@ -537,24 +542,24 @@ int vgic_v3_map_resources(struct kvm *kvm) struct vgic_dist *dist = &kvm->arch.vgic; struct kvm_vcpu *vcpu; int ret = 0; - int c; + unsigned long c; kvm_for_each_vcpu(c, vcpu, kvm) { struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; if (IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr)) { - kvm_debug("vcpu %d redistributor base not set\n", c); + kvm_debug("vcpu %ld redistributor base not set\n", c); return -ENXIO; } } if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base)) { - kvm_err("Need to set vgic distributor addresses first\n"); + kvm_debug("Need to set vgic distributor addresses first\n"); return -ENXIO; } if (!vgic_v3_check_base(kvm)) { - kvm_err("VGIC redist and dist frames overlap\n"); + kvm_debug("VGIC redist and dist frames overlap\n"); return -EINVAL; } @@ -604,6 +609,18 @@ static int __init early_gicv4_enable(char *buf) } early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable); +static const struct midr_range broken_seis[] = { + MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM), + MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM), + {}, +}; + +static bool vgic_v3_broken_seis(void) +{ + return ((kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK) && + is_midr_in_range_list(read_cpuid_id(), broken_seis)); +} + /** * vgic_v3_probe - probe for a VGICv3 compatible interrupt controller * @info: pointer to the GIC description @@ -646,7 +663,7 @@ int vgic_v3_probe(const struct gic_kvm_info *info) } else if (!PAGE_ALIGNED(info->vcpu.start)) { pr_warn("GICV physical address 0x%llx not page aligned\n", (unsigned long long)info->vcpu.start); - } else { + } else if (kvm_get_mode() != KVM_MODE_PROTECTED) { kvm_vgic_global_state.vcpu_base = info->vcpu.start; kvm_vgic_global_state.can_emulate_gicv2 = true; ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2); @@ -671,11 +688,24 @@ int vgic_v3_probe(const struct gic_kvm_info *info) group1_trap = true; } - if (group0_trap || group1_trap || common_trap) { - kvm_info("GICv3 sysreg trapping enabled ([%s%s%s], reduced performance)\n", + if (vgic_v3_broken_seis()) { + kvm_info("GICv3 with broken locally generated SEI\n"); + + kvm_vgic_global_state.ich_vtr_el2 &= ~ICH_VTR_SEIS_MASK; + group0_trap = true; + group1_trap = true; + if (ich_vtr_el2 & ICH_VTR_TDS_MASK) + dir_trap = true; + else + common_trap = true; + } + + if (group0_trap || group1_trap || common_trap | dir_trap) { + kvm_info("GICv3 sysreg trapping enabled ([%s%s%s%s], reduced performance)\n", group0_trap ? "G0" : "", group1_trap ? "G1" : "", - common_trap ? "C" : ""); + common_trap ? "C" : "", + dir_trap ? "D" : ""); static_branch_enable(&vgic_v3_cpuif_trap); } diff --git a/arch/arm64/kvm/vgic/vgic-v4.c b/arch/arm64/kvm/vgic/vgic-v4.c index c1845d8f5f..ad06ba6c9b 100644 --- a/arch/arm64/kvm/vgic/vgic-v4.c +++ b/arch/arm64/kvm/vgic/vgic-v4.c @@ -189,7 +189,7 @@ void vgic_v4_configure_vsgis(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; struct kvm_vcpu *vcpu; - int i; + unsigned long i; kvm_arm_halt_guest(kvm); @@ -235,7 +235,8 @@ int vgic_v4_init(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; struct kvm_vcpu *vcpu; - int i, nr_vcpus, ret; + int nr_vcpus, ret; + unsigned long i; if (!kvm_vgic_global_state.has_gicv4) return 0; /* Nothing to see here... move along. */ @@ -246,7 +247,7 @@ int vgic_v4_init(struct kvm *kvm) nr_vcpus = atomic_read(&kvm->online_vcpus); dist->its_vm.vpes = kcalloc(nr_vcpus, sizeof(*dist->its_vm.vpes), - GFP_KERNEL); + GFP_KERNEL_ACCOUNT); if (!dist->its_vm.vpes) return -ENOMEM; diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 5dad4996cf..9b98876a8a 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -990,7 +990,7 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu) void vgic_kick_vcpus(struct kvm *kvm) { struct kvm_vcpu *vcpu; - int c; + unsigned long c; /* * We've injected an interrupt, time to find out who deserves diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index 14a9218641..3fd6c86a7e 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -172,8 +172,9 @@ void vgic_kick_vcpus(struct kvm *kvm); void vgic_irq_handle_resampling(struct vgic_irq *irq, bool lr_deactivated, bool lr_pending); -int vgic_check_ioaddr(struct kvm *kvm, phys_addr_t *ioaddr, - phys_addr_t addr, phys_addr_t alignment); +int vgic_check_iorange(struct kvm *kvm, phys_addr_t ioaddr, + phys_addr_t addr, phys_addr_t alignment, + phys_addr_t size); void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu); void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr); diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 0941180a86..29490be254 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -9,6 +9,8 @@ ifeq ($(CONFIG_KERNEL_MODE_NEON), y) obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o CFLAGS_REMOVE_xor-neon.o += -mgeneral-regs-only CFLAGS_xor-neon.o += -ffreestanding +# Enable +CFLAGS_xor-neon.o += -isystem $(shell $(CC) -print-file-name=include) endif lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S index a7efb2ad2a..a5a5f5b97b 100644 --- a/arch/arm64/lib/clear_user.S +++ b/arch/arm64/lib/clear_user.S @@ -4,7 +4,7 @@ */ #include -#include +#include .text @@ -45,13 +45,11 @@ USER(9f, sttrh wzr, [x0]) USER(7f, sttrb wzr, [x2, #-1]) 5: mov x0, #0 ret -SYM_FUNC_END(__arch_clear_user) -EXPORT_SYMBOL(__arch_clear_user) - .section .fixup,"ax" - .align 2 + // Exception fixups 7: sub x0, x2, #5 // Adjust for faulting on the final byte... 8: add x0, x0, #4 // ...or the second word of the 4-7 byte case 9: sub x0, x2, x0 ret - .previous +SYM_FUNC_END(__arch_clear_user) +EXPORT_SYMBOL(__arch_clear_user) diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 2cf999e41d..34e3179075 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -60,11 +60,8 @@ SYM_FUNC_START(__arch_copy_from_user) #include "copy_template.S" mov x0, #0 // Nothing to copy ret -SYM_FUNC_END(__arch_copy_from_user) -EXPORT_SYMBOL(__arch_copy_from_user) - .section .fixup,"ax" - .align 2 + // Exception fixups 9997: cmp dst, dstin b.ne 9998f // Before being absolutely sure we couldn't copy anything, try harder @@ -72,4 +69,5 @@ USER(9998f, ldtrb tmp1w, [srcin]) strb tmp1w, [dst], #1 9998: sub x0, end, dst // bytes not copied ret - .previous +SYM_FUNC_END(__arch_copy_from_user) +EXPORT_SYMBOL(__arch_copy_from_user) diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index 9f380eecf6..8022317726 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -59,11 +59,8 @@ SYM_FUNC_START(__arch_copy_to_user) #include "copy_template.S" mov x0, #0 ret -SYM_FUNC_END(__arch_copy_to_user) -EXPORT_SYMBOL(__arch_copy_to_user) - .section .fixup,"ax" - .align 2 + // Exception fixups 9997: cmp dst, dstin b.ne 9998f // Before being absolutely sure we couldn't copy anything, try harder @@ -72,4 +69,5 @@ USER(9998f, sttrb tmp1w, [dst]) add dst, dst, #1 9998: sub x0, end, dst // bytes not copied ret - .previous +SYM_FUNC_END(__arch_copy_to_user) +EXPORT_SYMBOL(__arch_copy_to_user) diff --git a/arch/arm64/lib/kasan_sw_tags.S b/arch/arm64/lib/kasan_sw_tags.S index 5b04464c04..20784ce75d 100644 --- a/arch/arm64/lib/kasan_sw_tags.S +++ b/arch/arm64/lib/kasan_sw_tags.S @@ -38,9 +38,7 @@ * incremented by 256 prior to return). */ SYM_CODE_START(__hwasan_tag_mismatch) -#ifdef BTI_C - BTI_C -#endif + bti c add x29, sp, #232 stp x2, x3, [sp, #8 * 2] stp x4, x5, [sp, #8 * 4] diff --git a/arch/arm64/lib/xor-neon.c b/arch/arm64/lib/xor-neon.c index 11bf4f8aca..d189cf4e70 100644 --- a/arch/arm64/lib/xor-neon.c +++ b/arch/arm64/lib/xor-neon.c @@ -167,7 +167,7 @@ void xor_arm64_neon_5(unsigned long bytes, unsigned long *p1, } while (--lines > 0); } -struct xor_block_template const xor_block_inner_neon = { +struct xor_block_template xor_block_inner_neon __ro_after_init = { .name = "__inner_neon__", .do_2 = xor_arm64_neon_2, .do_3 = xor_arm64_neon_3, @@ -176,6 +176,151 @@ struct xor_block_template const xor_block_inner_neon = { }; EXPORT_SYMBOL(xor_block_inner_neon); +static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) +{ + uint64x2_t res; + + asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n" + "eor3 %0.16b, %1.16b, %2.16b, %3.16b" + : "=w"(res) : "w"(p), "w"(q), "w"(r)); + return res; +} + +static void xor_arm64_eor3_3(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + } while (--lines > 0); +} + +static void xor_arm64_eor3_4(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 */ + v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + } while (--lines > 0); +} + +static void xor_arm64_eor3_5(unsigned long bytes, unsigned long *p1, + unsigned long *p2, unsigned long *p3, + unsigned long *p4, unsigned long *p5) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + uint64_t *dp5 = (uint64_t *)p5; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 ^ p5 */ + v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0)); + v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2)); + v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4)); + v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + dp5 += 8; + } while (--lines > 0); +} + +static int __init xor_neon_init(void) +{ + if (IS_ENABLED(CONFIG_AS_HAS_SHA3) && cpu_have_named_feature(SHA3)) { + xor_block_inner_neon.do_3 = xor_arm64_eor3_3; + xor_block_inner_neon.do_4 = xor_arm64_eor3_4; + xor_block_inner_neon.do_5 = xor_arm64_eor3_5; + } + return 0; +} +module_init(xor_neon_init); + +static void __exit xor_neon_exit(void) +{ +} +module_exit(xor_neon_exit); + MODULE_AUTHOR("Jackie Liu "); MODULE_DESCRIPTION("ARMv8 XOR Extensions"); MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index f188c90926..ff1e800ba7 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -7,6 +7,7 @@ obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_PTDUMP_CORE) += ptdump.o obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o obj-$(CONFIG_TRANS_TABLE) += trans_pgd.o +obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o KASAN_SANITIZE_physaddr.o += n diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 5051b3c1a4..7d0563db42 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -140,15 +140,7 @@ SYM_FUNC_END(dcache_clean_pou) * - start - kernel start address of region * - end - kernel end address of region */ -SYM_FUNC_START_LOCAL(__dma_inv_area) SYM_FUNC_START_PI(dcache_inval_poc) - /* FALLTHROUGH */ - -/* - * __dma_inv_area(start, end) - * - start - virtual start address of region - * - end - virtual end address of region - */ dcache_line_size x2, x3 sub x3, x2, #1 tst x1, x3 // end cache line aligned? @@ -167,7 +159,6 @@ SYM_FUNC_START_PI(dcache_inval_poc) dsb sy ret SYM_FUNC_END_PI(dcache_inval_poc) -SYM_FUNC_END(__dma_inv_area) /* * dcache_clean_poc(start, end) @@ -178,19 +169,10 @@ SYM_FUNC_END(__dma_inv_area) * - start - virtual start address of region * - end - virtual end address of region */ -SYM_FUNC_START_LOCAL(__dma_clean_area) SYM_FUNC_START_PI(dcache_clean_poc) - /* FALLTHROUGH */ - -/* - * __dma_clean_area(start, end) - * - start - virtual start address of region - * - end - virtual end address of region - */ dcache_by_line_op cvac, sy, x0, x1, x2, x3 ret SYM_FUNC_END_PI(dcache_clean_poc) -SYM_FUNC_END(__dma_clean_area) /* * dcache_clean_pop(start, end) @@ -232,8 +214,8 @@ SYM_FUNC_END_PI(__dma_flush_area) SYM_FUNC_START_PI(__dma_map_area) add x1, x0, x1 cmp w2, #DMA_FROM_DEVICE - b.eq __dma_inv_area - b __dma_clean_area + b.eq __pi_dcache_inval_poc + b __pi_dcache_clean_poc SYM_FUNC_END_PI(__dma_map_area) /* @@ -245,6 +227,6 @@ SYM_FUNC_END_PI(__dma_map_area) SYM_FUNC_START_PI(__dma_unmap_area) add x1, x0, x1 cmp w2, #DMA_TO_DEVICE - b.ne __dma_inv_area + b.ne __pi_dcache_inval_poc ret SYM_FUNC_END_PI(__dma_unmap_area) diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index cd72576ae2..b8b4cf0bcf 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -35,8 +35,8 @@ static unsigned long *pinned_asid_map; #define ASID_FIRST_VERSION (1UL << asid_bits) #define NUM_USER_ASIDS ASID_FIRST_VERSION -#define asid2idx(asid) ((asid) & ~ASID_MASK) -#define idx2asid(idx) asid2idx(idx) +#define ctxid2asid(asid) ((asid) & ~ASID_MASK) +#define asid2ctxid(asid, genid) ((asid) | (genid)) /* Get the ASIDBits supported by the current CPU */ static u32 get_cpu_asid_bits(void) @@ -50,10 +50,10 @@ static u32 get_cpu_asid_bits(void) pr_warn("CPU%d: Unknown ASID size (%d); assuming 8-bit\n", smp_processor_id(), fld); fallthrough; - case 0: + case ID_AA64MMFR0_ASID_8: asid = 8; break; - case 2: + case ID_AA64MMFR0_ASID_16: asid = 16; } @@ -120,7 +120,7 @@ static void flush_context(void) */ if (asid == 0) asid = per_cpu(reserved_asids, i); - __set_bit(asid2idx(asid), asid_map); + __set_bit(ctxid2asid(asid), asid_map); per_cpu(reserved_asids, i) = asid; } @@ -162,7 +162,7 @@ static u64 new_context(struct mm_struct *mm) u64 generation = atomic64_read(&asid_generation); if (asid != 0) { - u64 newasid = generation | (asid & ~ASID_MASK); + u64 newasid = asid2ctxid(ctxid2asid(asid), generation); /* * If our current ASID was active during a rollover, we @@ -183,7 +183,7 @@ static u64 new_context(struct mm_struct *mm) * We had a valid ASID in a previous life, so try to re-use * it if possible. */ - if (!__test_and_set_bit(asid2idx(asid), asid_map)) + if (!__test_and_set_bit(ctxid2asid(asid), asid_map)) return newasid; } @@ -209,7 +209,7 @@ static u64 new_context(struct mm_struct *mm) set_asid: __set_bit(asid, asid_map); cur_idx = asid; - return idx2asid(asid) | generation; + return asid2ctxid(asid, generation); } void check_and_switch_context(struct mm_struct *mm) @@ -300,13 +300,13 @@ unsigned long arm64_mm_context_get(struct mm_struct *mm) } nr_pinned_asids++; - __set_bit(asid2idx(asid), pinned_asid_map); + __set_bit(ctxid2asid(asid), pinned_asid_map); refcount_set(&mm->context.pinned, 1); out_unlock: raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); - asid &= ~ASID_MASK; + asid = ctxid2asid(asid); /* Set the equivalent of USER_ASID_BIT */ if (asid && arm64_kernel_unmapped_at_el0()) @@ -327,7 +327,7 @@ void arm64_mm_context_put(struct mm_struct *mm) raw_spin_lock_irqsave(&cpu_asid_lock, flags); if (refcount_dec_and_test(&mm->context.pinned)) { - __clear_bit(asid2idx(asid), pinned_asid_map); + __clear_bit(ctxid2asid(asid), pinned_asid_map); nr_pinned_asids--; } diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c index 60a8b6a8a4..4894553096 100644 --- a/arch/arm64/mm/extable.c +++ b/arch/arm64/mm/extable.c @@ -3,25 +3,84 @@ * Based on arch/arm/mm/extable.c */ +#include #include #include -int fixup_exception(struct pt_regs *regs) +#include +#include + +static inline unsigned long +get_ex_fixup(const struct exception_table_entry *ex) { - const struct exception_table_entry *fixup; - unsigned long addr; - - addr = instruction_pointer(regs); - - /* Search the BPF tables first, these are formatted differently */ - fixup = search_bpf_extables(addr); - if (fixup) - return arm64_bpf_fixup_exception(fixup, regs); - - fixup = search_exception_tables(addr); - if (!fixup) - return 0; - - regs->pc = (unsigned long)&fixup->fixup + fixup->fixup; - return 1; + return ((unsigned long)&ex->fixup + ex->fixup); +} + +static bool ex_handler_fixup(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + regs->pc = get_ex_fixup(ex); + return true; +} + +static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + int reg_zero = FIELD_GET(EX_DATA_REG_ZERO, ex->data); + + pt_regs_write_reg(regs, reg_err, -EFAULT); + pt_regs_write_reg(regs, reg_zero, 0); + + regs->pc = get_ex_fixup(ex); + return true; +} + +static bool +ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + int reg_data = FIELD_GET(EX_DATA_REG_DATA, ex->data); + int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); + unsigned long data, addr, offset; + + addr = pt_regs_read_reg(regs, reg_addr); + + offset = addr & 0x7UL; + addr &= ~0x7UL; + + data = *(unsigned long*)addr; + +#ifndef __AARCH64EB__ + data >>= 8 * offset; +#else + data <<= 8 * offset; +#endif + + pt_regs_write_reg(regs, reg_data, data); + + regs->pc = get_ex_fixup(ex); + return true; +} + +bool fixup_exception(struct pt_regs *regs) +{ + const struct exception_table_entry *ex; + + ex = search_exception_tables(instruction_pointer(regs)); + if (!ex) + return false; + + switch (ex->type) { + case EX_TYPE_FIXUP: + return ex_handler_fixup(ex, regs); + case EX_TYPE_BPF: + return ex_handler_bpf(ex, regs); + case EX_TYPE_UACCESS_ERR_ZERO: + return ex_handler_uaccess_err_zero(ex, regs); + case EX_TYPE_LOAD_UNALIGNED_ZEROPAD: + return ex_handler_load_unaligned_zeropad(ex, regs); + } + + BUG(); } diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 9ae24e3b72..77341b160a 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -297,12 +297,14 @@ static void die_kernel_fault(const char *msg, unsigned long addr, pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg, addr); + kasan_non_canonical_hook(addr); + mem_abort_decode(esr); show_pte(addr); die("Oops", regs, esr); bust_spinlocks(0); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } #ifdef CONFIG_KASAN_HW_TAGS @@ -606,10 +608,8 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr, } if (fault & VM_FAULT_RETRY) { - if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { - mm_flags |= FAULT_FLAG_TRIED; - goto retry; - } + mm_flags |= FAULT_FLAG_TRIED; + goto retry; } mmap_read_unlock(mm); @@ -813,11 +813,8 @@ void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs) if (!inf->fn(far, esr, regs)) return; - if (!user_mode(regs)) { - pr_alert("Unhandled fault at 0x%016lx\n", addr); - mem_abort_decode(esr); - show_pte(addr); - } + if (!user_mode(regs)) + die_kernel_fault(inf->name, addr, esr, regs); /* * At this point we have an unrecognized fault type whose tag bits may diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index a8158c9489..ffb9c22961 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -40,11 +40,11 @@ void __init arm64_hugetlb_cma_reserve(void) { int order; -#ifdef CONFIG_ARM64_4K_PAGES - order = PUD_SHIFT - PAGE_SHIFT; -#else - order = CONT_PMD_SHIFT - PAGE_SHIFT; -#endif + if (pud_sect_supported()) + order = PUD_SHIFT - PAGE_SHIFT; + else + order = CONT_PMD_SHIFT - PAGE_SHIFT; + /* * HugeTLB CMA reservation is required for gigantic * huge pages which could not be allocated via the @@ -62,8 +62,9 @@ bool arch_hugetlb_migration_supported(struct hstate *h) size_t pagesize = huge_page_size(h); switch (pagesize) { -#ifdef CONFIG_ARM64_4K_PAGES +#ifndef __PAGETABLE_PMD_FOLDED case PUD_SIZE: + return pud_sect_supported(); #endif case PMD_SIZE: case CONT_PMD_SIZE: @@ -126,8 +127,11 @@ static inline int num_contig_ptes(unsigned long size, size_t *pgsize) *pgsize = size; switch (size) { -#ifdef CONFIG_ARM64_4K_PAGES +#ifndef __PAGETABLE_PMD_FOLDED case PUD_SIZE: + if (pud_sect_supported()) + contig_ptes = 1; + break; #endif case PMD_SIZE: contig_ptes = 1; @@ -489,9 +493,9 @@ void huge_ptep_clear_flush(struct vm_area_struct *vma, static int __init hugetlbpage_init(void) { -#ifdef CONFIG_ARM64_4K_PAGES - hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); -#endif + if (pud_sect_supported()) + hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); + hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT); hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); hugetlb_add_hstate(CONT_PTE_SHIFT - PAGE_SHIFT); @@ -503,8 +507,9 @@ arch_initcall(hugetlbpage_init); bool __init arch_hugetlb_valid_size(unsigned long size) { switch (size) { -#ifdef CONFIG_ARM64_4K_PAGES +#ifndef __PAGETABLE_PMD_FOLDED case PUD_SIZE: + return pud_sect_supported(); #endif case CONT_PMD_SIZE: case PMD_SIZE: diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 37a81754d9..db63cc8857 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -160,43 +160,6 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max) free_area_init(max_zone_pfns); } -int pfn_valid(unsigned long pfn) -{ - phys_addr_t addr = PFN_PHYS(pfn); - struct mem_section *ms; - - /* - * Ensure the upper PAGE_SHIFT bits are clear in the - * pfn. Else it might lead to false positives when - * some of the upper bits are set, but the lower bits - * match a valid pfn. - */ - if (PHYS_PFN(addr) != pfn) - return 0; - - if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) - return 0; - - ms = __pfn_to_section(pfn); - if (!valid_section(ms)) - return 0; - - /* - * ZONE_DEVICE memory does not have the memblock entries. - * memblock_is_map_memory() check for ZONE_DEVICE based - * addresses will always fail. Even the normal hotplugged - * memory will never have MEMBLOCK_NOMAP flag set in their - * memblock entries. Skip memblock search for all non early - * memory sections covering all of hotplug memory including - * both normal and ZONE_DEVICE based. - */ - if (!early_section(ms)) - return pfn_section_valid(ms, pfn); - - return memblock_is_memory(addr); -} -EXPORT_SYMBOL(pfn_valid); - int pfn_is_map_memory(unsigned long pfn) { phys_addr_t addr = PFN_PHYS(pfn); @@ -209,7 +172,7 @@ int pfn_is_map_memory(unsigned long pfn) } EXPORT_SYMBOL(pfn_is_map_memory); -static phys_addr_t memory_limit = PHYS_ADDR_MAX; +static phys_addr_t memory_limit __ro_after_init = PHYS_ADDR_MAX; /* * Limit the memory size that was specified via FDT. @@ -416,8 +379,6 @@ void __init mem_init(void) else if (!xen_swiotlb_detect()) swiotlb_force = SWIOTLB_NO_FORCE; - set_max_mapnr(max_pfn - PHYS_PFN_OFFSET); - /* this will put all unused low memory onto the freelists */ memblock_free_all(); diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index 61b52a92b8..c12cd70059 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -36,7 +36,7 @@ static phys_addr_t __init kasan_alloc_zeroed_page(int node) { void *p = memblock_alloc_try_nid(PAGE_SIZE, PAGE_SIZE, __pa(MAX_DMA_ADDRESS), - MEMBLOCK_ALLOC_KASAN, node); + MEMBLOCK_ALLOC_NOLEAKTRACE, node); if (!p) panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n", __func__, PAGE_SIZE, PAGE_SIZE, node, @@ -49,7 +49,8 @@ static phys_addr_t __init kasan_alloc_raw_page(int node) { void *p = memblock_alloc_try_nid_raw(PAGE_SIZE, PAGE_SIZE, __pa(MAX_DMA_ADDRESS), - MEMBLOCK_ALLOC_KASAN, node); + MEMBLOCK_ALLOC_NOLEAKTRACE, + node); if (!p) panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n", __func__, PAGE_SIZE, PAGE_SIZE, node, @@ -287,13 +288,29 @@ static void __init kasan_init_depth(void) init_task.kasan_depth = 0; } +#ifdef CONFIG_KASAN_VMALLOC +void __init kasan_populate_early_vm_area_shadow(void *start, unsigned long size) +{ + unsigned long shadow_start, shadow_end; + + if (!is_vmalloc_or_module_addr(start)) + return; + + shadow_start = (unsigned long)kasan_mem_to_shadow(start); + shadow_start = ALIGN_DOWN(shadow_start, PAGE_SIZE); + shadow_end = (unsigned long)kasan_mem_to_shadow(start + size); + shadow_end = ALIGN(shadow_end, PAGE_SIZE); + kasan_map_populate(shadow_start, shadow_end, NUMA_NO_NODE); +} +#endif + void __init kasan_init(void) { kasan_init_shadow(); kasan_init_depth(); #if defined(CONFIG_KASAN_GENERIC) /* CONFIG_KASAN_SW_TAGS also requires kasan_init_sw_tags(). */ - pr_info("KernelAddressSanitizer initialized\n"); + pr_info("KernelAddressSanitizer initialized (generic)\n"); #endif } diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 9d03806316..49abbf43bf 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -96,7 +96,8 @@ static phys_addr_t __init early_pgtable_alloc(int shift) phys_addr_t phys; void *ptr; - phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); + phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, + MEMBLOCK_ALLOC_NOLEAKTRACE); if (!phys) panic("Failed to allocate page table page\n"); @@ -744,8 +745,8 @@ void __init paging_init(void) cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); init_mm.pgd = swapper_pg_dir; - memblock_free(__pa_symbol(init_pg_dir), - __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); + memblock_phys_free(__pa_symbol(init_pg_dir), + __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir)); memblock_allow_resize(); } diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c index 527f0a39c3..d7da8ca40d 100644 --- a/arch/arm64/mm/trans_pgd.c +++ b/arch/arm64/mm/trans_pgd.c @@ -5,8 +5,8 @@ * * This file derived from: arch/arm64/kernel/hibernate.c * - * Copyright (c) 2020, Microsoft Corporation. - * Pavel Tatashin + * Copyright (c) 2021, Microsoft Corporation. + * Pasha Tatashin * */ @@ -217,63 +217,6 @@ int trans_pgd_create_copy(struct trans_pgd_info *info, pgd_t **dst_pgdp, return rc; } -/* - * Add map entry to trans_pgd for a base-size page at PTE level. - * info: contains allocator and its argument - * trans_pgd: page table in which new map is added. - * page: page to be mapped. - * dst_addr: new VA address for the page - * pgprot: protection for the page. - * - * Returns 0 on success, and -ENOMEM on failure. - */ -int trans_pgd_map_page(struct trans_pgd_info *info, pgd_t *trans_pgd, - void *page, unsigned long dst_addr, pgprot_t pgprot) -{ - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; - pmd_t *pmdp; - pte_t *ptep; - - pgdp = pgd_offset_pgd(trans_pgd, dst_addr); - if (pgd_none(READ_ONCE(*pgdp))) { - p4dp = trans_alloc(info); - if (!pgdp) - return -ENOMEM; - pgd_populate(NULL, pgdp, p4dp); - } - - p4dp = p4d_offset(pgdp, dst_addr); - if (p4d_none(READ_ONCE(*p4dp))) { - pudp = trans_alloc(info); - if (!pudp) - return -ENOMEM; - p4d_populate(NULL, p4dp, pudp); - } - - pudp = pud_offset(p4dp, dst_addr); - if (pud_none(READ_ONCE(*pudp))) { - pmdp = trans_alloc(info); - if (!pmdp) - return -ENOMEM; - pud_populate(NULL, pudp, pmdp); - } - - pmdp = pmd_offset(pudp, dst_addr); - if (pmd_none(READ_ONCE(*pmdp))) { - ptep = trans_alloc(info); - if (!ptep) - return -ENOMEM; - pmd_populate_kernel(NULL, pmdp, ptep); - } - - ptep = pte_offset_kernel(pmdp, dst_addr); - set_pte(ptep, pfn_pte(virt_to_pfn(page), pgprot)); - - return 0; -} - /* * The page we want to idmap may be outside the range covered by VA_BITS that * can be built using the kernel's p?d_populate() helpers. As a one off, for a @@ -322,3 +265,26 @@ int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0, return 0; } + +/* + * Create a copy of the vector table so we can call HVC_SET_VECTORS or + * HVC_SOFT_RESTART from contexts where the table may be overwritten. + */ +int trans_pgd_copy_el2_vectors(struct trans_pgd_info *info, + phys_addr_t *el2_vectors) +{ + void *hyp_stub = trans_alloc(info); + + if (!hyp_stub) + return -ENOMEM; + *el2_vectors = virt_to_phys(hyp_stub); + memcpy(hyp_stub, &trans_pgd_stub_vectors, ARM64_VECTOR_TABLE_LEN); + caches_clean_inval_pou((unsigned long)hyp_stub, + (unsigned long)hyp_stub + + ARM64_VECTOR_TABLE_LEN); + dcache_clean_inval_poc((unsigned long)hyp_stub, + (unsigned long)hyp_stub + + ARM64_VECTOR_TABLE_LEN); + + return 0; +} diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 465c44d0c7..e96d4d8729 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -43,7 +44,7 @@ static const int bpf2a64[] = { [BPF_REG_9] = A64_R(22), /* read-only frame pointer to access stack */ [BPF_REG_FP] = A64_R(25), - /* temporary registers for internal BPF JIT */ + /* temporary registers for BPF JIT */ [TMP_REG_1] = A64_R(10), [TMP_REG_2] = A64_R(11), [TMP_REG_3] = A64_R(12), @@ -286,13 +287,14 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) emit(A64_CMP(0, r3, tmp), ctx); emit(A64_B_(A64_COND_CS, jmp_offset), ctx); - /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) + /* + * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) * goto out; * tail_call_cnt++; */ emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx); emit(A64_CMP(1, tcc, tmp), ctx); - emit(A64_B_(A64_COND_HI, jmp_offset), ctx); + emit(A64_B_(A64_COND_CS, jmp_offset), ctx); emit(A64_ADD_I(1, tcc, tcc, 1), ctx); /* prog = array->ptrs[index]; @@ -358,15 +360,15 @@ static void build_epilogue(struct jit_ctx *ctx) #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) #define BPF_FIXUP_REG_MASK GENMASK(31, 27) -int arm64_bpf_fixup_exception(const struct exception_table_entry *ex, - struct pt_regs *regs) +bool ex_handler_bpf(const struct exception_table_entry *ex, + struct pt_regs *regs) { off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); regs->regs[dst_reg] = 0; regs->pc = (unsigned long)&ex->fixup - offset; - return 1; + return true; } /* For accesses to BTF pointers, add an entry to the exception table */ @@ -412,6 +414,8 @@ static int add_exception_handler(const struct bpf_insn *insn, ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); + ex->type = EX_TYPE_BPF; + ctx->exentry_idx++; return 0; } @@ -788,7 +792,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, u64 imm64; imm64 = (u64)insn1.imm << 32 | (u32)imm; - emit_a64_mov_i64(dst, imm64, ctx); + if (bpf_pseudo_func(insn)) + emit_addr_mov_i64(dst, imm64, ctx); + else + emit_a64_mov_i64(dst, imm64, ctx); return 1; } diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index b71c6cbb23..cea7533cb3 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -18,6 +18,7 @@ HAS_CRC32 HAS_DCPODP HAS_DCPOP HAS_E0PD +HAS_ECV HAS_EPAN HAS_GENERIC_AUTH HAS_GENERIC_AUTH_ARCH @@ -39,6 +40,7 @@ HW_DBM KVM_PROTECTED_MODE MISMATCHED_CACHE_TYPE MTE +MTE_ASYMM SPECTRE_V2 SPECTRE_V3A SPECTRE_V4 @@ -54,6 +56,13 @@ WORKAROUND_1418040 WORKAROUND_1463225 WORKAROUND_1508412 WORKAROUND_1542419 +WORKAROUND_1902691 +WORKAROUND_2038923 +WORKAROUND_2064142 +WORKAROUND_2077057 +WORKAROUND_TRBE_OVERWRITE_FILL_MODE +WORKAROUND_TSB_FLUSH_FAILURE +WORKAROUND_TRBE_WRITE_OUT_OF_RANGE WORKAROUND_CAVIUM_23154 WORKAROUND_CAVIUM_27456 WORKAROUND_CAVIUM_30115 diff --git a/arch/arm64/xen/hypercall.S b/arch/arm64/xen/hypercall.S index 5b09aca551..9d01361696 100644 --- a/arch/arm64/xen/hypercall.S +++ b/arch/arm64/xen/hypercall.S @@ -80,7 +80,6 @@ HYPERCALL2(hvm_op); HYPERCALL2(memory_op); HYPERCALL2(physdev_op); HYPERCALL3(vcpu_op); -HYPERCALL1(tmem_op); HYPERCALL1(platform_op_raw); HYPERCALL2(multicall); HYPERCALL2(vm_assist); diff --git a/arch/csky/Kbuild b/arch/csky/Kbuild index a4e40e534e..4e39f7abde 100644 --- a/arch/csky/Kbuild +++ b/arch/csky/Kbuild @@ -1 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only + +# for cleaning +subdir- += boot diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 823d3d5a9e..132f43f12d 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -17,7 +17,6 @@ config CSKY select CSKY_APB_INTC select DMA_DIRECT_REMAP select IRQ_DOMAIN - select HANDLE_DOMAIN_IRQ select DW_APB_TIMER_OF select GENERIC_IOREMAP select GENERIC_LIB_ASHLDI3 @@ -53,7 +52,6 @@ config CSKY select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_ERROR_INJECTION - select HAVE_FUTEX_CMPXCHG if FUTEX && SMP select HAVE_FTRACE_MCOUNT_RECORD select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZO diff --git a/arch/csky/Makefile b/arch/csky/Makefile index 37f593a4bf..8668050776 100644 --- a/arch/csky/Makefile +++ b/arch/csky/Makefile @@ -76,9 +76,6 @@ all: zImage zImage Image uImage: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - define archhelp echo '* zImage - Compressed kernel image (arch/$(ARCH)/boot/zImage)' echo ' Image - Uncompressed kernel image (arch/$(ARCH)/boot/Image)' diff --git a/arch/csky/abiv1/alignment.c b/arch/csky/abiv1/alignment.c index cb2a0d94a1..2df115d0e2 100644 --- a/arch/csky/abiv1/alignment.c +++ b/arch/csky/abiv1/alignment.c @@ -294,7 +294,7 @@ void csky_alignment(struct pt_regs *regs) __func__, opcode, rz, rx, imm, addr); show_regs(regs); bust_spinlocks(0); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)addr); diff --git a/arch/csky/include/asm/bitops.h b/arch/csky/include/asm/bitops.h index 02b72a0007..72e1b2aa29 100644 --- a/arch/csky/include/asm/bitops.h +++ b/arch/csky/include/asm/bitops.h @@ -59,7 +59,6 @@ static __always_inline unsigned long __fls(unsigned long x) #include #include -#include #ifndef _LINUX_BITOPS_H #error only can be included directly diff --git a/arch/csky/include/asm/kprobes.h b/arch/csky/include/asm/kprobes.h index b647bbde4d..55267cbf52 100644 --- a/arch/csky/include/asm/kprobes.h +++ b/arch/csky/include/asm/kprobes.h @@ -41,7 +41,7 @@ void arch_remove_kprobe(struct kprobe *p); int kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr); int kprobe_breakpoint_handler(struct pt_regs *regs); int kprobe_single_step_handler(struct pt_regs *regs); -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); void __kprobes *trampoline_probe_handler(struct pt_regs *regs); #endif /* CONFIG_KPROBES */ diff --git a/arch/csky/include/asm/processor.h b/arch/csky/include/asm/processor.h index 9e933021fe..817dd60ff1 100644 --- a/arch/csky/include/asm/processor.h +++ b/arch/csky/include/asm/processor.h @@ -81,7 +81,7 @@ static inline void release_thread(struct task_struct *dead_task) extern int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->usp) diff --git a/arch/csky/include/asm/spinlock_types.h b/arch/csky/include/asm/spinlock_types.h index 8ff0f6ff3a..db87a12c38 100644 --- a/arch/csky/include/asm/spinlock_types.h +++ b/arch/csky/include/asm/spinlock_types.h @@ -3,7 +3,7 @@ #ifndef __ASM_CSKY_SPINLOCK_TYPES_H #define __ASM_CSKY_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/csky/include/asm/syscall.h b/arch/csky/include/asm/syscall.h index f624fa3bbc..0de5734950 100644 --- a/arch/csky/include/asm/syscall.h +++ b/arch/csky/include/asm/syscall.h @@ -59,15 +59,6 @@ syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, memcpy(args, ®s->a1, 5 * sizeof(args[0])); } -static inline void -syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, - const unsigned long *args) -{ - regs->orig_a0 = args[0]; - args++; - memcpy(®s->a1, args, 5 * sizeof(regs->a1)); -} - static inline int syscall_get_arch(struct task_struct *task) { diff --git a/arch/csky/include/asm/uaccess.h b/arch/csky/include/asm/uaccess.h index c40f06ee8d..ac5a54f57d 100644 --- a/arch/csky/include/asm/uaccess.h +++ b/arch/csky/include/asm/uaccess.h @@ -3,14 +3,13 @@ #ifndef __ASM_CSKY_UACCESS_H #define __ASM_CSKY_UACCESS_H -#define user_addr_max() \ - (uaccess_kernel() ? KERNEL_DS.seg : get_fs().seg) +#define user_addr_max() (current_thread_info()->addr_limit.seg) static inline int __access_ok(unsigned long addr, unsigned long size) { - unsigned long limit = current_thread_info()->addr_limit.seg; + unsigned long limit = user_addr_max(); - return ((addr < limit) && ((addr + size) < limit)); + return (size <= limit) && (addr <= (limit - size)); } #define __access_ok __access_ok diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index 00e3c8ebf9..a4ababf25e 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -249,7 +249,7 @@ ENTRY(csky_irq) mov a0, sp - jbsr csky_do_IRQ + jbsr generic_handle_arch_irq jmpi ret_from_exception diff --git a/arch/csky/kernel/ftrace.c b/arch/csky/kernel/ftrace.c index b4a7ec1517..50bfcf1290 100644 --- a/arch/csky/kernel/ftrace.c +++ b/arch/csky/kernel/ftrace.c @@ -133,11 +133,6 @@ int ftrace_update_ftrace_func(ftrace_func_t func) (unsigned long)func, true, true); return ret; } - -int __init ftrace_dyn_arch_init(void) -{ - return 0; -} #endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS diff --git a/arch/csky/kernel/irq.c b/arch/csky/kernel/irq.c index 03a1930f1c..fcdaf31562 100644 --- a/arch/csky/kernel/irq.c +++ b/arch/csky/kernel/irq.c @@ -15,8 +15,3 @@ void __init init_IRQ(void) setup_smp_ipi(); #endif } - -asmlinkage void __irq_entry csky_do_IRQ(struct pt_regs *regs) -{ - handle_arch_irq(regs); -} diff --git a/arch/csky/kernel/perf_callchain.c b/arch/csky/kernel/perf_callchain.c index 35318a635a..92057de08f 100644 --- a/arch/csky/kernel/perf_callchain.c +++ b/arch/csky/kernel/perf_callchain.c @@ -86,13 +86,8 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); unsigned long fp = 0; - /* C-SKY does not support virtualization. */ - if (guest_cbs && guest_cbs->is_in_guest()) - return; - fp = regs->regs[4]; perf_callchain_store(entry, regs->pc); @@ -111,15 +106,8 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stackframe fr; - /* C-SKY does not support virtualization. */ - if (guest_cbs && guest_cbs->is_in_guest()) { - pr_warn("C-SKY does not support perf in guest mode!"); - return; - } - fr.fp = regs->regs[4]; fr.lr = regs->lr; walk_stackframe(&fr, entry); diff --git a/arch/csky/kernel/probes/ftrace.c b/arch/csky/kernel/probes/ftrace.c index ef2bb9bd96..834cffcfbc 100644 --- a/arch/csky/kernel/probes/ftrace.c +++ b/arch/csky/kernel/probes/ftrace.c @@ -2,13 +2,6 @@ #include -int arch_check_ftrace_location(struct kprobe *p) -{ - if (ftrace_location((unsigned long)p->addr)) - p->flags |= KPROBE_FLAG_FTRACE; - return 0; -} - /* Ftrace callback handler for kprobes -- called under preepmt disabled */ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs) @@ -24,7 +17,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, return; regs = ftrace_get_regs(fregs); - preempt_disable_notrace(); p = get_kprobe((kprobe_opcode_t *)ip); if (!p) { p = get_kprobe((kprobe_opcode_t *)(ip - MCOUNT_INSN_SIZE)); @@ -64,7 +56,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, __this_cpu_write(current_kprobe, NULL); } out: - preempt_enable_notrace(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(kprobe_ftrace_handler); diff --git a/arch/csky/kernel/probes/kprobes.c b/arch/csky/kernel/probes/kprobes.c index 8fffa34d4e..42920f25e7 100644 --- a/arch/csky/kernel/probes/kprobes.c +++ b/arch/csky/kernel/probes/kprobes.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ +#define pr_fmt(fmt) "kprobes: " fmt + #include #include #include @@ -77,10 +79,8 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) { unsigned long probe_addr = (unsigned long)p->addr; - if (probe_addr & 0x1) { - pr_warn("Address not aligned.\n"); - return -EINVAL; - } + if (probe_addr & 0x1) + return -EILSEQ; /* copy instruction */ p->opcode = le32_to_cpu(*p->addr); @@ -225,7 +225,7 @@ static int __kprobes reenter_kprobe(struct kprobe *p, break; case KPROBE_HIT_SS: case KPROBE_REENTER: - pr_warn("Unrecoverable kprobe detected.\n"); + pr_warn("Failed to recover from reentered kprobes.\n"); dump_kprobe(p); BUG(); break; @@ -386,7 +386,7 @@ int __init arch_populate_kprobe_blacklist(void) void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs) { - return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL); + return (void *)kretprobe_trampoline_handler(regs, NULL); } void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, @@ -394,7 +394,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, { ri->ret_addr = (kprobe_opcode_t *)regs->lr; ri->fp = NULL; - regs->lr = (unsigned long) &kretprobe_trampoline; + regs->lr = (unsigned long) &__kretprobe_trampoline; } int __kprobes arch_trampoline_kprobe(struct kprobe *p) diff --git a/arch/csky/kernel/probes/kprobes_trampoline.S b/arch/csky/kernel/probes/kprobes_trampoline.S index b1fe3af24f..ba48ad04a8 100644 --- a/arch/csky/kernel/probes/kprobes_trampoline.S +++ b/arch/csky/kernel/probes/kprobes_trampoline.S @@ -4,7 +4,7 @@ #include -ENTRY(kretprobe_trampoline) +ENTRY(__kretprobe_trampoline) SAVE_REGS_FTRACE mov a0, sp /* pt_regs */ @@ -16,4 +16,4 @@ ENTRY(kretprobe_trampoline) RESTORE_REGS_FTRACE rts -ENDPROC(kretprobe_trampoline) +ENDPROC(__kretprobe_trampoline) diff --git a/arch/csky/kernel/smp.c b/arch/csky/kernel/smp.c index e2993539af..6bb38bc2f3 100644 --- a/arch/csky/kernel/smp.c +++ b/arch/csky/kernel/smp.c @@ -180,15 +180,13 @@ void __init setup_smp_ipi(void) void __init setup_smp(void) { struct device_node *node = NULL; - int cpu; + unsigned int cpu; for_each_of_cpu_node(node) { if (!of_device_is_available(node)) continue; - if (of_property_read_u32(node, "reg", &cpu)) - continue; - + cpu = of_get_cpu_hwid(node, 0); if (cpu >= NR_CPUS) continue; diff --git a/arch/csky/kernel/stacktrace.c b/arch/csky/kernel/stacktrace.c index 1b280ef080..9f78f5d215 100644 --- a/arch/csky/kernel/stacktrace.c +++ b/arch/csky/kernel/stacktrace.c @@ -111,12 +111,11 @@ static bool save_wchan(unsigned long pc, void *arg) return false; } -unsigned long get_wchan(struct task_struct *task) +unsigned long __get_wchan(struct task_struct *task) { unsigned long pc = 0; - if (likely(task && task != current && !task_is_running(task))) - walk_stackframe(task, NULL, save_wchan, &pc); + walk_stackframe(task, NULL, save_wchan, &pc); return pc; } diff --git a/arch/csky/kernel/traps.c b/arch/csky/kernel/traps.c index 2020af88b6..6e426fba01 100644 --- a/arch/csky/kernel/traps.c +++ b/arch/csky/kernel/traps.c @@ -109,7 +109,7 @@ void die(struct pt_regs *regs, const char *str) if (panic_on_oops) panic("Fatal exception"); if (ret != NOTIFY_STOP) - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index 466ad94981..7215a46b6b 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -67,7 +67,7 @@ static inline void no_context(struct pt_regs *regs, unsigned long addr) pr_alert("Unable to handle kernel paging request at virtual " "addr 0x%08lx, pc: 0x%08lx\n", addr, regs->pc); die(regs, "Oops"); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault) diff --git a/arch/h8300/Kbuild b/arch/h8300/Kbuild index b2583e7efb..e4703f3534 100644 --- a/arch/h8300/Kbuild +++ b/arch/h8300/Kbuild @@ -1,2 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y += kernel/ mm/ boot/dts/ + +# for cleaning +subdir- += boot diff --git a/arch/h8300/Makefile b/arch/h8300/Makefile index eb4cb8f683..807f41e60e 100644 --- a/arch/h8300/Makefile +++ b/arch/h8300/Makefile @@ -34,9 +34,6 @@ libs-y += arch/$(ARCH)/lib/ boot := arch/h8300/boot -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - vmlinux.srec vmlinux.bin zImage uImage.bin: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ diff --git a/arch/h8300/boot/compressed/Makefile b/arch/h8300/boot/compressed/Makefile index 5942793f77..6ab2fa5ba1 100644 --- a/arch/h8300/boot/compressed/Makefile +++ b/arch/h8300/boot/compressed/Makefile @@ -30,9 +30,11 @@ $(obj)/vmlinux.bin: vmlinux FORCE suffix-$(CONFIG_KERNEL_GZIP) := gzip suffix-$(CONFIG_KERNEL_LZO) := lzo +compress-$(CONFIG_KERNEL_GZIP) := gzip +compress-$(CONFIG_KERNEL_LZO) := lzo_with_size $(obj)/vmlinux.bin.$(suffix-y): $(obj)/vmlinux.bin FORCE - $(call if_changed,$(suffix-y)) + $(call if_changed,$(compress-y)) LDFLAGS_piggy.o := -r --format binary --oformat elf32-h8300-linux -T OBJCOPYFLAGS := -O binary diff --git a/arch/h8300/boot/dts/Makefile b/arch/h8300/boot/dts/Makefile index 69fcd81789..c36bbd1f25 100644 --- a/arch/h8300/boot/dts/Makefile +++ b/arch/h8300/boot/dts/Makefile @@ -1,9 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -ifneq '$(CONFIG_H8300_BUILTIN_DTB)' '""' -BUILTIN_DTB := $(patsubst "%",%,$(CONFIG_H8300_BUILTIN_DTB)).dtb.o -endif - -obj-y += $(BUILTIN_DTB) +obj-y += $(addsuffix .dtb.o, $(CONFIG_H8300_BUILTIN_DTB)) dtb-$(CONFIG_H8300H_SIM) := h8300h_sim.dtb dtb-$(CONFIG_H8S_SIM) := h8s_sim.dtb diff --git a/arch/h8300/include/asm/bitops.h b/arch/h8300/include/asm/bitops.h index c867a80cab..4489e3d6ed 100644 --- a/arch/h8300/include/asm/bitops.h +++ b/arch/h8300/include/asm/bitops.h @@ -168,7 +168,6 @@ static inline unsigned long __ffs(unsigned long word) return result; } -#include #include #include #include diff --git a/arch/h8300/include/asm/irq.h b/arch/h8300/include/asm/irq.h index 5fc5b436dd..776cf06d7a 100644 --- a/arch/h8300/include/asm/irq.h +++ b/arch/h8300/include/asm/irq.h @@ -2,8 +2,6 @@ #ifndef _H8300_IRQ_H_ #define _H8300_IRQ_H_ -#include - #if defined(CONFIG_CPU_H8300H) #define NR_IRQS 64 #define IRQ_CHIP h8300h_irq_chip diff --git a/arch/h8300/include/asm/processor.h b/arch/h8300/include/asm/processor.h index a060b41b2d..141a23eb62 100644 --- a/arch/h8300/include/asm/processor.h +++ b/arch/h8300/include/asm/processor.h @@ -105,7 +105,7 @@ static inline void release_thread(struct task_struct *dead_task) { } -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) \ ({ \ diff --git a/arch/h8300/kernel/irq.c b/arch/h8300/kernel/irq.c index 834e4d7b1b..8ad6d702cd 100644 --- a/arch/h8300/kernel/irq.c +++ b/arch/h8300/kernel/irq.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/h8300/kernel/process.c b/arch/h8300/kernel/process.c index 2ac27e4248..8833fa4f5d 100644 --- a/arch/h8300/kernel/process.c +++ b/arch/h8300/kernel/process.c @@ -128,15 +128,12 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, return 0; } -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { unsigned long fp, pc; unsigned long stack_page; int count = 0; - if (!p || p == current || task_is_running(p)) - return 0; - stack_page = (unsigned long)p; fp = ((struct pt_regs *)p->thread.ksp)->er6; do { diff --git a/arch/h8300/kernel/traps.c b/arch/h8300/kernel/traps.c index bdbe988d8d..a92c39e038 100644 --- a/arch/h8300/kernel/traps.c +++ b/arch/h8300/kernel/traps.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -106,7 +107,7 @@ void die(const char *str, struct pt_regs *fp, unsigned long err) dump(fp); spin_unlock_irq(&die_lock); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } static int kstack_depth_to_print = 24; diff --git a/arch/h8300/mm/fault.c b/arch/h8300/mm/fault.c index d4bc9c16f2..b465441f49 100644 --- a/arch/h8300/mm/fault.c +++ b/arch/h8300/mm/fault.c @@ -51,7 +51,7 @@ asmlinkage int do_page_fault(struct pt_regs *regs, unsigned long address, printk(" at virtual address %08lx\n", address); if (!user_mode(regs)) die("Oops", regs, error_code); - do_exit(SIGKILL); + make_task_dead(SIGKILL); return 1; } diff --git a/arch/hexagon/include/asm/bitops.h b/arch/hexagon/include/asm/bitops.h index 71429f756a..75d6ba3643 100644 --- a/arch/hexagon/include/asm/bitops.h +++ b/arch/hexagon/include/asm/bitops.h @@ -271,7 +271,6 @@ static inline unsigned long __fls(unsigned long word) } #include -#include #include #include diff --git a/arch/hexagon/include/asm/processor.h b/arch/hexagon/include/asm/processor.h index 9f0cc99420..615f7e4996 100644 --- a/arch/hexagon/include/asm/processor.h +++ b/arch/hexagon/include/asm/processor.h @@ -64,7 +64,7 @@ struct thread_struct { extern void release_thread(struct task_struct *dead_task); /* Get wait channel for task P. */ -extern unsigned long get_wchan(struct task_struct *p); +extern unsigned long __get_wchan(struct task_struct *p); /* The following stuff is pretty HEXAGON specific. */ diff --git a/arch/hexagon/include/asm/spinlock_types.h b/arch/hexagon/include/asm/spinlock_types.h index 19d233497b..d5f66495b6 100644 --- a/arch/hexagon/include/asm/spinlock_types.h +++ b/arch/hexagon/include/asm/spinlock_types.h @@ -8,7 +8,7 @@ #ifndef _ASM_SPINLOCK_TYPES_H #define _ASM_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h index ef5bfef8d4..719ba3f3c4 100644 --- a/arch/hexagon/include/asm/uaccess.h +++ b/arch/hexagon/include/asm/uaccess.h @@ -25,17 +25,17 @@ * Returns true (nonzero) if the memory block *may* be valid, false (zero) * if it is definitely invalid. * - * User address space in Hexagon, like x86, goes to 0xbfffffff, so the - * simple MSB-based tests used by MIPS won't work. Some further - * optimization is probably possible here, but for now, keep it - * reasonably simple and not *too* slow. After all, we've got the - * MMU for backup. */ +#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) +#define user_addr_max() (uaccess_kernel() ? ~0UL : TASK_SIZE) -#define __access_ok(addr, size) \ - ((get_fs().seg == KERNEL_DS.seg) || \ - (((unsigned long)addr < get_fs().seg) && \ - (unsigned long)size < (get_fs().seg - (unsigned long)addr))) +static inline int __access_ok(unsigned long addr, unsigned long size) +{ + unsigned long limit = TASK_SIZE; + + return (size <= limit) && (addr <= (limit - size)); +} +#define __access_ok __access_ok /* * When a kernel-mode page fault is taken, the faulting instruction diff --git a/arch/hexagon/kernel/process.c b/arch/hexagon/kernel/process.c index 6a6835fb42..232dfd8956 100644 --- a/arch/hexagon/kernel/process.c +++ b/arch/hexagon/kernel/process.c @@ -130,13 +130,11 @@ void flush_thread(void) * is an identification of the point at which the scheduler * was invoked by a blocked thread. */ -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { unsigned long fp, pc; unsigned long stack_page; int count = 0; - if (!p || p == current || task_is_running(p)) - return 0; stack_page = (unsigned long)task_stack_page(p); fp = ((struct hexagon_switch_stack *)p->thread.switch_sp)->fp; diff --git a/arch/hexagon/kernel/traps.c b/arch/hexagon/kernel/traps.c index edfc35dafe..1240f038cc 100644 --- a/arch/hexagon/kernel/traps.c +++ b/arch/hexagon/kernel/traps.c @@ -214,7 +214,7 @@ int die(const char *str, struct pt_regs *regs, long err) panic("Fatal exception"); oops_exit(); - do_exit(err); + make_task_dead(err); return 0; } diff --git a/arch/hexagon/mm/vm_fault.c b/arch/hexagon/mm/vm_fault.c index ef32c5a84f..4fac4b9eb3 100644 --- a/arch/hexagon/mm/vm_fault.c +++ b/arch/hexagon/mm/vm_fault.c @@ -98,11 +98,9 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs) /* The most common case -- we are done. */ if (likely(!(fault & VM_FAULT_ERROR))) { - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; - goto retry; - } + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; + goto retry; } mmap_read_unlock(mm); diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 1e33666fa6..a7e01573ab 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -32,6 +32,7 @@ config IA64 select HAVE_FTRACE_MCOUNT_RECORD select HAVE_DYNAMIC_FTRACE if (!ITANIUM) select HAVE_FUNCTION_TRACER + select HAVE_SETUP_PER_CPU_AREA select TTY select HAVE_ARCH_TRACEHOOK select HAVE_VIRT_CPU_ACCOUNTING @@ -88,9 +89,6 @@ config GENERIC_CALIBRATE_DELAY bool default y -config HAVE_SETUP_PER_CPU_AREA - def_bool y - config DMI bool default y @@ -292,6 +290,7 @@ config NUMA bool "NUMA support" depends on !FLATMEM select SMP + select USE_PERCPU_NUMA_NODE_ID help Say Y to compile the kernel to support NUMA (Non-Uniform Memory Access). This option is for configuring high-end multiprocessor @@ -311,10 +310,6 @@ config HAVE_ARCH_NODEDATA_EXTENSION def_bool y depends on NUMA -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - config HAVE_MEMORYLESS_NODES def_bool NUMA @@ -323,7 +318,7 @@ config ARCH_PROC_KCORE_TEXT depends on PROC_KCORE config IA64_MCA_RECOVERY - tristate "MCA recovery from errors other than TLB." + bool "MCA recovery from errors other than TLB." config IA64_PALINFO tristate "/proc/pal support" diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile index 7e548c654a..3b3ac3e1f2 100644 --- a/arch/ia64/Makefile +++ b/arch/ia64/Makefile @@ -67,8 +67,6 @@ vmlinux.bin: vmlinux FORCE unwcheck: vmlinux -$(Q)READELF=$(READELF) $(PYTHON3) $(srctree)/arch/ia64/scripts/unwcheck.py $< -archclean: - archheaders: $(Q)$(MAKE) $(build)=arch/ia64/kernel/syscalls all diff --git a/arch/ia64/include/asm/bitops.h b/arch/ia64/include/asm/bitops.h index 2f24ee6459..577be93c08 100644 --- a/arch/ia64/include/asm/bitops.h +++ b/arch/ia64/include/asm/bitops.h @@ -441,8 +441,6 @@ static __inline__ unsigned long __arch_hweight64(unsigned long x) #endif /* __KERNEL__ */ -#include - #ifdef __KERNEL__ #include diff --git a/arch/ia64/include/asm/processor.h b/arch/ia64/include/asm/processor.h index 2d8bcdc27d..45365c2ef5 100644 --- a/arch/ia64/include/asm/processor.h +++ b/arch/ia64/include/asm/processor.h @@ -330,7 +330,7 @@ struct task_struct; #define release_thread(dead_task) /* Get wait channel for task P. */ -extern unsigned long get_wchan (struct task_struct *p); +extern unsigned long __get_wchan (struct task_struct *p); /* Return instruction pointer of blocked task TSK. */ #define KSTK_EIP(tsk) \ diff --git a/arch/ia64/include/asm/ptrace.h b/arch/ia64/include/asm/ptrace.h index 0817913590..a10a498eed 100644 --- a/arch/ia64/include/asm/ptrace.h +++ b/arch/ia64/include/asm/ptrace.h @@ -51,6 +51,11 @@ * the canonical representation by adding to instruction pointer. */ # define instruction_pointer(regs) ((regs)->cr_iip + ia64_psr(regs)->ri) +# define instruction_pointer_set(regs, val) \ +({ \ + ia64_psr(regs)->ri = (val & 0xf); \ + regs->cr_iip = (val & ~0xfULL); \ +}) static inline unsigned long user_stack_pointer(struct pt_regs *regs) { @@ -129,9 +134,9 @@ static inline long regs_return_value(struct pt_regs *regs) extern void ia64_decrement_ip (struct pt_regs *pt); extern void ia64_ptrace_stop(void); - #define arch_ptrace_stop(code, info) \ + #define arch_ptrace_stop() \ ia64_ptrace_stop() - #define arch_ptrace_stop_needed(code, info) \ + #define arch_ptrace_stop_needed() \ (!test_thread_flag(TIF_RESTORE_RSE)) extern void ptrace_attach_sync_user_rbs (struct task_struct *); diff --git a/arch/ia64/include/asm/spinlock.h b/arch/ia64/include/asm/spinlock.h index 864775970c..0e5c1ad323 100644 --- a/arch/ia64/include/asm/spinlock.h +++ b/arch/ia64/include/asm/spinlock.h @@ -124,18 +124,13 @@ static __always_inline void arch_spin_unlock(arch_spinlock_t *lock) __ticket_spin_unlock(lock); } -static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock, - unsigned long flags) -{ - arch_spin_lock(lock); -} -#define arch_spin_lock_flags arch_spin_lock_flags - #ifdef ASM_SUPPORTED static __always_inline void -arch_read_lock_flags(arch_rwlock_t *lock, unsigned long flags) +arch_read_lock(arch_rwlock_t *lock) { + unsigned long flags = 0; + __asm__ __volatile__ ( "tbit.nz p6, p0 = %1,%2\n" "br.few 3f\n" @@ -157,13 +152,8 @@ arch_read_lock_flags(arch_rwlock_t *lock, unsigned long flags) : "p6", "p7", "r2", "memory"); } -#define arch_read_lock_flags arch_read_lock_flags -#define arch_read_lock(lock) arch_read_lock_flags(lock, 0) - #else /* !ASM_SUPPORTED */ -#define arch_read_lock_flags(rw, flags) arch_read_lock(rw) - #define arch_read_lock(rw) \ do { \ arch_rwlock_t *__read_lock_ptr = (rw); \ @@ -186,8 +176,10 @@ do { \ #ifdef ASM_SUPPORTED static __always_inline void -arch_write_lock_flags(arch_rwlock_t *lock, unsigned long flags) +arch_write_lock(arch_rwlock_t *lock) { + unsigned long flags = 0; + __asm__ __volatile__ ( "tbit.nz p6, p0 = %1, %2\n" "mov ar.ccv = r0\n" @@ -210,9 +202,6 @@ arch_write_lock_flags(arch_rwlock_t *lock, unsigned long flags) : "ar.ccv", "p6", "p7", "r2", "r29", "memory"); } -#define arch_write_lock_flags arch_write_lock_flags -#define arch_write_lock(rw) arch_write_lock_flags(rw, 0) - #define arch_write_trylock(rw) \ ({ \ register long result; \ diff --git a/arch/ia64/include/asm/spinlock_types.h b/arch/ia64/include/asm/spinlock_types.h index 6e345fefcd..14b8a161c1 100644 --- a/arch/ia64/include/asm/spinlock_types.h +++ b/arch/ia64/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef _ASM_IA64_SPINLOCK_TYPES_H #define _ASM_IA64_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/ia64/include/asm/syscall.h b/arch/ia64/include/asm/syscall.h index 0d23c00493..2b02a3fb86 100644 --- a/arch/ia64/include/asm/syscall.h +++ b/arch/ia64/include/asm/syscall.h @@ -55,21 +55,8 @@ static inline void syscall_set_return_value(struct task_struct *task, } } -extern void ia64_syscall_get_set_arguments(struct task_struct *task, - struct pt_regs *regs, unsigned long *args, int rw); -static inline void syscall_get_arguments(struct task_struct *task, - struct pt_regs *regs, - unsigned long *args) -{ - ia64_syscall_get_set_arguments(task, regs, args, 0); -} - -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - unsigned long *args) -{ - ia64_syscall_get_set_arguments(task, regs, args, 1); -} +extern void syscall_get_arguments(struct task_struct *task, + struct pt_regs *regs, unsigned long *args); static inline int syscall_get_arch(struct task_struct *task) { diff --git a/arch/ia64/kernel/audit.c b/arch/ia64/kernel/audit.c index 5192ca899f..ec61f20ca6 100644 --- a/arch/ia64/kernel/audit.c +++ b/arch/ia64/kernel/audit.c @@ -38,13 +38,15 @@ int audit_classify_syscall(int abi, unsigned syscall) { switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_execve: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/ia64/kernel/ftrace.c b/arch/ia64/kernel/ftrace.c index b2ab2d58fb..d6360fd404 100644 --- a/arch/ia64/kernel/ftrace.c +++ b/arch/ia64/kernel/ftrace.c @@ -194,9 +194,3 @@ int ftrace_update_ftrace_func(ftrace_func_t func) flush_icache_range(addr, addr + 16); return 0; } - -/* run from kstop_machine */ -int __init ftrace_dyn_arch_init(void) -{ - return 0; -} diff --git a/arch/ia64/kernel/kprobes.c b/arch/ia64/kernel/kprobes.c index d4048518a1..1a7bab1c5d 100644 --- a/arch/ia64/kernel/kprobes.c +++ b/arch/ia64/kernel/kprobes.c @@ -392,14 +392,13 @@ static void __kprobes set_current_kprobe(struct kprobe *p, __this_cpu_write(current_kprobe, p); } -static void kretprobe_trampoline(void) +void __kretprobe_trampoline(void) { } int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) { - regs->cr_iip = __kretprobe_trampoline_handler(regs, - dereference_function_descriptor(kretprobe_trampoline), NULL); + regs->cr_iip = __kretprobe_trampoline_handler(regs, NULL); /* * By returning a non-zero value, we are telling * kprobe_handler() that we don't want the post_handler @@ -415,7 +414,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, ri->fp = NULL; /* Replace the return addr with trampoline addr */ - regs->b0 = (unsigned long)dereference_function_descriptor(kretprobe_trampoline); + regs->b0 = (unsigned long)dereference_function_descriptor(__kretprobe_trampoline); } /* Check the instruction in the slot is break */ @@ -891,11 +890,6 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, return ret; } -unsigned long arch_deref_entry_point(void *entry) -{ - return ((struct fnptr *)entry)->ip; -} - static struct kprobe trampoline_p = { .pre_handler = trampoline_probe_handler }; @@ -903,14 +897,14 @@ static struct kprobe trampoline_p = { int __init arch_init_kprobes(void) { trampoline_p.addr = - dereference_function_descriptor(kretprobe_trampoline); + dereference_function_descriptor(__kretprobe_trampoline); return register_kprobe(&trampoline_p); } int __kprobes arch_trampoline_kprobe(struct kprobe *p) { if (p->addr == - dereference_function_descriptor(kretprobe_trampoline)) + dereference_function_descriptor(__kretprobe_trampoline)) return 1; return 0; diff --git a/arch/ia64/kernel/mca_drv.c b/arch/ia64/kernel/mca_drv.c index 5bfc79be4c..23c203639a 100644 --- a/arch/ia64/kernel/mca_drv.c +++ b/arch/ia64/kernel/mca_drv.c @@ -176,7 +176,7 @@ mca_handler_bh(unsigned long paddr, void *iip, unsigned long ipsr) spin_unlock(&mca_bh_lock); /* This process is about to be killed itself */ - do_exit(SIGKILL); + make_task_dead(SIGKILL); } /** diff --git a/arch/ia64/kernel/module.c b/arch/ia64/kernel/module.c index 2cba53c1da..360f36b0eb 100644 --- a/arch/ia64/kernel/module.c +++ b/arch/ia64/kernel/module.c @@ -848,7 +848,7 @@ register_unwind_table (struct module *mod) { struct unw_table_entry *start = (void *) mod->arch.unwind->sh_addr; struct unw_table_entry *end = start + mod->arch.unwind->sh_size / sizeof (*start); - struct unw_table_entry tmp, *e1, *e2, *core, *init; + struct unw_table_entry *e1, *e2, *core, *init; unsigned long num_init = 0, num_core = 0; /* First, count how many init and core unwind-table entries there are. */ @@ -865,9 +865,7 @@ register_unwind_table (struct module *mod) for (e1 = start; e1 < end; ++e1) { for (e2 = e1 + 1; e2 < end; ++e2) { if (e2->start_offset < e1->start_offset) { - tmp = *e1; - *e1 = *e2; - *e2 = tmp; + swap(*e1, *e2); } } } diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index e56d63f4ab..834df24a88 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -523,15 +523,12 @@ exit_thread (struct task_struct *tsk) } unsigned long -get_wchan (struct task_struct *p) +__get_wchan (struct task_struct *p) { struct unw_frame_info info; unsigned long ip; int count = 0; - if (!p || p == current || task_is_running(p)) - return 0; - /* * Note: p may not be a blocked task (it could be current or * another process running on some other CPU. Rather than diff --git a/arch/ia64/kernel/ptrace.c b/arch/ia64/kernel/ptrace.c index df28c7dd16..6a1439eaa0 100644 --- a/arch/ia64/kernel/ptrace.c +++ b/arch/ia64/kernel/ptrace.c @@ -2001,17 +2001,16 @@ const struct user_regset_view *task_user_regset_view(struct task_struct *tsk) return &user_ia64_view; } -struct syscall_get_set_args { +struct syscall_get_args { unsigned int i; unsigned int n; unsigned long *args; struct pt_regs *regs; - int rw; }; -static void syscall_get_set_args_cb(struct unw_frame_info *info, void *data) +static void syscall_get_args_cb(struct unw_frame_info *info, void *data) { - struct syscall_get_set_args *args = data; + struct syscall_get_args *args = data; struct pt_regs *pt = args->regs; unsigned long *krbs, cfm, ndirty, nlocals, nouts; int i, count; @@ -2042,37 +2041,31 @@ static void syscall_get_set_args_cb(struct unw_frame_info *info, void *data) /* Iterate over outs. */ for (i = 0; i < count; i++) { int j = ndirty + nlocals + i + args->i; - if (args->rw) - *ia64_rse_skip_regs(krbs, j) = args->args[i]; - else - args->args[i] = *ia64_rse_skip_regs(krbs, j); + args->args[i] = *ia64_rse_skip_regs(krbs, j); } - if (!args->rw) { - while (i < args->n) { - args->args[i] = 0; - i++; - } + while (i < args->n) { + args->args[i] = 0; + i++; } } -void ia64_syscall_get_set_arguments(struct task_struct *task, - struct pt_regs *regs, unsigned long *args, int rw) +void syscall_get_arguments(struct task_struct *task, + struct pt_regs *regs, unsigned long *args) { - struct syscall_get_set_args data = { + struct syscall_get_args data = { .i = 0, .n = 6, .args = args, .regs = regs, - .rw = rw, }; if (task == current) - unw_init_running(syscall_get_set_args_cb, &data); + unw_init_running(syscall_get_args_cb, &data); else { struct unw_frame_info ufi; memset(&ufi, 0, sizeof(ufi)); unw_init_from_blocked_task(&ufi, task); - syscall_get_set_args_cb(&ufi, &data); + syscall_get_args_cb(&ufi, &data); } } diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index 31fb84de2d..5010348fa2 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -208,10 +208,7 @@ sort_regions (struct rsvd_region *rsvd_region, int max) while (max--) { for (j = 0; j < max; ++j) { if (rsvd_region[j].start > rsvd_region[j+1].start) { - struct rsvd_region tmp; - tmp = rsvd_region[j]; - rsvd_region[j] = rsvd_region[j + 1]; - rsvd_region[j + 1] = tmp; + swap(rsvd_region[j], rsvd_region[j + 1]); } } } diff --git a/arch/ia64/kernel/syscalls/syscall.tbl b/arch/ia64/kernel/syscalls/syscall.tbl index 6fea1844fb..78b1d03e86 100644 --- a/arch/ia64/kernel/syscalls/syscall.tbl +++ b/arch/ia64/kernel/syscalls/syscall.tbl @@ -369,3 +369,5 @@ 446 common landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c index 3639e0a7cb..e4992917a2 100644 --- a/arch/ia64/kernel/topology.c +++ b/arch/ia64/kernel/topology.c @@ -264,6 +264,7 @@ static struct attribute * cache_default_attrs[] = { &shared_cpu_map.attr, NULL }; +ATTRIBUTE_GROUPS(cache_default); #define to_object(k) container_of(k, struct cache_info, kobj) #define to_attr(a) container_of(a, struct cache_attr, attr) @@ -284,7 +285,7 @@ static const struct sysfs_ops cache_sysfs_ops = { static struct kobj_type cache_ktype = { .sysfs_ops = &cache_sysfs_ops, - .default_attrs = cache_default_attrs, + .default_groups = cache_default_groups, }; static struct kobj_type cache_ktype_percpu_entry = { diff --git a/arch/ia64/kernel/traps.c b/arch/ia64/kernel/traps.c index e13cb90593..753642366e 100644 --- a/arch/ia64/kernel/traps.c +++ b/arch/ia64/kernel/traps.c @@ -85,7 +85,7 @@ die (const char *str, struct pt_regs *regs, long err) if (panic_on_oops) panic("Fatal exception"); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); return 0; } diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c index 51883a66ae..816803636a 100644 --- a/arch/ia64/kernel/uncached.c +++ b/arch/ia64/kernel/uncached.c @@ -171,7 +171,7 @@ static int uncached_add_chunk(struct uncached_pool *uc_pool, int nid) * @n_pages: number of contiguous pages to allocate * * Allocate the specified number of contiguous uncached pages on the - * the requested node. If not enough contiguous uncached pages are available + * requested node. If not enough contiguous uncached pages are available * on the requested node, roundrobin starting with the next higher node. */ unsigned long uncached_alloc_page(int starting_nid, int n_pages) diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c index 42e025cfbd..24901d8093 100644 --- a/arch/ia64/mm/contig.c +++ b/arch/ia64/mm/contig.c @@ -153,7 +153,7 @@ find_memory (void) efi_memmap_walk(find_max_min_low_pfn, NULL); max_pfn = max_low_pfn; - memblock_add_node(0, PFN_PHYS(max_low_pfn), 0); + memblock_add_node(0, PFN_PHYS(max_low_pfn), 0, MEMBLOCK_NONE); find_initrd(); diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index 02de2e70c5..07379d1a22 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -156,17 +156,15 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + /* No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ - goto retry; - } + goto retry; } mmap_read_unlock(mm); @@ -259,7 +257,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re regs = NULL; bust_spinlocks(0); if (regs) - do_exit(SIGKILL); + make_task_dead(SIGKILL); return; out_of_memory: diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 5c6da8d83c..5d165607bf 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -378,7 +378,7 @@ int __init register_active_ranges(u64 start, u64 len, int nid) #endif if (start < end) - memblock_add_node(__pa(start), end - start, nid); + memblock_add_node(__pa(start), end - start, nid, MEMBLOCK_NONE); return 0; } diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 0b50da08a9..936e1803c7 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -9,6 +9,7 @@ config M68K select ARCH_HAVE_NMI_SAFE_CMPXCHG if RMW_INSNS select ARCH_MIGHT_HAVE_PC_PARPORT if ISA select ARCH_NO_PREEMPT if !COLDFIRE + select ARCH_USE_MEMTEST if MMU_MOTOROLA select ARCH_WANT_IPC_PARSE_VERSION select BINFMT_FLAT_ARGVP_ENVP_ON_STACK select DMA_DIRECT_REMAP if HAS_DMA && MMU && !COLDFIRE @@ -20,7 +21,6 @@ config M68K select HAVE_ASM_MODVERSIONS select HAVE_DEBUG_BUGVERBOSE select HAVE_EFFICIENT_UNALIGNED_ACCESS if !CPU_HAS_NO_UNALIGNED - select HAVE_FUTEX_CMPXCHG if MMU && FUTEX select HAVE_MOD_ARCH_SPECIFIC select HAVE_UID16 select MMU_GATHER_NO_RANGE if MMU diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu index 277d61a094..0d00ef5117 100644 --- a/arch/m68k/Kconfig.cpu +++ b/arch/m68k/Kconfig.cpu @@ -53,17 +53,6 @@ config M68000 System-On-Chip devices (eg 68328, 68302, etc). It does not contain a paging MMU. -config MCPU32 - bool - select CPU_HAS_NO_BITFIELDS - select CPU_HAS_NO_CAS - select CPU_HAS_NO_UNALIGNED - select CPU_NO_EFFICIENT_FFS - help - The Freescale (was then Motorola) CPU32 is a CPU core that is - based on the 68020 processor. For the most part it is used in - System-On-Chip parts, and does not contain a paging MMU. - config M68020 bool "68020 support" depends on MMU diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile index dd0c0ec67f..740fc97b9c 100644 --- a/arch/m68k/Makefile +++ b/arch/m68k/Makefile @@ -2,9 +2,7 @@ # m68k/Makefile # # This file is included by the global makefile so that you can add your own -# architecture-specific flags and dependencies. Remember to do have actions -# for "archclean" and "archdep" for cleaning up and making dependencies for -# this architecture +# architecture-specific flags and dependencies. # # This file is subject to the terms and conditions of the GNU General Public # License. See the file "COPYING" in the main directory of this archive diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 5f536286f5..bc9952f8be 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -45,7 +45,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -316,7 +315,6 @@ CONFIG_PARPORT_1284=y CONFIG_AMIGA_FLOPPY=y CONFIG_AMIGA_Z2RAM=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -474,6 +472,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -580,6 +580,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -618,7 +619,6 @@ CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m CONFIG_GLOB_SELFTEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -627,12 +627,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index d956864405..a77269c6e5 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -41,7 +41,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -306,7 +305,6 @@ CONFIG_DEVTMPFS_MOUNT=y CONFIG_TEST_ASYNC_DRIVER_PROBE=m CONFIG_CONNECTOR=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -431,6 +429,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -537,6 +537,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -574,7 +575,6 @@ CONFIG_PRIME_NUMBERS=m CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -583,12 +583,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index dbf1960c66..7a74efa6b9 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -48,7 +48,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -317,7 +316,6 @@ CONFIG_PARPORT_ATARI=m CONFIG_PARPORT_1284=y CONFIG_ATARI_FLOPPY=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -452,6 +450,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -558,6 +558,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -596,7 +597,6 @@ CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m CONFIG_GLOB_SELFTEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -605,12 +605,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index 7620db3e33..a5323bf2eb 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -38,7 +38,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -303,7 +302,6 @@ CONFIG_DEVTMPFS_MOUNT=y CONFIG_TEST_ASYNC_DRIVER_PROBE=m CONFIG_CONNECTOR=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -424,6 +422,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -530,6 +530,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -567,7 +568,6 @@ CONFIG_PRIME_NUMBERS=m CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -576,12 +576,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 113a02d47e..5e80aa0869 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -40,7 +40,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -305,7 +304,6 @@ CONFIG_DEVTMPFS_MOUNT=y CONFIG_TEST_ASYNC_DRIVER_PROBE=m CONFIG_CONNECTOR=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -433,6 +431,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -539,6 +539,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -576,7 +577,6 @@ CONFIG_PRIME_NUMBERS=m CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -585,12 +585,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index a8e006e8da..e84326a3f6 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -39,7 +39,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -308,7 +307,6 @@ CONFIG_TEST_ASYNC_DRIVER_PROBE=m CONFIG_CONNECTOR=m CONFIG_BLK_DEV_SWIM=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -454,6 +452,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -560,6 +560,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -598,7 +599,6 @@ CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m CONFIG_GLOB_SELFTEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -607,12 +607,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index b6655907a1..337552f433 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -59,7 +59,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -337,7 +336,6 @@ CONFIG_ATARI_FLOPPY=y CONFIG_BLK_DEV_SWIM=m CONFIG_AMIGA_Z2RAM=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -539,6 +537,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -645,6 +645,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -683,7 +684,6 @@ CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m CONFIG_GLOB_SELFTEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -692,12 +692,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 563ba47db8..7b688f7d27 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -37,7 +37,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -302,7 +301,6 @@ CONFIG_DEVTMPFS_MOUNT=y CONFIG_TEST_ASYNC_DRIVER_PROBE=m CONFIG_CONNECTOR=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -423,6 +421,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -529,6 +529,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -566,7 +567,6 @@ CONFIG_PRIME_NUMBERS=m CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -575,12 +575,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 9f1b44de47..7c2cb31d63 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -38,7 +38,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -303,7 +302,6 @@ CONFIG_DEVTMPFS_MOUNT=y CONFIG_TEST_ASYNC_DRIVER_PROBE=m CONFIG_CONNECTOR=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -424,6 +422,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -530,6 +530,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -567,7 +568,6 @@ CONFIG_PRIME_NUMBERS=m CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -576,12 +576,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 1993433d08..ca43897af2 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -39,7 +39,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -307,7 +306,6 @@ CONFIG_PARPORT=m CONFIG_PARPORT_PC=m CONFIG_PARPORT_1284=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -441,6 +439,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -547,6 +547,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -585,7 +586,6 @@ CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m CONFIG_GLOB_SELFTEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -594,12 +594,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 56dbc63cef..e3d515f371 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -35,7 +35,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -300,7 +299,6 @@ CONFIG_DEVTMPFS_MOUNT=y CONFIG_TEST_ASYNC_DRIVER_PROBE=m CONFIG_CONNECTOR=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -426,6 +424,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -532,6 +532,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -569,7 +570,6 @@ CONFIG_PRIME_NUMBERS=m CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -577,12 +577,12 @@ CONFIG_WW_MUTEX_SELFTEST=m CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 6bd1bba81a..d601606c96 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -35,7 +35,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y @@ -300,7 +299,6 @@ CONFIG_DEVTMPFS_MOUNT=y CONFIG_TEST_ASYNC_DRIVER_PROBE=m CONFIG_CONNECTOR=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_DRBD=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y @@ -425,6 +423,8 @@ CONFIG_UDF_FS=m CONFIG_MSDOS_FS=m CONFIG_VFAT_FS=m CONFIG_EXFAT_FS=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_LZX_XPRESS=y CONFIG_PROC_KCORE=y CONFIG_PROC_CHILDREN=y CONFIG_TMPFS=y @@ -531,6 +531,7 @@ CONFIG_CRYPTO_ADIANTUM=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m @@ -568,7 +569,6 @@ CONFIG_PRIME_NUMBERS=m CONFIG_CRC32_SELFTEST=m CONFIG_CRC64=m CONFIG_XZ_DEC_TEST=m -CONFIG_STRING_SELFTEST=m # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m @@ -577,12 +577,12 @@ CONFIG_EARLY_PRINTK=y CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_MIN_HEAP=m -CONFIG_TEST_SORT=m CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_HEXDUMP=m +CONFIG_STRING_SELFTEST=m CONFIG_TEST_STRING_HELPERS=m CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index 9a8394e963..9c57b245dc 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -58,7 +58,7 @@ struct nfhd_device { struct gendisk *disk; }; -static blk_qc_t nfhd_submit_bio(struct bio *bio) +static void nfhd_submit_bio(struct bio *bio) { struct nfhd_device *dev = bio->bi_bdev->bd_disk->private_data; struct bio_vec bvec; @@ -76,7 +76,6 @@ static blk_qc_t nfhd_submit_bio(struct bio *bio) sec += len; } bio_endio(bio); - return BLK_QC_T_NONE; } static int nfhd_getgeo(struct block_device *bdev, struct hd_geometry *geo) @@ -100,6 +99,7 @@ static int __init nfhd_init_one(int id, u32 blocks, u32 bsize) { struct nfhd_device *dev; int dev_id = id - NFHD_DEV_OFFSET; + int err = -ENOMEM; pr_info("nfhd%u: found device with %u blocks (%u bytes)\n", dev_id, blocks, bsize); @@ -130,16 +130,20 @@ static int __init nfhd_init_one(int id, u32 blocks, u32 bsize) sprintf(dev->disk->disk_name, "nfhd%u", dev_id); set_capacity(dev->disk, (sector_t)blocks * (bsize / 512)); blk_queue_logical_block_size(dev->disk->queue, bsize); - add_disk(dev->disk); + err = add_disk(dev->disk); + if (err) + goto out_cleanup_disk; list_add_tail(&dev->list, &nfhd_list); return 0; +out_cleanup_disk: + blk_cleanup_disk(dev->disk); free_dev: kfree(dev); out: - return -ENOMEM; + return err; } static int __init nfhd_init(void) diff --git a/arch/m68k/emu/nfeth.c b/arch/m68k/emu/nfeth.c index 79e55421cf..1a5d1e8eb4 100644 --- a/arch/m68k/emu/nfeth.c +++ b/arch/m68k/emu/nfeth.c @@ -200,7 +200,7 @@ static struct net_device * __init nfeth_probe(int unit) dev->irq = nfEtherIRQ; dev->netdev_ops = &nfeth_netdev_ops; - memcpy(dev->dev_addr, mac, ETH_ALEN); + eth_hw_addr_set(dev, mac); priv = netdev_priv(dev); priv->ethX = unit; diff --git a/arch/m68k/include/asm/bitops.h b/arch/m68k/include/asm/bitops.h index 7b414099e5..51283db536 100644 --- a/arch/m68k/include/asm/bitops.h +++ b/arch/m68k/include/asm/bitops.h @@ -451,7 +451,7 @@ static inline unsigned long ffz(unsigned long word) * generic functions for those. */ #if (defined(__mcfisaaplus__) || defined(__mcfisac__)) && \ - !defined(CONFIG_M68000) && !defined(CONFIG_MCPU32) + !defined(CONFIG_M68000) static inline unsigned long __ffs(unsigned long x) { __asm__ __volatile__ ("bitrev %0; ff1 %0" @@ -529,6 +529,4 @@ static inline int __fls(int x) #include #endif /* __KERNEL__ */ -#include - #endif /* _M68K_BITOPS_H */ diff --git a/arch/m68k/include/asm/processor.h b/arch/m68k/include/asm/processor.h index f4d82c619a..ffeda9aa52 100644 --- a/arch/m68k/include/asm/processor.h +++ b/arch/m68k/include/asm/processor.h @@ -150,7 +150,7 @@ static inline void release_thread(struct task_struct *dead_task) { } -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) \ ({ \ diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h index ba67052388..60b786eb22 100644 --- a/arch/m68k/include/asm/uaccess.h +++ b/arch/m68k/include/asm/uaccess.h @@ -12,14 +12,17 @@ #include /* We let the MMU do all checking */ -static inline int access_ok(const void __user *addr, +static inline int access_ok(const void __user *ptr, unsigned long size) { - /* - * XXX: for !CONFIG_CPU_HAS_ADDRESS_SPACES this really needs to check - * for TASK_SIZE! - */ - return 1; + unsigned long limit = TASK_SIZE; + unsigned long addr = (unsigned long)ptr; + + if (IS_ENABLED(CONFIG_CPU_HAS_ADDRESS_SPACES) || + !IS_ENABLED(CONFIG_MMU)) + return 1; + + return (size <= limit) && (addr <= (limit - size)); } /* diff --git a/arch/m68k/kernel/process.c b/arch/m68k/kernel/process.c index 1ab692b952..a6030dbaa0 100644 --- a/arch/m68k/kernel/process.c +++ b/arch/m68k/kernel/process.c @@ -263,13 +263,11 @@ int dump_fpu (struct pt_regs *regs, struct user_m68kfp_struct *fpu) } EXPORT_SYMBOL(dump_fpu); -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { unsigned long fp, pc; unsigned long stack_page; int count = 0; - if (!p || p == current || task_is_running(p)) - return 0; stack_page = (unsigned long)task_stack_page(p); fp = ((struct switch_stack *)p->thread.ksp)->a6; diff --git a/arch/m68k/kernel/ptrace.c b/arch/m68k/kernel/ptrace.c index 94b3b27418..aa3a0b8d07 100644 --- a/arch/m68k/kernel/ptrace.c +++ b/arch/m68k/kernel/ptrace.c @@ -273,17 +273,7 @@ long arch_ptrace(struct task_struct *child, long request, asmlinkage void syscall_trace(void) { - ptrace_notify(SIGTRAP | ((current->ptrace & PT_TRACESYSGOOD) - ? 0x80 : 0)); - /* - * this isn't the same as continuing with a signal, but it will do - * for normal use. strace only continues with a signal if the - * stopping signal is not SIGTRAP. -brl - */ - if (current->exit_code) { - send_sig(current->exit_code, current, 1); - current->exit_code = 0; - } + ptrace_report_syscall(0); } #if defined(CONFIG_COLDFIRE) || !defined(CONFIG_MMU) diff --git a/arch/m68k/kernel/setup_mm.c b/arch/m68k/kernel/setup_mm.c index 4b51bfd38e..49e573b943 100644 --- a/arch/m68k/kernel/setup_mm.c +++ b/arch/m68k/kernel/setup_mm.c @@ -338,13 +338,6 @@ void __init setup_arch(char **cmdline_p) panic("No configuration setup"); } - paging_init(); - -#ifdef CONFIG_NATFEAT - nf_init(); -#endif - -#ifndef CONFIG_SUN3 #ifdef CONFIG_BLK_DEV_INITRD if (m68k_ramdisk.size) { memblock_reserve(m68k_ramdisk.addr, m68k_ramdisk.size); @@ -354,6 +347,12 @@ void __init setup_arch(char **cmdline_p) } #endif + paging_init(); + +#ifdef CONFIG_NATFEAT + nf_init(); +#endif + #ifdef CONFIG_ATARI if (MACH_IS_ATARI) atari_stram_reserve_pages((void *)availmem); @@ -364,8 +363,6 @@ void __init setup_arch(char **cmdline_p) } #endif -#endif /* !CONFIG_SUN3 */ - /* set ISA defs early as possible */ #if defined(CONFIG_ISA) && defined(MULTI_ISA) if (MACH_IS_Q40) { diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl index 7976dff8f8..b1f3940bc2 100644 --- a/arch/m68k/kernel/syscalls/syscall.tbl +++ b/arch/m68k/kernel/syscalls/syscall.tbl @@ -448,3 +448,5 @@ 446 common landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/m68k/kernel/traps.c b/arch/m68k/kernel/traps.c index 34d6458340..59fc63feb0 100644 --- a/arch/m68k/kernel/traps.c +++ b/arch/m68k/kernel/traps.c @@ -1131,7 +1131,7 @@ void die_if_kernel (char *str, struct pt_regs *fp, int nr) pr_crit("%s: %08x\n", str, nr); show_registers(fp); add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } asmlinkage void set_esp0(unsigned long ssp) diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index ef46e77e97..1493cf5eac 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -48,7 +48,7 @@ int send_fault_sig(struct pt_regs *regs) pr_alert("Unable to handle kernel access"); pr_cont(" at virtual address %p\n", addr); die_if_kernel("Oops", regs, 0 /*error_code*/); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } return 1; @@ -153,18 +153,16 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* - * No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ - goto retry; - } + goto retry; } mmap_read_unlock(mm); diff --git a/arch/m68k/mm/mcfmmu.c b/arch/m68k/mm/mcfmmu.c index eac9dde651..6f1f251252 100644 --- a/arch/m68k/mm/mcfmmu.c +++ b/arch/m68k/mm/mcfmmu.c @@ -174,7 +174,8 @@ void __init cf_bootmem_alloc(void) m68k_memory[0].addr = _rambase; m68k_memory[0].size = _ramend - _rambase; - memblock_add_node(m68k_memory[0].addr, m68k_memory[0].size, 0); + memblock_add_node(m68k_memory[0].addr, m68k_memory[0].size, 0, + MEMBLOCK_NONE); /* compute total pages in system */ num_pages = PFN_DOWN(_ramend - _rambase); diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c index 9f3f77785a..ecbe948f4c 100644 --- a/arch/m68k/mm/motorola.c +++ b/arch/m68k/mm/motorola.c @@ -410,7 +410,8 @@ void __init paging_init(void) min_addr = m68k_memory[0].addr; max_addr = min_addr + m68k_memory[0].size; - memblock_add_node(m68k_memory[0].addr, m68k_memory[0].size, 0); + memblock_add_node(m68k_memory[0].addr, m68k_memory[0].size, 0, + MEMBLOCK_NONE); for (i = 1; i < m68k_num_memory;) { if (m68k_memory[i].addr < min_addr) { printk("Ignoring memory chunk at 0x%lx:0x%lx before the first chunk\n", @@ -421,7 +422,8 @@ void __init paging_init(void) (m68k_num_memory - i) * sizeof(struct m68k_mem_info)); continue; } - memblock_add_node(m68k_memory[i].addr, m68k_memory[i].size, i); + memblock_add_node(m68k_memory[i].addr, m68k_memory[i].size, i, + MEMBLOCK_NONE); addr = m68k_memory[i].addr + m68k_memory[i].size; if (addr > max_addr) max_addr = addr; @@ -455,6 +457,8 @@ void __init paging_init(void) flush_tlb_all(); + early_memtest(min_addr, max_addr); + /* * initialize the bad page table and bad page to point * to a couple of allocated pages diff --git a/arch/microblaze/Kbuild b/arch/microblaze/Kbuild index a1c5978893..077a0b8e96 100644 --- a/arch/microblaze/Kbuild +++ b/arch/microblaze/Kbuild @@ -3,3 +3,6 @@ obj-y += kernel/ obj-y += mm/ obj-$(CONFIG_PCI) += pci/ obj-y += boot/dts/ + +# for cleaning +subdir- += boot diff --git a/arch/microblaze/Makefile b/arch/microblaze/Makefile index 9adc6b6434..1826d9ce44 100644 --- a/arch/microblaze/Makefile +++ b/arch/microblaze/Makefile @@ -5,10 +5,10 @@ UTS_SYSNAME = -DUTS_SYSNAME=\"Linux\" # What CPU version are we building for, and crack it open # as major.minor.rev -CPU_VER := $(shell echo $(CONFIG_XILINX_MICROBLAZE0_HW_VER)) -CPU_MAJOR := $(shell echo $(CPU_VER) | cut -d '.' -f 1) -CPU_MINOR := $(shell echo $(CPU_VER) | cut -d '.' -f 2) -CPU_REV := $(shell echo $(CPU_VER) | cut -d '.' -f 3) +CPU_VER := $(CONFIG_XILINX_MICROBLAZE0_HW_VER) +CPU_MAJOR := $(word 1, $(subst ., , $(CPU_VER))) +CPU_MINOR := $(word 2, $(subst ., , $(CPU_VER))) +CPU_REV := $(word 3, $(subst ., , $(CPU_VER))) export CPU_VER CPU_MAJOR CPU_MINOR CPU_REV @@ -60,9 +60,6 @@ export DTB all: linux.bin -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - archheaders: $(Q)$(MAKE) $(build)=arch/microblaze/kernel/syscalls all diff --git a/arch/microblaze/boot/dts/system.dts b/arch/microblaze/boot/dts/system.dts index b7ee105677..22252451ec 100644 --- a/arch/microblaze/boot/dts/system.dts +++ b/arch/microblaze/boot/dts/system.dts @@ -347,12 +347,7 @@ xps_timer_1: timer@83c00000 { interrupts = < 3 2 >; reg = < 0x83c00000 0x10000 >; xlnx,count-width = <0x20>; - xlnx,family = "virtex5"; - xlnx,gen0-assert = <0x1>; - xlnx,gen1-assert = <0x1>; xlnx,one-timer-only = <0x0>; - xlnx,trig0-assert = <0x1>; - xlnx,trig1-assert = <0x1>; } ; } ; } ; diff --git a/arch/microblaze/include/asm/processor.h b/arch/microblaze/include/asm/processor.h index 06c6e49359..7e9e92670d 100644 --- a/arch/microblaze/include/asm/processor.h +++ b/arch/microblaze/include/asm/processor.h @@ -68,7 +68,7 @@ static inline void release_thread(struct task_struct *dead_task) { } -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); /* The size allocated for kernel stacks. This _must_ be a power of two! */ # define KERNEL_STACK_SIZE 0x2000 diff --git a/arch/microblaze/include/asm/syscall.h b/arch/microblaze/include/asm/syscall.h index 3a6924f3cb..5eb3f624cc 100644 --- a/arch/microblaze/include/asm/syscall.h +++ b/arch/microblaze/include/asm/syscall.h @@ -58,28 +58,6 @@ static inline microblaze_reg_t microblaze_get_syscall_arg(struct pt_regs *regs, return ~0; } -static inline void microblaze_set_syscall_arg(struct pt_regs *regs, - unsigned int n, - unsigned long val) -{ - switch (n) { - case 5: - regs->r10 = val; - case 4: - regs->r9 = val; - case 3: - regs->r8 = val; - case 2: - regs->r7 = val; - case 1: - regs->r6 = val; - case 0: - regs->r5 = val; - default: - BUG(); - } -} - static inline void syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, unsigned long *args) @@ -91,17 +69,6 @@ static inline void syscall_get_arguments(struct task_struct *task, *args++ = microblaze_get_syscall_arg(regs, i++); } -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - const unsigned long *args) -{ - unsigned int i = 0; - unsigned int n = 6; - - while (n--) - microblaze_set_syscall_arg(regs, i++, *args++); -} - asmlinkage unsigned long do_syscall_trace_enter(struct pt_regs *regs); asmlinkage void do_syscall_trace_leave(struct pt_regs *regs); diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index d2a8ef9f89..5b6e0e7788 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -39,24 +39,13 @@ # define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) -static inline int access_ok(const void __user *addr, unsigned long size) +static inline int __access_ok(unsigned long addr, unsigned long size) { - if (!size) - goto ok; + unsigned long limit = user_addr_max(); - if ((get_fs().seg < ((unsigned long)addr)) || - (get_fs().seg < ((unsigned long)addr + size - 1))) { - pr_devel("ACCESS fail at 0x%08x (size 0x%x), seg 0x%08x\n", - (__force u32)addr, (u32)size, - (u32)get_fs().seg); - return 0; - } -ok: - pr_devel("ACCESS OK at 0x%08x (size 0x%x), seg 0x%08x\n", - (__force u32)addr, (u32)size, - (u32)get_fs().seg); - return 1; + return (size <= limit) && (addr <= (limit - size)); } +#define access_ok(addr, size) __access_ok((unsigned long)addr, size) # define __FIXUP_SECTION ".section .fixup,\"ax\"\n" # define __EX_TABLE_SECTION ".section __ex_table,\"a\"\n" diff --git a/arch/microblaze/kernel/exceptions.c b/arch/microblaze/kernel/exceptions.c index 908788497b..fd153d5fab 100644 --- a/arch/microblaze/kernel/exceptions.c +++ b/arch/microblaze/kernel/exceptions.c @@ -44,10 +44,10 @@ void die(const char *str, struct pt_regs *fp, long err) pr_warn("Oops: %s, sig: %ld\n", str, err); show_regs(fp); spin_unlock_irq(&die_lock); - /* do_exit() should take care of panic'ing from an interrupt + /* make_task_dead() should take care of panic'ing from an interrupt * context so we don't handle it here */ - do_exit(err); + make_task_dead(err); } /* for user application debugging */ diff --git a/arch/microblaze/kernel/ftrace.c b/arch/microblaze/kernel/ftrace.c index 224eea40e1..188749d627 100644 --- a/arch/microblaze/kernel/ftrace.c +++ b/arch/microblaze/kernel/ftrace.c @@ -163,11 +163,6 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) return ret; } -int __init ftrace_dyn_arch_init(void) -{ - return 0; -} - int ftrace_update_ftrace_func(ftrace_func_t func) { unsigned long ip = (unsigned long)(&ftrace_call); diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c index 62aa237180..5e2b91c1e8 100644 --- a/arch/microblaze/kernel/process.c +++ b/arch/microblaze/kernel/process.c @@ -112,7 +112,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, return 0; } -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { /* TBD (used by procfs) */ return 0; diff --git a/arch/microblaze/kernel/signal.c b/arch/microblaze/kernel/signal.c index fc61eb0eb8..23e8a9336a 100644 --- a/arch/microblaze/kernel/signal.c +++ b/arch/microblaze/kernel/signal.c @@ -283,7 +283,7 @@ static void do_signal(struct pt_regs *regs, int in_syscall) #ifdef DEBUG_SIG pr_info("do signal: %p %d\n", regs, in_syscall); pr_info("do signal2: %lx %lx %ld [%lx]\n", regs->pc, regs->r1, - regs->r12, current_thread_info()->flags); + regs->r12, read_thread_flags()); #endif if (get_signal(&ksig)) { diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl index 6b0e11362b..820145e473 100644 --- a/arch/microblaze/kernel/syscalls/syscall.tbl +++ b/arch/microblaze/kernel/syscalls/syscall.tbl @@ -454,3 +454,5 @@ 446 common landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index b3fed2cecf..a9626e6a68 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c @@ -232,18 +232,16 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* - * No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ - goto retry; - } + goto retry; } mmap_read_unlock(mm); diff --git a/arch/microblaze/mm/pgtable.c b/arch/microblaze/mm/pgtable.c index c1833b159d..9f73265aad 100644 --- a/arch/microblaze/mm/pgtable.c +++ b/arch/microblaze/mm/pgtable.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -171,7 +172,7 @@ void __init mapin_ram(void) for (s = 0; s < lowmem_size; s += PAGE_SIZE) { f = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_SHARED | _PAGE_HWEXEC; - if ((char *) v < _stext || (char *) v >= _etext) + if (!is_kernel_text(v)) f |= _PAGE_WRENABLE; else /* On the MicroBlaze, no user access diff --git a/arch/microblaze/pci/pci-common.c b/arch/microblaze/pci/pci-common.c index 557585f1be..622a4867f9 100644 --- a/arch/microblaze/pci/pci-common.c +++ b/arch/microblaze/pci/pci-common.c @@ -587,13 +587,12 @@ static void pcibios_fixup_resources(struct pci_dev *dev) } DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pcibios_fixup_resources); -int pcibios_add_device(struct pci_dev *dev) +int pcibios_device_add(struct pci_dev *dev) { dev->irq = of_irq_parse_and_map_pci(dev, 0, 0); return 0; } -EXPORT_SYMBOL(pcibios_add_device); /* * Reparent resource children of pr that conflict with res diff --git a/arch/mips/Kbuild b/arch/mips/Kbuild index d5d6ef9bb9..9e8071f0e5 100644 --- a/arch/mips/Kbuild +++ b/arch/mips/Kbuild @@ -25,3 +25,6 @@ obj-y += vdso/ ifdef CONFIG_KVM obj-y += kvm/ endif + +# for cleaning +subdir- += boot diff --git a/arch/mips/Kbuild.platforms b/arch/mips/Kbuild.platforms index 6e3f36c841..30193bcf9c 100644 --- a/arch/mips/Kbuild.platforms +++ b/arch/mips/Kbuild.platforms @@ -19,7 +19,6 @@ platform-$(CONFIG_MACH_LOONGSON32) += loongson32/ platform-$(CONFIG_MACH_LOONGSON64) += loongson64/ platform-$(CONFIG_MIPS_MALTA) += mti-malta/ platform-$(CONFIG_MACH_NINTENDO64) += n64/ -platform-$(CONFIG_NLM_COMMON) += netlogic/ platform-$(CONFIG_PIC32MZDA) += pic32/ platform-$(CONFIG_RALINK) += ralink/ platform-$(CONFIG_MIKROTIK_RB532) += rb532/ diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 393eb21332..058446f014 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -32,7 +32,6 @@ config MIPS select GENERIC_ATOMIC64 if !64BIT select GENERIC_CMOS_UPDATE select GENERIC_CPU_AUTOPROBE - select GENERIC_FIND_FIRST_BIT select GENERIC_GETTIMEOFDAY select GENERIC_IOMAP select GENERIC_IRQ_PROBE @@ -47,7 +46,6 @@ config MIPS select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL select GUP_GET_PTE_LOW_HIGH if CPU_MIPS32 && PHYS_ADDR_T_64BIT - select HANDLE_DOMAIN_IRQ select HAVE_ARCH_COMPILER_H select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KGDB if MIPS_FP_SUPPORT @@ -57,7 +55,6 @@ config MIPS select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES select HAVE_ASM_MODVERSIONS - select HAVE_CBPF_JIT if !64BIT && !CPU_MICROMIPS select HAVE_CONTEXT_TRACKING select HAVE_TIF_NOHZ select HAVE_C_RECORDMCOUNT @@ -65,7 +62,10 @@ config MIPS select HAVE_DEBUG_STACKOVERFLOW select HAVE_DMA_CONTIGUOUS select HAVE_DYNAMIC_FTRACE - select HAVE_EBPF_JIT if 64BIT && !CPU_MICROMIPS && TARGET_ISA_REV >= 2 + select HAVE_EBPF_JIT if !CPU_MICROMIPS && \ + !CPU_DADDI_WORKAROUNDS && \ + !CPU_R4000_WORKAROUNDS && \ + !CPU_R4400_WORKAROUNDS select HAVE_EXIT_THREAD select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD @@ -263,7 +263,6 @@ config BMIPS_GENERIC bool "Broadcom Generic BMIPS kernel" select ARCH_HAS_RESET_CONTROLLER select ARCH_HAS_SYNC_DMA_FOR_CPU_ALL - select ARCH_HAS_PHYS_TO_DMA select BOOT_RAW select NO_EXCEPT_FILL select USE_OF @@ -291,6 +290,8 @@ config BMIPS_GENERIC select USB_OHCI_BIG_ENDIAN_DESC if CPU_BIG_ENDIAN select USB_OHCI_BIG_ENDIAN_MMIO if CPU_BIG_ENDIAN select HARDIRQS_SW_RESEND + select HAVE_PCI + select PCI_DRIVERS_GENERIC help Build a generic DT-based kernel image that boots on select BCM33xx cable modem chips, BCM63xx DSL chips, and BCM7xxx set-top @@ -637,9 +638,6 @@ config MACH_REALTEK_RTL select SYS_SUPPORTS_MIPS16 select SYS_SUPPORTS_MULTITHREADING select SYS_SUPPORTS_VPE_LOADER - select SYS_HAS_EARLY_PRINTK - select SYS_HAS_EARLY_PRINTK_8250 - select USE_GENERIC_EARLY_PRINTK_8250 select BOOT_RAW select PINCTRL select USE_OF @@ -762,7 +760,6 @@ config SGI_IP30 select HAVE_PCI select IRQ_MIPS_CPU select IRQ_DOMAIN_HIERARCHY - select NR_CPUS_DEFAULT_2 select PCI_DRIVERS_GENERIC select PCI_XTALK_BRIDGE select SYS_HAS_EARLY_PRINTK @@ -997,60 +994,6 @@ config CAVIUM_OCTEON_SOC Hikari Say Y here for most Octeon reference boards. -config NLM_XLR_BOARD - bool "Netlogic XLR/XLS based systems" - select BOOT_ELF32 - select NLM_COMMON - select SYS_HAS_CPU_XLR - select SYS_SUPPORTS_SMP - select HAVE_PCI - select SWAP_IO_SPACE - select SYS_SUPPORTS_32BIT_KERNEL - select SYS_SUPPORTS_64BIT_KERNEL - select PHYS_ADDR_T_64BIT - select SYS_SUPPORTS_BIG_ENDIAN - select SYS_SUPPORTS_HIGHMEM - select NR_CPUS_DEFAULT_32 - select CEVT_R4K - select CSRC_R4K - select IRQ_MIPS_CPU - select ZONE_DMA32 if 64BIT - select SYNC_R4K - select SYS_HAS_EARLY_PRINTK - select SYS_SUPPORTS_ZBOOT - select SYS_SUPPORTS_ZBOOT_UART16550 - help - Support for systems based on Netlogic XLR and XLS processors. - Say Y here if you have a XLR or XLS based board. - -config NLM_XLP_BOARD - bool "Netlogic XLP based systems" - select BOOT_ELF32 - select NLM_COMMON - select SYS_HAS_CPU_XLP - select SYS_SUPPORTS_SMP - select HAVE_PCI - select SYS_SUPPORTS_32BIT_KERNEL - select SYS_SUPPORTS_64BIT_KERNEL - select PHYS_ADDR_T_64BIT - select GPIOLIB - select SYS_SUPPORTS_BIG_ENDIAN - select SYS_SUPPORTS_LITTLE_ENDIAN - select SYS_SUPPORTS_HIGHMEM - select NR_CPUS_DEFAULT_32 - select CEVT_R4K - select CSRC_R4K - select IRQ_MIPS_CPU - select ZONE_DMA32 if 64BIT - select SYNC_R4K - select SYS_HAS_EARLY_PRINTK - select USE_OF - select SYS_SUPPORTS_ZBOOT - select SYS_SUPPORTS_ZBOOT_UART16550 - help - This board is based on Netlogic XLP Processor. - Say Y here if you have a XLP based board. - endchoice source "arch/mips/alchemy/Kconfig" @@ -1073,7 +1016,6 @@ source "arch/mips/cavium-octeon/Kconfig" source "arch/mips/loongson2ef/Kconfig" source "arch/mips/loongson32/Kconfig" source "arch/mips/loongson64/Kconfig" -source "arch/mips/netlogic/Kconfig" endmenu @@ -1215,15 +1157,6 @@ config SYS_SUPPORTS_RELOCATABLE The platform must provide plat_get_fdt() if it selects CONFIG_USE_OF to allow access to command line and entropy sources. -config MIPS_CBPF_JIT - def_bool y - depends on BPF_JIT && HAVE_CBPF_JIT - -config MIPS_EBPF_JIT - def_bool y - depends on BPF_JIT && HAVE_EBPF_JIT - - # # Endianness selection. Sufficiently obscure so many users don't know what to # answer,so we try hard to limit the available choices. Also the use of a @@ -1672,7 +1605,6 @@ config CPU_R4300 depends on SYS_HAS_CPU_R4300 select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_64BIT_KERNEL - select CPU_HAS_LOAD_STORE_LR help MIPS Technologies R4300-series processors. @@ -1786,35 +1718,10 @@ config CPU_BMIPS select CPU_HAS_PREFETCH select CPU_SUPPORTS_CPUFREQ select MIPS_EXTERNAL_TIMER + select GENERIC_IRQ_MIGRATION if HOTPLUG_CPU help Support for BMIPS32/3300/4350/4380 and BMIPS5000 processors. -config CPU_XLR - bool "Netlogic XLR SoC" - depends on SYS_HAS_CPU_XLR - select CPU_SUPPORTS_32BIT_KERNEL - select CPU_SUPPORTS_64BIT_KERNEL - select CPU_SUPPORTS_HIGHMEM - select CPU_SUPPORTS_HUGEPAGES - select WEAK_ORDERING - select WEAK_REORDERING_BEYOND_LLSC - help - Netlogic Microsystems XLR/XLS processors. - -config CPU_XLP - bool "Netlogic XLP SoC" - depends on SYS_HAS_CPU_XLP - select CPU_SUPPORTS_32BIT_KERNEL - select CPU_SUPPORTS_64BIT_KERNEL - select CPU_SUPPORTS_HIGHMEM - select WEAK_ORDERING - select WEAK_REORDERING_BEYOND_LLSC - select CPU_HAS_PREFETCH - select CPU_MIPSR2 - select CPU_SUPPORTS_HUGEPAGES - select MIPS_ASID_BITS_VARIABLE - help - Netlogic Microsystems XLP processors. endchoice config CPU_MIPS32_3_5_FEATURES @@ -2065,12 +1972,6 @@ config SYS_HAS_CPU_BMIPS5000 select SYS_HAS_CPU_BMIPS select ARCH_HAS_SYNC_DMA_FOR_CPU -config SYS_HAS_CPU_XLR - bool - -config SYS_HAS_CPU_XLP - bool - # # CPU may reorder R->R, R->W, W->R, W->W # Reordering beyond LL and SC is handled in WEAK_REORDERING_BEYOND_LLSC @@ -2165,7 +2066,7 @@ config CPU_SUPPORTS_HUGEPAGES config MIPS_PGD_C0_CONTEXT bool depends on 64BIT - default y if (CPU_MIPSR2 || CPU_MIPSR6) && !CPU_XLP + default y if (CPU_MIPSR2 || CPU_MIPSR6) # # Set to y for ptrace access to watch registers. @@ -2212,6 +2113,16 @@ config MIPS_VA_BITS_48 If unsure, say N. +config ZBOOT_LOAD_ADDRESS + hex "Compressed kernel load address" + default 0xffffffff80400000 if BCM47XX + default 0x0 + depends on SYS_SUPPORTS_ZBOOT + help + The address to load compressed kernel, aka vmlinuz. + + This is only used if non-zero. + choice prompt "Kernel page size" default PAGE_SIZE_4KB @@ -2762,6 +2673,8 @@ config NUMA bool "NUMA Support" depends on SYS_SUPPORTS_NUMA select SMP + select HAVE_SETUP_PER_CPU_AREA + select NEED_PER_CPU_EMBED_FIRST_CHUNK help Say Y to compile the kernel to support NUMA (Non-Uniform Memory Access). This option improves performance on systems with more @@ -2772,14 +2685,6 @@ config NUMA config SYS_SUPPORTS_NUMA bool -config HAVE_SETUP_PER_CPU_AREA - def_bool y - depends on NUMA - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y - depends on NUMA - config RELOCATABLE bool "Relocatable kernel" depends on SYS_SUPPORTS_RELOCATABLE @@ -2848,7 +2753,7 @@ config NODES_SHIFT config HW_PERF_EVENTS bool "Enable hardware performance counter support for perf events" - depends on PERF_EVENTS && (CPU_MIPS32 || CPU_MIPS64 || CPU_R10000 || CPU_SB1 || CPU_CAVIUM_OCTEON || CPU_XLP || CPU_LOONGSON64) + depends on PERF_EVENTS && (CPU_MIPS32 || CPU_MIPS64 || CPU_R10000 || CPU_SB1 || CPU_CAVIUM_OCTEON || CPU_LOONGSON64) default y help Enable hardware performance counter support for perf events. If diff --git a/arch/mips/Makefile b/arch/mips/Makefile index f7b58da2f3..e036fc025c 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -8,8 +8,7 @@ # Copyright (C) 2002, 2003, 2004 Maciej W. Rozycki # # This file is included by the global makefile so that you can add your own -# architecture-specific flags and dependencies. Remember to do have actions -# for "archclean" cleaning up for this architecture. +# architecture-specific flags and dependencies. # archscripts: scripts_basic @@ -254,9 +253,7 @@ endif # # Board-dependent options and extra files # -ifdef need-compiler include $(srctree)/arch/mips/Kbuild.platforms -endif ifdef CONFIG_PHYSICAL_START load-y = $(CONFIG_PHYSICAL_START) @@ -428,11 +425,6 @@ endif $(Q)install -D -m 644 .config $(INSTALL_PATH)/config-$(KERNELRELEASE) $(Q)install -D -m 644 System.map $(INSTALL_PATH)/System.map-$(KERNELRELEASE) -archclean: - $(Q)$(MAKE) $(clean)=arch/mips/boot - $(Q)$(MAKE) $(clean)=arch/mips/boot/compressed - $(Q)$(MAKE) $(clean)=arch/mips/boot/tools - archheaders: $(Q)$(MAKE) $(build)=arch/mips/kernel/syscalls all diff --git a/arch/mips/alchemy/common/gpiolib.c b/arch/mips/alchemy/common/gpiolib.c index 7d5da5edd7..a17d7a8909 100644 --- a/arch/mips/alchemy/common/gpiolib.c +++ b/arch/mips/alchemy/common/gpiolib.c @@ -23,8 +23,6 @@ * 675 Mass Ave, Cambridge, MA 02139, USA. * * Notes : - * This file must ONLY be built when CONFIG_GPIOLIB=y and - * CONFIG_ALCHEMY_GPIO_INDIRECT=n, otherwise compilation will fail! * au1000 SoC have only one GPIO block : GPIO1 * Au1100, Au15x0, Au12x0 have a second one : GPIO2 * Au1300 is totally different: 1 block with up to 128 GPIOs diff --git a/arch/mips/alchemy/devboards/db1550.c b/arch/mips/alchemy/devboards/db1550.c index 752b93d91a..fd91d9c9a2 100644 --- a/arch/mips/alchemy/devboards/db1550.c +++ b/arch/mips/alchemy/devboards/db1550.c @@ -66,6 +66,7 @@ int __init db1550_board_setup(void) case BCSR_WHOAMI_PB1550_DDR: bcsr_init(PB1550_BCSR_PHYS_ADDR, PB1550_BCSR_PHYS_ADDR + PB1550_BCSR_HEXLED_OFS); + break; case BCSR_WHOAMI_DB1550: break; default: diff --git a/arch/mips/ath79/setup.c b/arch/mips/ath79/setup.c index 891f495c4c..0ac435fe2d 100644 --- a/arch/mips/ath79/setup.c +++ b/arch/mips/ath79/setup.c @@ -34,15 +34,6 @@ static char ath79_sys_type[ATH79_SYS_TYPE_LEN]; -static void ath79_restart(char *command) -{ - local_irq_disable(); - ath79_device_reset_set(AR71XX_RESET_FULL_CHIP); - for (;;) - if (cpu_wait) - cpu_wait(); -} - static void ath79_halt(void) { while (1) @@ -234,7 +225,6 @@ void __init plat_mem_setup(void) detect_memory_region(0, ATH79_MEM_SIZE_MIN, ATH79_MEM_SIZE_MAX); - _machine_restart = ath79_restart; _machine_halt = ath79_halt; pm_power_off = ath79_halt; } diff --git a/arch/mips/bcm47xx/Platform b/arch/mips/bcm47xx/Platform index 833b204fe5..fe6daba3f9 100644 --- a/arch/mips/bcm47xx/Platform +++ b/arch/mips/bcm47xx/Platform @@ -4,4 +4,3 @@ cflags-$(CONFIG_BCM47XX) += \ -I$(srctree)/arch/mips/include/asm/mach-bcm47xx load-$(CONFIG_BCM47XX) := 0xffffffff80001000 -zload-$(CONFIG_BCM47XX) += 0xffffffff80400000 diff --git a/arch/mips/bcm47xx/board.c b/arch/mips/bcm47xx/board.c index 35266a70e2..87dc76a1f9 100644 --- a/arch/mips/bcm47xx/board.c +++ b/arch/mips/bcm47xx/board.c @@ -141,6 +141,7 @@ struct bcm47xx_board_type_list2 bcm47xx_board_list_boot_hw[] __initconst = { {{BCM47XX_BOARD_LINKSYS_WRT300NV11, "Linksys WRT300N V1.1"}, "WRT300N", "1.1"}, {{BCM47XX_BOARD_LINKSYS_WRT310NV1, "Linksys WRT310N V1"}, "WRT310N", "1.0"}, {{BCM47XX_BOARD_LINKSYS_WRT310NV2, "Linksys WRT310N V2"}, "WRT310N", "2.0"}, + {{BCM47XX_BOARD_LINKSYS_WRT320N_V1, "Linksys WRT320N V1"}, "WRT320N", "1.0"}, {{BCM47XX_BOARD_LINKSYS_WRT54G3GV2, "Linksys WRT54G3GV2-VF"}, "WRT54G3GV2-VF", "1.0"}, {{BCM47XX_BOARD_LINKSYS_WRT610NV1, "Linksys WRT610N V1"}, "WRT610N", "1.0"}, {{BCM47XX_BOARD_LINKSYS_WRT610NV2, "Linksys WRT610N V2"}, "WRT610N", "2.0"}, @@ -161,9 +162,12 @@ struct bcm47xx_board_type_list1 bcm47xx_board_list_board_id[] __initconst = { {{BCM47XX_BOARD_LUXUL_XWR_600_V1, "Luxul XWR-600 V1"}, "luxul_xwr600_v1"}, {{BCM47XX_BOARD_LUXUL_XWR_1750_V1, "Luxul XWR-1750 V1"}, "luxul_xwr1750_v1"}, {{BCM47XX_BOARD_NETGEAR_R6200_V1, "Netgear R6200 V1"}, "U12H192T00_NETGEAR"}, + {{BCM47XX_BOARD_NETGEAR_R6300_V1, "Netgear R6300 V1"}, "U12H218T00_NETGEAR"}, {{BCM47XX_BOARD_NETGEAR_WGR614V8, "Netgear WGR614 V8"}, "U12H072T00_NETGEAR"}, {{BCM47XX_BOARD_NETGEAR_WGR614V9, "Netgear WGR614 V9"}, "U12H094T00_NETGEAR"}, {{BCM47XX_BOARD_NETGEAR_WGR614_V10, "Netgear WGR614 V10"}, "U12H139T01_NETGEAR"}, + {{BCM47XX_BOARD_NETGEAR_WN2500RP_V1, "Netgear WN2500RP V1"}, "U12H197T00_NETGEAR"}, + {{BCM47XX_BOARD_NETGEAR_WN2500RP_V2, "Netgear WN2500RP V2"}, "U12H294T00_NETGEAR"}, {{BCM47XX_BOARD_NETGEAR_WNDR3300, "Netgear WNDR3300"}, "U12H093T00_NETGEAR"}, {{BCM47XX_BOARD_NETGEAR_WNDR3400V1, "Netgear WNDR3400 V1"}, "U12H155T00_NETGEAR"}, {{BCM47XX_BOARD_NETGEAR_WNDR3400V2, "Netgear WNDR3400 V2"}, "U12H187T00_NETGEAR"}, @@ -345,7 +349,7 @@ void __init bcm47xx_board_detect(void) board_detected = bcm47xx_board_get_nvram(); bcm47xx_board.board = board_detected->board; - strlcpy(bcm47xx_board.name, board_detected->name, + strscpy(bcm47xx_board.name, board_detected->name, BCM47XX_BOARD_MAX_NAME); } diff --git a/arch/mips/bcm47xx/buttons.c b/arch/mips/bcm47xx/buttons.c index 535d84addc..36f0b1aafa 100644 --- a/arch/mips/bcm47xx/buttons.c +++ b/arch/mips/bcm47xx/buttons.c @@ -26,6 +26,12 @@ /* Asus */ +static const struct gpio_keys_button +bcm47xx_buttons_asus_rtn10u[] __initconst = { + BCM47XX_GPIO_KEY(20, KEY_WPS_BUTTON), + BCM47XX_GPIO_KEY(21, KEY_RESTART), +}; + static const struct gpio_keys_button bcm47xx_buttons_asus_rtn12[] __initconst = { BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON), @@ -276,6 +282,18 @@ bcm47xx_buttons_linksys_wrt310nv1[] __initconst = { BCM47XX_GPIO_KEY(8, KEY_UNKNOWN), }; +static const struct gpio_keys_button +bcm47xx_buttons_linksys_wrt310n_v2[] __initconst = { + BCM47XX_GPIO_KEY(5, KEY_WPS_BUTTON), + BCM47XX_GPIO_KEY(6, KEY_RESTART), +}; + +static const struct gpio_keys_button +bcm47xx_buttons_linksys_wrt320n_v1[] __initconst = { + BCM47XX_GPIO_KEY(5, KEY_WPS_BUTTON), + BCM47XX_GPIO_KEY(8, KEY_RESTART), +}; + static const struct gpio_keys_button bcm47xx_buttons_linksys_wrt54g3gv2[] __initconst = { BCM47XX_GPIO_KEY(5, KEY_WIMAX), @@ -391,6 +409,17 @@ bcm47xx_buttons_netgear_r6200_v1[] __initconst = { BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON), }; +static const struct gpio_keys_button +bcm47xx_buttons_netgear_r6300_v1[] __initconst = { + BCM47XX_GPIO_KEY(6, KEY_RESTART), +}; + +static const struct gpio_keys_button +bcm47xx_buttons_netgear_wn2500rp_v1[] __initconst = { + BCM47XX_GPIO_KEY(12, KEY_RESTART), + BCM47XX_GPIO_KEY(31, KEY_WPS_BUTTON), +}; + static const struct gpio_keys_button bcm47xx_buttons_netgear_wndr3400v1[] __initconst = { BCM47XX_GPIO_KEY(4, KEY_RESTART), @@ -478,6 +507,9 @@ int __init bcm47xx_buttons_register(void) int err; switch (board) { + case BCM47XX_BOARD_ASUS_RTN10U: + err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_rtn10u); + break; case BCM47XX_BOARD_ASUS_RTN12: err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_rtn12); break; @@ -608,6 +640,12 @@ int __init bcm47xx_buttons_register(void) case BCM47XX_BOARD_LINKSYS_WRT310NV1: err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt310nv1); break; + case BCM47XX_BOARD_LINKSYS_WRT310NV2: + err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt310n_v2); + break; + case BCM47XX_BOARD_LINKSYS_WRT320N_V1: + err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt320n_v1); + break; case BCM47XX_BOARD_LINKSYS_WRT54G3GV2: err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt54g3gv2); break; @@ -674,6 +712,12 @@ int __init bcm47xx_buttons_register(void) case BCM47XX_BOARD_NETGEAR_R6200_V1: err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_r6200_v1); break; + case BCM47XX_BOARD_NETGEAR_R6300_V1: + err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_r6300_v1); + break; + case BCM47XX_BOARD_NETGEAR_WN2500RP_V1: + err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wn2500rp_v1); + break; case BCM47XX_BOARD_NETGEAR_WNDR3400V1: err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wndr3400v1); break; diff --git a/arch/mips/bcm47xx/leds.c b/arch/mips/bcm47xx/leds.c index 167c42c71e..4648a302a3 100644 --- a/arch/mips/bcm47xx/leds.c +++ b/arch/mips/bcm47xx/leds.c @@ -29,6 +29,14 @@ /* Asus */ +static const struct gpio_led +bcm47xx_leds_asus_rtn10u[] __initconst = { + BCM47XX_GPIO_LED(5, "green", "wlan", 0, LEDS_GPIO_DEFSTATE_OFF), + BCM47XX_GPIO_LED(6, "green", "power", 1, LEDS_GPIO_DEFSTATE_ON), + BCM47XX_GPIO_LED(7, "green", "wps", 0, LEDS_GPIO_DEFSTATE_OFF), + BCM47XX_GPIO_LED(8, "green", "usb", 0, LEDS_GPIO_DEFSTATE_OFF), +}; + static const struct gpio_led bcm47xx_leds_asus_rtn12[] __initconst = { BCM47XX_GPIO_LED(2, "unk", "power", 1, LEDS_GPIO_DEFSTATE_ON), @@ -313,6 +321,13 @@ bcm47xx_leds_linksys_wrt310nv1[] __initconst = { BCM47XX_GPIO_LED(9, "blue", "wps", 1, LEDS_GPIO_DEFSTATE_OFF), }; +static const struct gpio_led +bcm47xx_leds_linksys_wrt320n_v1[] __initconst = { + BCM47XX_GPIO_LED(1, "blue", "wlan", 1, LEDS_GPIO_DEFSTATE_OFF), + BCM47XX_GPIO_LED(2, "blue", "power", 0, LEDS_GPIO_DEFSTATE_ON), + BCM47XX_GPIO_LED(4, "amber", "wps", 1, LEDS_GPIO_DEFSTATE_OFF), +}; + static const struct gpio_led bcm47xx_leds_linksys_wrt54g_generic[] __initconst = { BCM47XX_GPIO_LED(0, "unk", "dmz", 1, LEDS_GPIO_DEFSTATE_OFF), @@ -556,6 +571,9 @@ void __init bcm47xx_leds_register(void) enum bcm47xx_board board = bcm47xx_board_get(); switch (board) { + case BCM47XX_BOARD_ASUS_RTN10U: + bcm47xx_set_pdata(bcm47xx_leds_asus_rtn10u); + break; case BCM47XX_BOARD_ASUS_RTN12: bcm47xx_set_pdata(bcm47xx_leds_asus_rtn12); break; @@ -689,6 +707,9 @@ void __init bcm47xx_leds_register(void) case BCM47XX_BOARD_LINKSYS_WRT310NV1: bcm47xx_set_pdata(bcm47xx_leds_linksys_wrt310nv1); break; + case BCM47XX_BOARD_LINKSYS_WRT320N_V1: + bcm47xx_set_pdata(bcm47xx_leds_linksys_wrt320n_v1); + break; case BCM47XX_BOARD_LINKSYS_WRT54G3GV2: bcm47xx_set_pdata(bcm47xx_leds_linksys_wrt54g3gv2); break; diff --git a/arch/mips/bcm63xx/dev-wdt.c b/arch/mips/bcm63xx/dev-wdt.c index 2a2346a99b..42130914a3 100644 --- a/arch/mips/bcm63xx/dev-wdt.c +++ b/arch/mips/bcm63xx/dev-wdt.c @@ -9,6 +9,7 @@ #include #include #include +#include #include static struct resource wdt_resources[] = { @@ -19,11 +20,18 @@ static struct resource wdt_resources[] = { }, }; +static struct bcm7038_wdt_platform_data bcm63xx_wdt_pdata = { + .clk_name = "periph", +}; + static struct platform_device bcm63xx_wdt_device = { .name = "bcm63xx-wdt", .id = -1, .num_resources = ARRAY_SIZE(wdt_resources), .resource = wdt_resources, + .dev = { + .platform_data = &bcm63xx_wdt_pdata, + }, }; int __init bcm63xx_wdt_register(void) diff --git a/arch/mips/bmips/dma.c b/arch/mips/bmips/dma.c index 915ce4b189..c535f9cb75 100644 --- a/arch/mips/bmips/dma.c +++ b/arch/mips/bmips/dma.c @@ -1,68 +1,8 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2014 Kevin Cernekee - */ +// SPDX-License-Identifier: GPL-2.0+ -#define pr_fmt(fmt) "bmips-dma: " fmt - -#include -#include -#include -#include -#include -#include -#include -#include #include #include - -/* - * BCM338x has configurable address translation windows which allow the - * peripherals' DMA addresses to be different from the Zephyr-visible - * physical addresses. e.g. usb_dma_addr = zephyr_pa ^ 0x08000000 - * - * If the "brcm,ubus" node has a "dma-ranges" property we will enable this - * translation globally using the provided information. This implements a - * very limited subset of "dma-ranges" support and it will probably be - * replaced by a more generic version later. - */ - -struct bmips_dma_range { - u32 child_addr; - u32 parent_addr; - u32 size; -}; - -static struct bmips_dma_range *bmips_dma_ranges; - -#define FLUSH_RAC 0x100 - -dma_addr_t phys_to_dma(struct device *dev, phys_addr_t pa) -{ - struct bmips_dma_range *r; - - for (r = bmips_dma_ranges; r && r->size; r++) { - if (pa >= r->child_addr && - pa < (r->child_addr + r->size)) - return pa - r->child_addr + r->parent_addr; - } - return pa; -} - -phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dma_addr) -{ - struct bmips_dma_range *r; - - for (r = bmips_dma_ranges; r && r->size; r++) { - if (dma_addr >= r->parent_addr && - dma_addr < (r->parent_addr + r->size)) - return dma_addr - r->parent_addr + r->child_addr; - } - return dma_addr; -} +#include void arch_sync_dma_for_cpu_all(void) { @@ -79,45 +19,3 @@ void arch_sync_dma_for_cpu_all(void) __raw_writel(cfg | 0x100, cbr + BMIPS_RAC_CONFIG); __raw_readl(cbr + BMIPS_RAC_CONFIG); } - -static int __init bmips_init_dma_ranges(void) -{ - struct device_node *np = - of_find_compatible_node(NULL, NULL, "brcm,ubus"); - const __be32 *data; - struct bmips_dma_range *r; - int len; - - if (!np) - return 0; - - data = of_get_property(np, "dma-ranges", &len); - if (!data) - goto out_good; - - len /= sizeof(*data) * 3; - if (!len) - goto out_bad; - - /* add a dummy (zero) entry at the end as a sentinel */ - bmips_dma_ranges = kcalloc(len + 1, sizeof(struct bmips_dma_range), - GFP_KERNEL); - if (!bmips_dma_ranges) - goto out_bad; - - for (r = bmips_dma_ranges; len; len--, r++) { - r->child_addr = be32_to_cpup(data++); - r->parent_addr = be32_to_cpup(data++); - r->size = be32_to_cpup(data++); - } - -out_good: - of_node_put(np); - return 0; - -out_bad: - pr_err("error parsing dma-ranges property\n"); - of_node_put(np); - return -EINVAL; -} -arch_initcall(bmips_init_dma_ranges); diff --git a/arch/mips/boot/Makefile b/arch/mips/boot/Makefile index a3da2c5d63..196c44fa72 100644 --- a/arch/mips/boot/Makefile +++ b/arch/mips/boot/Makefile @@ -171,3 +171,6 @@ $(obj)/vmlinux.itb: $(obj)/vmlinux.its $(obj)/vmlinux.bin FORCE $(obj)/vmlinux.%.itb: $(obj)/vmlinux.%.its $(obj)/vmlinux.bin.% FORCE $(call if_changed,itb-image,$<) + +# for cleaning +subdir- += compressed tools diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile index 705b9e7f80..5a15d51e88 100644 --- a/arch/mips/boot/compressed/Makefile +++ b/arch/mips/boot/compressed/Makefile @@ -50,26 +50,10 @@ vmlinuzobjs-$(CONFIG_MIPS_ALCHEMY) += $(obj)/uart-alchemy.o vmlinuzobjs-$(CONFIG_ATH79) += $(obj)/uart-ath79.o endif -extra-y += uart-ath79.c -$(obj)/uart-ath79.c: $(srctree)/arch/mips/ath79/early_printk.c - $(call cmd,shipped) - vmlinuzobjs-$(CONFIG_KERNEL_XZ) += $(obj)/ashldi3.o vmlinuzobjs-$(CONFIG_KERNEL_ZSTD) += $(obj)/bswapdi.o $(obj)/ashldi3.o $(obj)/clz_ctz.o -extra-y += ashldi3.c -$(obj)/ashldi3.c: $(obj)/%.c: $(srctree)/lib/%.c FORCE - $(call if_changed,shipped) - -extra-y += bswapsi.c -$(obj)/bswapsi.c: $(obj)/%.c: $(srctree)/arch/mips/lib/%.c FORCE - $(call if_changed,shipped) - -extra-y += bswapdi.c -$(obj)/bswapdi.c: $(obj)/%.c: $(srctree)/arch/mips/lib/%.c FORCE - $(call if_changed,shipped) - targets := $(notdir $(vmlinuzobjs-y)) targets += vmlinux.bin @@ -80,12 +64,12 @@ $(obj)/vmlinux.bin: $(KBUILD_IMAGE) FORCE $(call if_changed,objcopy) tool_$(CONFIG_KERNEL_GZIP) = gzip -tool_$(CONFIG_KERNEL_BZIP2) = bzip2 -tool_$(CONFIG_KERNEL_LZ4) = lz4 -tool_$(CONFIG_KERNEL_LZMA) = lzma -tool_$(CONFIG_KERNEL_LZO) = lzo -tool_$(CONFIG_KERNEL_XZ) = xzkern -tool_$(CONFIG_KERNEL_ZSTD) = zstd22 +tool_$(CONFIG_KERNEL_BZIP2) = bzip2_with_size +tool_$(CONFIG_KERNEL_LZ4) = lz4_with_size +tool_$(CONFIG_KERNEL_LZMA) = lzma_with_size +tool_$(CONFIG_KERNEL_LZO) = lzo_with_size +tool_$(CONFIG_KERNEL_XZ) = xzkern_with_size +tool_$(CONFIG_KERNEL_ZSTD) = zstd22_with_size targets += vmlinux.bin.z @@ -105,6 +89,10 @@ HOSTCFLAGS_calc_vmlinuz_load_addr.o += $(LINUXINCLUDE) # Calculate the load address of the compressed kernel image hostprogs := calc_vmlinuz_load_addr +ifneq (0x0,$(CONFIG_ZBOOT_LOAD_ADDRESS)) +zload-y = $(CONFIG_ZBOOT_LOAD_ADDRESS) +endif + ifneq ($(zload-y),) VMLINUZ_LOAD_ADDRESS := $(zload-y) else diff --git a/arch/mips/boot/compressed/uart-16550.c b/arch/mips/boot/compressed/uart-16550.c index c18d7f72d9..96d28f2111 100644 --- a/arch/mips/boot/compressed/uart-16550.c +++ b/arch/mips/boot/compressed/uart-16550.c @@ -23,18 +23,6 @@ #define PORT(offset) (CKSEG1ADDR(INGENIC_UART_BASE_ADDR) + (4 * offset)) #endif -#ifdef CONFIG_CPU_XLR -#define UART0_BASE 0x1EF14000 -#define PORT(offset) (CKSEG1ADDR(UART0_BASE) + (4 * offset)) -#define IOTYPE unsigned int -#endif - -#ifdef CONFIG_CPU_XLP -#define UART0_BASE 0x18030100 -#define PORT(offset) (CKSEG1ADDR(UART0_BASE) + (4 * offset)) -#define IOTYPE unsigned int -#endif - #ifndef IOTYPE #define IOTYPE char #endif diff --git a/arch/mips/boot/dts/Makefile b/arch/mips/boot/dts/Makefile index be96d35eb5..928f38a79d 100644 --- a/arch/mips/boot/dts/Makefile +++ b/arch/mips/boot/dts/Makefile @@ -9,7 +9,6 @@ subdir-$(CONFIG_MACH_LOONGSON64) += loongson subdir-$(CONFIG_SOC_VCOREIII) += mscc subdir-$(CONFIG_MIPS_MALTA) += mti subdir-$(CONFIG_LEGACY_BOARD_SEAD3) += mti -subdir-$(CONFIG_NLM_XLP_BOARD) += netlogic subdir-$(CONFIG_FIT_IMAGE_FDT_NI169445) += ni subdir-$(CONFIG_MACH_PIC32) += pic32 subdir-$(CONFIG_ATH79) += qca diff --git a/arch/mips/boot/dts/brcm/bcm7425.dtsi b/arch/mips/boot/dts/brcm/bcm7425.dtsi index aa0b2d39c9..62588c53d3 100644 --- a/arch/mips/boot/dts/brcm/bcm7425.dtsi +++ b/arch/mips/boot/dts/brcm/bcm7425.dtsi @@ -584,4 +584,34 @@ shimphy@8000 { }; }; }; + + pcie_0: pcie@8b20000 { + status = "disabled"; + compatible = "brcm,bcm7425-pcie"; + + ranges = <0x02000000 0x0 0xd0000000 0xd0000000 0x0 0x08000000 + 0x02000000 0x0 0xd8000000 0xd8000000 0x0 0x08000000 + 0x02000000 0x0 0xe0000000 0xe0000000 0x0 0x08000000 + 0x02000000 0x0 0xe8000000 0xe8000000 0x0 0x08000000>; + + reg = <0x10410000 0x19310>; + aspm-no-l0s; + device_type = "pci"; + msi-controller; + msi-parent = <&pcie_0>; + #address-cells = <0x3>; + #size-cells = <0x2>; + bus-range = <0x0 0xff>; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + linux,pci-domain = <0x0>; + + interrupt-parent = <&periph_intc>; + interrupts = <37>, <37>; + interrupt-names = "pcie", "msi"; + #interrupt-cells = <0x1>; + interrupt-map = <0 0 0 1 &periph_intc 0x21 + 0 0 0 1 &periph_intc 0x22 + 0 0 0 1 &periph_intc 0x23 + 0 0 0 1 &periph_intc 0x24>; + }; }; diff --git a/arch/mips/boot/dts/brcm/bcm7435.dtsi b/arch/mips/boot/dts/brcm/bcm7435.dtsi index 8398b7f68b..8c001b944c 100644 --- a/arch/mips/boot/dts/brcm/bcm7435.dtsi +++ b/arch/mips/boot/dts/brcm/bcm7435.dtsi @@ -599,4 +599,34 @@ shimphy@8000 { }; }; }; + + pcie_0: pcie@8b20000 { + status = "disabled"; + compatible = "brcm,bcm7435-pcie"; + + ranges = <0x02000000 0x0 0xd0000000 0xd0000000 0x0 0x08000000 + 0x02000000 0x0 0xd8000000 0xd8000000 0x0 0x08000000 + 0x02000000 0x0 0xe0000000 0xe0000000 0x0 0x08000000 + 0x02000000 0x0 0xe8000000 0xe8000000 0x0 0x08000000>; + + reg = <0x10410000 0x19310>; + aspm-no-l0s; + device_type = "pci"; + msi-controller; + msi-parent = <&pcie_0>; + #address-cells = <0x3>; + #size-cells = <0x2>; + bus-range = <0x0 0xff>; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + linux,pci-domain = <0x0>; + + interrupt-parent = <&periph_intc>; + interrupts = <39>, <39>; + interrupt-names = "pcie", "msi"; + #interrupt-cells = <0x1>; + interrupt-map = <0 0 0 1 &periph_intc 0x23 + 0 0 0 1 &periph_intc 0x24 + 0 0 0 1 &periph_intc 0x25 + 0 0 0 1 &periph_intc 0x26>; + }; }; diff --git a/arch/mips/boot/dts/brcm/bcm97425svmb.dts b/arch/mips/boot/dts/brcm/bcm97425svmb.dts index 9efecfe1e0..f389349343 100644 --- a/arch/mips/boot/dts/brcm/bcm97425svmb.dts +++ b/arch/mips/boot/dts/brcm/bcm97425svmb.dts @@ -152,3 +152,12 @@ &mspi { &waketimer { status = "okay"; }; + +&pcie_0 { + status = "okay"; + /* 1GB Memc0, 1GB Memc1 */ + brcm,scb-sizes = <0 0x40000000 0 0x40000000>; + dma-ranges = <0x43000000 0x00000000 0x00000000 0x00000000 0x0 0x10000000 + 0x43000000 0x00000000 0x10000000 0x20000000 0x0 0x30000000 + 0x43000000 0x00000000 0x40000000 0x90000000 0x0 0x40000000>; +}; diff --git a/arch/mips/boot/dts/brcm/bcm97435svmb.dts b/arch/mips/boot/dts/brcm/bcm97435svmb.dts index b653c6ff74..a0cf53e23c 100644 --- a/arch/mips/boot/dts/brcm/bcm97435svmb.dts +++ b/arch/mips/boot/dts/brcm/bcm97435svmb.dts @@ -128,3 +128,12 @@ &mspi { &waketimer { status = "okay"; }; + +&pcie_0 { + status = "okay"; + /* 1GB Memc0, 1GB Memc1 */ + brcm,scb-sizes = <0 0x40000000 0 0x40000000>; + dma-ranges = <0x43000000 0x00000000 0x00000000 0x00000000 0x0 0x10000000 + 0x43000000 0x00000000 0x10000000 0x20000000 0x0 0x30000000 + 0x43000000 0x00000000 0x40000000 0x90000000 0x0 0x40000000>; +}; diff --git a/arch/mips/boot/dts/ingenic/ci20.dts b/arch/mips/boot/dts/ingenic/ci20.dts index a688809bee..ab6e3dc0bc 100644 --- a/arch/mips/boot/dts/ingenic/ci20.dts +++ b/arch/mips/boot/dts/ingenic/ci20.dts @@ -78,6 +78,20 @@ eth0_power: fixedregulator@0 { enable-active-high; }; + hdmi_out: connector { + compatible = "hdmi-connector"; + label = "HDMI OUT"; + type = "a"; + + ddc-en-gpios = <&gpa 25 GPIO_ACTIVE_HIGH>; + + port { + hdmi_con: endpoint { + remote-endpoint = <&dw_hdmi_out>; + }; + }; + }; + ir: ir { compatible = "gpio-ir-receiver"; gpios = <&gpe 3 GPIO_ACTIVE_LOW>; @@ -113,9 +127,13 @@ &cgu { * Use the 32.768 kHz oscillator as the parent of the RTC for a higher * precision. */ - assigned-clocks = <&cgu JZ4780_CLK_OTGPHY>, <&cgu JZ4780_CLK_RTC>; - assigned-clock-parents = <0>, <&cgu JZ4780_CLK_RTCLK>; - assigned-clock-rates = <48000000>; + assigned-clocks = <&cgu JZ4780_CLK_OTGPHY>, <&cgu JZ4780_CLK_RTC>, + <&cgu JZ4780_CLK_SSIPLL>, <&cgu JZ4780_CLK_SSI>, + <&cgu JZ4780_CLK_HDMI>; + assigned-clock-parents = <0>, <&cgu JZ4780_CLK_RTCLK>, + <&cgu JZ4780_CLK_MPLL>, + <&cgu JZ4780_CLK_SSIPLL>; + assigned-clock-rates = <48000000>, <0>, <54000000>, <0>, <27000000>; }; &tcu { @@ -506,6 +524,12 @@ pins_i2c4: i2c4 { bias-disable; }; + pins_hdmi_ddc: hdmi_ddc { + function = "hdmi-ddc"; + groups = "hdmi-ddc"; + bias-disable; + }; + pins_nemc: nemc { function = "nemc"; groups = "nemc-data", "nemc-cle-ale", "nemc-rd-we", "nemc-frd-fwe"; @@ -536,3 +560,39 @@ pins_mmc1: mmc1 { bias-disable; }; }; + +&hdmi { + status = "okay"; + + pinctrl-names = "default"; + pinctrl-0 = <&pins_hdmi_ddc>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dw_hdmi_in: endpoint { + remote-endpoint = <&lcd_out>; + }; + }; + + port@1 { + reg = <1>; + dw_hdmi_out: endpoint { + remote-endpoint = <&hdmi_con>; + }; + }; + }; +}; + +&lcdc0 { + status = "okay"; + + port { + lcd_out: endpoint { + remote-endpoint = <&dw_hdmi_in>; + }; + }; +}; diff --git a/arch/mips/boot/dts/ingenic/jz4725b.dtsi b/arch/mips/boot/dts/ingenic/jz4725b.dtsi index a1f0b71c92..e9e48022f6 100644 --- a/arch/mips/boot/dts/ingenic/jz4725b.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4725b.dtsi @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include +#include #include / { @@ -321,7 +321,7 @@ udc: usb@13040000 { lcd: lcd-controller@13050000 { compatible = "ingenic,jz4725b-lcd"; - reg = <0x13050000 0x1000>; + reg = <0x13050000 0x130>; /* tbc */ interrupt-parent = <&intc>; interrupts = <31>; diff --git a/arch/mips/boot/dts/ingenic/jz4740.dtsi b/arch/mips/boot/dts/ingenic/jz4740.dtsi index c1afdfdaa8..7f76cba03a 100644 --- a/arch/mips/boot/dts/ingenic/jz4740.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4740.dtsi @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include +#include #include / { @@ -323,7 +323,7 @@ udc: usb@13040000 { lcd: lcd-controller@13050000 { compatible = "ingenic,jz4740-lcd"; - reg = <0x13050000 0x1000>; + reg = <0x13050000 0x60>; /* LCDCMD1+4 */ interrupt-parent = <&intc>; interrupts = <30>; diff --git a/arch/mips/boot/dts/ingenic/jz4770.dtsi b/arch/mips/boot/dts/ingenic/jz4770.dtsi index 05c00b9308..bda0a3a86e 100644 --- a/arch/mips/boot/dts/ingenic/jz4770.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4770.dtsi @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include +#include #include / { @@ -399,7 +399,7 @@ gpu: gpu@13040000 { lcd: lcd-controller@13050000 { compatible = "ingenic,jz4770-lcd"; - reg = <0x13050000 0x300>; + reg = <0x13050000 0x130>; /* tbc */ interrupt-parent = <&intc>; interrupts = <31>; diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi index 9e34f433b9..3f9ea47a10 100644 --- a/arch/mips/boot/dts/ingenic/jz4780.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include +#include #include #include @@ -255,22 +255,23 @@ gpf: gpio@5 { }; }; - spi_gpio { - compatible = "spi-gpio"; + spi0: spi@10043000 { + compatible = "ingenic,jz4780-spi"; + reg = <0x10043000 0x1c>; #address-cells = <1>; #size-cells = <0>; - num-chipselects = <2>; - gpio-miso = <&gpe 14 0>; - gpio-sck = <&gpe 15 0>; - gpio-mosi = <&gpe 17 0>; - cs-gpios = <&gpe 16 0>, <&gpe 18 0>; + interrupt-parent = <&intc>; + interrupts = <8>; - spidev@0 { - compatible = "spidev"; - reg = <0>; - spi-max-frequency = <1000000>; - }; + clocks = <&cgu JZ4780_CLK_SSI0>; + clock-names = "spi"; + + dmas = <&dma JZ4780_DMA_SSI0_RX 0xffffffff>, + <&dma JZ4780_DMA_SSI0_TX 0xffffffff>; + dma-names = "rx", "tx"; + + status = "disabled"; }; uart0: serial@10030000 { @@ -338,6 +339,25 @@ uart4: serial@10034000 { status = "disabled"; }; + spi1: spi@10044000 { + compatible = "ingenic,jz4780-spi"; + reg = <0x10044000 0x1c>; + #address-cells = <1>; + #size-sells = <0>; + + interrupt-parent = <&intc>; + interrupts = <7>; + + clocks = <&cgu JZ4780_CLK_SSI1>; + clock-names = "spi"; + + dmas = <&dma JZ4780_DMA_SSI1_RX 0xffffffff>, + <&dma JZ4780_DMA_SSI1_TX 0xffffffff>; + dma-names = "rx", "tx"; + + status = "disabled"; + }; + i2c0: i2c@10050000 { compatible = "ingenic,jz4780-i2c", "ingenic,jz4770-i2c"; #address-cells = <1>; @@ -424,6 +444,46 @@ i2c4: i2c@10054000 { status = "disabled"; }; + hdmi: hdmi@10180000 { + compatible = "ingenic,jz4780-dw-hdmi"; + reg = <0x10180000 0x8000>; + reg-io-width = <4>; + + clocks = <&cgu JZ4780_CLK_AHB0>, <&cgu JZ4780_CLK_HDMI>; + clock-names = "iahb", "isfr"; + + interrupt-parent = <&intc>; + interrupts = <3>; + + status = "disabled"; + }; + + lcdc0: lcdc0@13050000 { + compatible = "ingenic,jz4780-lcd"; + reg = <0x13050000 0x1800>; + + clocks = <&cgu JZ4780_CLK_TVE>, <&cgu JZ4780_CLK_LCD0PIXCLK>; + clock-names = "lcd", "lcd_pclk"; + + interrupt-parent = <&intc>; + interrupts = <31>; + + status = "disabled"; + }; + + lcdc1: lcdc1@130a0000 { + compatible = "ingenic,jz4780-lcd"; + reg = <0x130a0000 0x1800>; + + clocks = <&cgu JZ4780_CLK_TVE>, <&cgu JZ4780_CLK_LCD1PIXCLK>; + clock-names = "lcd", "lcd_pclk"; + + interrupt-parent = <&intc>; + interrupts = <23>; + + status = "disabled"; + }; + nemc: nemc@13410000 { compatible = "ingenic,jz4780-nemc", "simple-mfd"; reg = <0x13410000 0x10000>; diff --git a/arch/mips/boot/dts/ingenic/x1000.dtsi b/arch/mips/boot/dts/ingenic/x1000.dtsi index dec7909d4b..8bd27edef2 100644 --- a/arch/mips/boot/dts/ingenic/x1000.dtsi +++ b/arch/mips/boot/dts/ingenic/x1000.dtsi @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include -#include +#include #include / { diff --git a/arch/mips/boot/dts/ingenic/x1830.dtsi b/arch/mips/boot/dts/ingenic/x1830.dtsi index 215257f8bb..2595df8671 100644 --- a/arch/mips/boot/dts/ingenic/x1830.dtsi +++ b/arch/mips/boot/dts/ingenic/x1830.dtsi @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include -#include +#include #include / { diff --git a/arch/mips/boot/dts/loongson/loongson64-2k1000.dtsi b/arch/mips/boot/dts/loongson/loongson64-2k1000.dtsi index bfc3d3243e..8143a61111 100644 --- a/arch/mips/boot/dts/loongson/loongson64-2k1000.dtsi +++ b/arch/mips/boot/dts/loongson/loongson64-2k1000.dtsi @@ -52,6 +52,11 @@ package0: bus@10000000 { 0 0x40000000 0 0x40000000 0 0x40000000 0xfe 0x00000000 0xfe 0x00000000 0 0x40000000>; + pm: reset-controller@1fe07000 { + compatible = "loongson,ls2k-pm"; + reg = <0 0x1fe07000 0 0x422>; + }; + liointc0: interrupt-controller@1fe11400 { compatible = "loongson,liointc-2.0"; reg = <0 0x1fe11400 0 0x40>, diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c index 30f1d75208..37a07b3c45 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -71,20 +72,6 @@ static void octeon_sha1_transform(const void *_block) octeon_sha1_start(block[7]); } -static int octeon_sha1_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA1_H0; - sctx->state[1] = SHA1_H1; - sctx->state[2] = SHA1_H2; - sctx->state[3] = SHA1_H3; - sctx->state[4] = SHA1_H4; - sctx->count = 0; - - return 0; -} - static void __octeon_sha1_update(struct sha1_state *sctx, const u8 *data, unsigned int len) { @@ -200,7 +187,7 @@ static int octeon_sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg octeon_sha1_alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = octeon_sha1_init, + .init = sha1_base_init, .update = octeon_sha1_update, .final = octeon_sha1_final, .export = octeon_sha1_export, diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index 36cb92895d..435e4a6e7f 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -63,40 +64,6 @@ static void octeon_sha256_transform(const void *_block) octeon_sha256_start(block[7]); } -static int octeon_sha224_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - - return 0; -} - -static int octeon_sha256_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; -} - static void __octeon_sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { @@ -224,7 +191,7 @@ static int octeon_sha256_import(struct shash_desc *desc, const void *in) static struct shash_alg octeon_sha256_algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, - .init = octeon_sha256_init, + .init = sha256_base_init, .update = octeon_sha256_update, .final = octeon_sha256_final, .export = octeon_sha256_export, @@ -240,7 +207,7 @@ static struct shash_alg octeon_sha256_algs[2] = { { } }, { .digestsize = SHA224_DIGEST_SIZE, - .init = octeon_sha224_init, + .init = sha224_base_init, .update = octeon_sha256_update, .final = octeon_sha224_final, .descsize = sizeof(struct sha256_state), diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c index 359f039820..2dee9354e3 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -74,40 +75,6 @@ static void octeon_sha512_transform(const void *_block) octeon_sha512_start(block[15]); } -static int octeon_sha512_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA512_H0; - sctx->state[1] = SHA512_H1; - sctx->state[2] = SHA512_H2; - sctx->state[3] = SHA512_H3; - sctx->state[4] = SHA512_H4; - sctx->state[5] = SHA512_H5; - sctx->state[6] = SHA512_H6; - sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static int octeon_sha384_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA384_H0; - sctx->state[1] = SHA384_H1; - sctx->state[2] = SHA384_H2; - sctx->state[3] = SHA384_H3; - sctx->state[4] = SHA384_H4; - sctx->state[5] = SHA384_H5; - sctx->state[6] = SHA384_H6; - sctx->state[7] = SHA384_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - static void __octeon_sha512_update(struct sha512_state *sctx, const u8 *data, unsigned int len) { @@ -223,7 +190,7 @@ static int octeon_sha384_final(struct shash_desc *desc, u8 *hash) static struct shash_alg octeon_sha512_algs[2] = { { .digestsize = SHA512_DIGEST_SIZE, - .init = octeon_sha512_init, + .init = sha512_base_init, .update = octeon_sha512_update, .final = octeon_sha512_final, .descsize = sizeof(struct sha512_state), @@ -236,7 +203,7 @@ static struct shash_alg octeon_sha512_algs[2] = { { } }, { .digestsize = SHA384_DIGEST_SIZE, - .init = octeon_sha384_init, + .init = sha384_base_init, .update = octeon_sha512_update, .final = octeon_sha384_final, .descsize = sizeof(struct sha512_state), diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper.c b/arch/mips/cavium-octeon/executive/cvmx-helper.c index 6044ff4710..b22f664e2d 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper.c @@ -1055,16 +1055,6 @@ int cvmx_helper_initialize_packet_io_global(void) } EXPORT_SYMBOL_GPL(cvmx_helper_initialize_packet_io_global); -/** - * Does core local initialization for packet io - * - * Returns Zero on success, non-zero on failure - */ -int cvmx_helper_initialize_packet_io_local(void) -{ - return cvmx_pko_initialize_local(); -} - /** * Return the link state of an IPD/PKO port as returned by * auto negotiation. The result of this function may not match diff --git a/arch/mips/cavium-octeon/executive/cvmx-pko.c b/arch/mips/cavium-octeon/executive/cvmx-pko.c index 7c4879e743..ae8806e7bc 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-pko.c +++ b/arch/mips/cavium-octeon/executive/cvmx-pko.c @@ -229,20 +229,6 @@ void cvmx_pko_initialize_global(void) } } -/* - * This function does per-core initialization required by the PKO routines. - * This must be called on all cores that will do packet output, and must - * be called after the FPA has been initialized and filled with pages. - * - * Returns 0 on success - * !0 on failure - */ -int cvmx_pko_initialize_local(void) -{ - /* Nothing to do */ - return 0; -} - /* * Enables the packet output hardware. It must already be * configured. diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c index be5d4afcd3..844f882096 100644 --- a/arch/mips/cavium-octeon/octeon-irq.c +++ b/arch/mips/cavium-octeon/octeon-irq.c @@ -2609,7 +2609,10 @@ static void octeon_irq_ciu3_ip2(void) else hw = intsn; - ret = handle_domain_irq(domain, hw, NULL); + irq_enter(); + ret = generic_handle_domain_irq(domain, hw); + irq_exit(); + if (ret < 0) { union cvmx_ciu3_iscx_w1c isc_w1c; u64 isc_w1c_addr = ciu3_addr + CIU3_ISC_W1C(intsn); diff --git a/arch/mips/configs/bmips_stb_defconfig b/arch/mips/configs/bmips_stb_defconfig index 625bd2d7e6..5956fb95c1 100644 --- a/arch/mips/configs/bmips_stb_defconfig +++ b/arch/mips/configs/bmips_stb_defconfig @@ -1,6 +1,7 @@ # CONFIG_LOCALVERSION_AUTO is not set # CONFIG_SWAP is not set CONFIG_NO_HZ=y +CONFIG_HZ=1000 CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_VM_EVENT_COUNTERS is not set @@ -8,17 +9,34 @@ CONFIG_EXPERT=y CONFIG_BMIPS_GENERIC=y CONFIG_CPU_LITTLE_ENDIAN=y CONFIG_HIGHMEM=y +CONFIG_HIGH_RES_TIMERS=y CONFIG_SMP=y CONFIG_NR_CPUS=4 +CONFIG_CC_STACKPROTECTOR_STRONG=y # CONFIG_SECCOMP is not set CONFIG_MIPS_O32_FP64_SUPPORT=y +# CONFIG_RD_GZIP is not set +# CONFIG_RD_BZIP2 is not set +# CONFIG_RD_LZMA is not set +CONFIG_RD_XZ=y +# CONFIG_RD_LZO is not set +# CONFIG_RD_LZ4 is not set +# CONFIG_IOSCHED_DEADLINE is not set +# CONFIG_IOSCHED_CFQ is not set +CONFIG_PCI=y +CONFIG_PCI_MSI=y +CONFIG_PCIEASPM_POWERSAVE=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCIE_BRCMSTB=y CONFIG_CPU_FREQ=y CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_STAT_DETAILS=y +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y +CONFIG_CPU_FREQ_GOV_PERFORMANCE=y CONFIG_CPU_FREQ_GOV_POWERSAVE=y CONFIG_CPU_FREQ_GOV_USERSPACE=y CONFIG_CPU_FREQ_GOV_ONDEMAND=y CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y -CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y CONFIG_BMIPS_CPUFREQ=y # CONFIG_BLK_DEV_BSG is not set CONFIG_NET=y @@ -32,32 +50,99 @@ CONFIG_INET=y # CONFIG_INET_DIAG is not set CONFIG_CFG80211=y CONFIG_NL80211_TESTMODE=y +CONFIG_WIRELESS=y CONFIG_MAC80211=y +CONFIG_NL80211=y CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_STANDALONE is not set # CONFIG_PREVENT_FIRMWARE_BUILD is not set +CONFIG_BRCMSTB_GISB_ARB=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_IP_MROUTE=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +# CONFIG_INET_XFRM_MODE_TRANSPORT is not set +# CONFIG_INET_XFRM_MODE_TUNNEL is not set +# CONFIG_INET_XFRM_MODE_BEET is not set +# CONFIG_INET_LRO is not set +CONFIG_INET_UDP_DIAG=y +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_BIC=y +# CONFIG_TCP_CONG_WESTWOOD is not set +# CONFIG_TCP_CONG_HTCP is not set +# CONFIG_IPV6 is not set +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_FILTER=y +CONFIG_NETFILTER=y +CONFIG_NETFILTER_XTABLES=y +CONFIG_BRIDGE=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_NET_DSA=y +CONFIG_NET_SWITCHDEV=y +CONFIG_DMA_CMA=y +CONFIG_CMA_ALIGNMENT=12 +CONFIG_SPI=y +CONFIG_SPI_BRCMSTB=y CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y CONFIG_MTD_CFI=y +CONFIG_MTD_JEDECPROBE=y CONFIG_MTD_CFI_INTELEXT=y CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_CFI_STAA=y +CONFIG_MTD_ROM=y +CONFIG_MTD_ABSENT=y +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_M25P80=y +CONFIG_MTD_NAND=y +CONFIG_MTD_NAND_BRCMNAND=y +CONFIG_MTD_SPI_NOR=y +# CONFIG_MTD_SPI_NOR_USE_4K_SECTORS is not set +CONFIG_MTD_UBI=y +CONFIG_MTD_UBI_GLUEBI=y +CONFIG_PROC_DEVICETREE=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=8192 # CONFIG_BLK_DEV is not set CONFIG_SCSI=y CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_SG=y +CONFIG_SCSI_MULTI_LUN=y # CONFIG_SCSI_LOWLEVEL is not set CONFIG_NETDEVICES=y +CONFIG_VLAN_8021Q=y +CONFIG_MACVLAN=y CONFIG_BCMGENET=y CONFIG_USB_USBNET=y -# CONFIG_INPUT is not set +CONFIG_INPUT_EVDEV=y +# CONFIG_INPUT_KEYBOARD is not set +# CONFIG_INPUT_MOUSE is not set +CONFIG_INPUT_MISC=y +CONFIG_INPUT_UINPUT=y # CONFIG_SERIO is not set -# CONFIG_VT is not set +CONFIG_VT=y +CONFIG_VT_HW_CONSOLE_BINDING=y +# CONFIG_DEVKMEM is not set CONFIG_SERIAL_8250=y # CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y # CONFIG_HW_RANDOM is not set CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_BRCMSTB=y CONFIG_POWER_RESET_SYSCON=y CONFIG_POWER_SUPPLY=y # CONFIG_HWMON is not set @@ -69,22 +154,76 @@ CONFIG_USB_OHCI_HCD=y CONFIG_USB_OHCI_HCD_PLATFORM=y CONFIG_USB_STORAGE=y CONFIG_SOC_BRCMSTB=y +CONFIG_MMC=y +CONFIG_MMC_BLOCK_MINORS=16 +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_PLTFM=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y # CONFIG_DNOTIFY is not set -CONFIG_FUSE_FS=y -CONFIG_VFAT_FS=y CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_NFS_FS=y CONFIG_CIFS=y +CONFIG_JBD2_DEBUG=y +CONFIG_FUSE_FS=y +CONFIG_FHANDLE=y +CONFIG_CGROUPS=y +CONFIG_CUSE=y +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_TMPFS=y +CONFIG_JFFS2_FS=y +CONFIG_UBIFS_FS=y +CONFIG_SQUASHFS=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XZ=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_ROOT_NFS=y CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=y -# CONFIG_CRYPTO_HW is not set CONFIG_PRINTK_TIME=y +CONFIG_DYNAMIC_DEBUG=y +# CONFIG_DEBUG_INFO is not set +# CONFIG_DEBUG_INFO_REDUCED is not set CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y +CONFIG_LOCKUP_DETECTOR=y +CONFIG_DEBUG_USER=y CONFIG_CMDLINE_BOOL=y CONFIG_CMDLINE="earlycon" +# CONFIG_MIPS_CMDLINE_FROM_DTB is not set +CONFIG_MIPS_CMDLINE_DTB_EXTEND=y +# CONFIG_MIPS_CMDLINE_FROM_BOOTLOADER is not set +# CONFIG_CRYPTO_HW is not set +CONFIG_DT_BCM974XX=y +CONFIG_FW_CFE=y +CONFIG_ATA=y +CONFIG_SATA_AHCI_PLATFORM=y +CONFIG_AHCI_BRCMSTB=y +CONFIG_GENERIC_PHY=y +CONFIG_GPIOLIB=y +CONFIG_GPIO_SYSFS=y +CONFIG_PHY_BRCM_USB=y +CONFIG_PHY_BRCM_SATA=y +CONFIG_PM_RUNTIME=y +CONFIG_PM_DEBUG=y +CONFIG_SYSVIPC=y +CONFIG_FUNCTION_GRAPH_TRACER=y +CONFIG_DYNAMIC_FTRACE=y +CONFIG_FUNCTION_TRACER=y +CONFIG_FUNCTION_PROFILER=y +CONFIG_IRQSOFF_TRACER=y +CONFIG_SCHED_TRACER=y +CONFIG_BLK_DEV_IO_TRACE=y +CONFIG_FTRACE_SYSCALLS=y +CONFIG_TRACER_SNAPSHOT=y +CONFIG_TRACER_SNAPSHOT_PER_CPU_SWAP=y +CONFIG_STACK_TRACER=y diff --git a/arch/mips/configs/ci20_defconfig b/arch/mips/configs/ci20_defconfig index ab7ebb0668..cc69b21585 100644 --- a/arch/mips/configs/ci20_defconfig +++ b/arch/mips/configs/ci20_defconfig @@ -98,7 +98,13 @@ CONFIG_RC_DEVICES=y CONFIG_IR_GPIO_CIR=m CONFIG_IR_GPIO_TX=m CONFIG_MEDIA_SUPPORT=m +CONFIG_DRM=m +CONFIG_DRM_INGENIC=m +CONFIG_DRM_INGENIC_DW_HDMI=m +CONFIG_DRM_DISPLAY_CONNECTOR=m # CONFIG_VGA_CONSOLE is not set +CONFIG_FB=y +CONFIG_FRAMEBUFFER_CONSOLE=y # CONFIG_HID is not set CONFIG_USB=y CONFIG_USB_STORAGE=y diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig index 85f1955b4b..e2ed105f8c 100644 --- a/arch/mips/configs/decstation_64_defconfig +++ b/arch/mips/configs/decstation_64_defconfig @@ -144,7 +144,6 @@ CONFIG_EXT2_FS_SECURITY=y CONFIG_EXT3_FS=y CONFIG_EXT3_FS_POSIX_ACL=y CONFIG_EXT3_FS_SECURITY=y -# CONFIG_MANDATORY_FILE_LOCKING is not set CONFIG_ISO9660_FS=y CONFIG_JOLIET=y CONFIG_PROC_KCORE=y diff --git a/arch/mips/configs/decstation_defconfig b/arch/mips/configs/decstation_defconfig index 30a6eafdb1..7e987d6f5e 100644 --- a/arch/mips/configs/decstation_defconfig +++ b/arch/mips/configs/decstation_defconfig @@ -140,7 +140,6 @@ CONFIG_EXT2_FS_SECURITY=y CONFIG_EXT3_FS=y CONFIG_EXT3_FS_POSIX_ACL=y CONFIG_EXT3_FS_SECURITY=y -# CONFIG_MANDATORY_FILE_LOCKING is not set CONFIG_ISO9660_FS=y CONFIG_JOLIET=y CONFIG_PROC_KCORE=y diff --git a/arch/mips/configs/decstation_r4k_defconfig b/arch/mips/configs/decstation_r4k_defconfig index e2b58dbf4a..6df5f6f2ac 100644 --- a/arch/mips/configs/decstation_r4k_defconfig +++ b/arch/mips/configs/decstation_r4k_defconfig @@ -140,7 +140,6 @@ CONFIG_EXT2_FS_SECURITY=y CONFIG_EXT3_FS=y CONFIG_EXT3_FS_POSIX_ACL=y CONFIG_EXT3_FS_SECURITY=y -# CONFIG_MANDATORY_FILE_LOCKING is not set CONFIG_ISO9660_FS=y CONFIG_JOLIET=y CONFIG_PROC_KCORE=y diff --git a/arch/mips/configs/ip27_defconfig b/arch/mips/configs/ip27_defconfig index 638d7cf5ef..821630ac1b 100644 --- a/arch/mips/configs/ip27_defconfig +++ b/arch/mips/configs/ip27_defconfig @@ -223,7 +223,6 @@ CONFIG_TMD_HERMES=m CONFIG_NORTEL_HERMES=m CONFIG_P54_COMMON=m CONFIG_P54_PCI=m -CONFIG_PRISM54=m CONFIG_LIBERTAS=m CONFIG_LIBERTAS_THINFIRM=m CONFIG_MWL8K=m diff --git a/arch/mips/configs/loongson3_defconfig b/arch/mips/configs/loongson3_defconfig index f02101ff04..25ecd15bc9 100644 --- a/arch/mips/configs/loongson3_defconfig +++ b/arch/mips/configs/loongson3_defconfig @@ -282,6 +282,7 @@ CONFIG_DRM=y CONFIG_DRM_RADEON=m CONFIG_DRM_QXL=y CONFIG_DRM_VIRTIO_GPU=y +CONFIG_FB=y CONFIG_FB_RADEON=y CONFIG_LCD_CLASS_DEVICE=y CONFIG_LCD_PLATFORM=m diff --git a/arch/mips/configs/malta_defconfig b/arch/mips/configs/malta_defconfig index 9cb2cf2595..3321bb5769 100644 --- a/arch/mips/configs/malta_defconfig +++ b/arch/mips/configs/malta_defconfig @@ -302,7 +302,6 @@ CONFIG_HOSTAP_FIRMWARE=y CONFIG_HOSTAP_FIRMWARE_NVRAM=y CONFIG_HOSTAP_PLX=m CONFIG_HOSTAP_PCI=m -CONFIG_PRISM54=m CONFIG_LIBERTAS=m CONFIG_INPUT_MOUSEDEV=y CONFIG_MOUSE_PS2_ELANTECH=y diff --git a/arch/mips/configs/malta_kvm_defconfig b/arch/mips/configs/malta_kvm_defconfig index 5924e48fd3..009b303722 100644 --- a/arch/mips/configs/malta_kvm_defconfig +++ b/arch/mips/configs/malta_kvm_defconfig @@ -310,7 +310,6 @@ CONFIG_HOSTAP_FIRMWARE=y CONFIG_HOSTAP_FIRMWARE_NVRAM=y CONFIG_HOSTAP_PLX=m CONFIG_HOSTAP_PCI=m -CONFIG_PRISM54=m CONFIG_LIBERTAS=m CONFIG_INPUT_MOUSEDEV=y CONFIG_SERIAL_8250=y diff --git a/arch/mips/configs/maltaup_xpa_defconfig b/arch/mips/configs/maltaup_xpa_defconfig index c0d3156ef6..e214e13610 100644 --- a/arch/mips/configs/maltaup_xpa_defconfig +++ b/arch/mips/configs/maltaup_xpa_defconfig @@ -309,7 +309,6 @@ CONFIG_HOSTAP_FIRMWARE=y CONFIG_HOSTAP_FIRMWARE_NVRAM=y CONFIG_HOSTAP_PLX=m CONFIG_HOSTAP_PCI=m -CONFIG_PRISM54=m CONFIG_LIBERTAS=m CONFIG_INPUT_MOUSEDEV=y CONFIG_MOUSE_PS2_ELANTECH=y diff --git a/arch/mips/configs/rbtx49xx_defconfig b/arch/mips/configs/rbtx49xx_defconfig index 69f2300107..f8212a813b 100644 --- a/arch/mips/configs/rbtx49xx_defconfig +++ b/arch/mips/configs/rbtx49xx_defconfig @@ -10,9 +10,6 @@ CONFIG_EXPERT=y CONFIG_SLAB=y CONFIG_MACH_TX49XX=y CONFIG_TOSHIBA_RBTX4927=y -CONFIG_TOSHIBA_RBTX4938=y -CONFIG_TOSHIBA_RBTX4939=y -CONFIG_TOSHIBA_RBTX4938_MPLEX_KEEP=y # CONFIG_SECCOMP is not set CONFIG_PCI=y CONFIG_MODULES=y @@ -38,7 +35,6 @@ CONFIG_MTD_JEDECPROBE=y CONFIG_MTD_CFI_AMDSTD=y CONFIG_MTD_COMPLEX_MAPPINGS=y CONFIG_MTD_PHYSMAP=y -CONFIG_MTD_RBTX4939=y CONFIG_MTD_RAW_NAND=m CONFIG_MTD_NAND_TXX9NDFMC=m CONFIG_BLK_DEV_LOOP=y diff --git a/arch/mips/dec/setup.c b/arch/mips/dec/setup.c index eaad0ed4b5..a8a30bb1de 100644 --- a/arch/mips/dec/setup.c +++ b/arch/mips/dec/setup.c @@ -117,21 +117,21 @@ static void __init dec_be_init(void) { switch (mips_machtype) { case MACH_DS23100: /* DS2100/DS3100 Pmin/Pmax */ - board_be_handler = dec_kn01_be_handler; + mips_set_be_handler(dec_kn01_be_handler); busirq_handler = dec_kn01_be_interrupt; busirq_flags |= IRQF_SHARED; dec_kn01_be_init(); break; case MACH_DS5000_1XX: /* DS5000/1xx 3min */ case MACH_DS5000_XX: /* DS5000/xx Maxine */ - board_be_handler = dec_kn02xa_be_handler; + mips_set_be_handler(dec_kn02xa_be_handler); busirq_handler = dec_kn02xa_be_interrupt; dec_kn02xa_be_init(); break; case MACH_DS5000_200: /* DS5000/200 3max */ case MACH_DS5000_2X0: /* DS5000/240 3max+ */ case MACH_DS5900: /* DS5900 bigmax */ - board_be_handler = dec_ecc_be_handler; + mips_set_be_handler(dec_ecc_be_handler); busirq_handler = dec_ecc_be_interrupt; dec_ecc_be_init(); break; diff --git a/arch/mips/generic/Platform b/arch/mips/generic/Platform index e1abc113b4..0c03623f38 100644 --- a/arch/mips/generic/Platform +++ b/arch/mips/generic/Platform @@ -13,8 +13,7 @@ cflags-$(CONFIG_MACH_INGENIC_SOC) += -I$(srctree)/arch/mips/include/asm/mach-ing cflags-$(CONFIG_MIPS_GENERIC) += -I$(srctree)/arch/mips/include/asm/mach-generic load-$(CONFIG_MIPS_GENERIC) += 0xffffffff80100000 -zload-$(CONFIG_MIPS_GENERIC) += 0xffffffff81000000 -all-$(CONFIG_MIPS_GENERIC) := vmlinux.gz.itb +all-$(CONFIG_MIPS_GENERIC) += vmlinux.gz.itb its-y := vmlinux.its.S its-$(CONFIG_FIT_IMAGE_FDT_BOSTON) += board-boston.its.S diff --git a/arch/mips/generic/init.c b/arch/mips/generic/init.c index 1842cddd83..1d712eac16 100644 --- a/arch/mips/generic/init.c +++ b/arch/mips/generic/init.c @@ -110,14 +110,15 @@ void __init plat_mem_setup(void) void __init device_tree_init(void) { - int err; - unflatten_and_copy_device_tree(); mips_cpc_probe(); - err = register_cps_smp_ops(); - if (err) - err = register_up_smp_ops(); + if (!register_cps_smp_ops()) + return; + if (!register_vsmp_smp_ops()) + return; + + register_up_smp_ops(); } int __init apply_mips_fdt_fixups(void *fdt_out, size_t fdt_out_size, diff --git a/arch/mips/include/asm/asm.h b/arch/mips/include/asm/asm.h index cc69f1deb1..336ac9b652 100644 --- a/arch/mips/include/asm/asm.h +++ b/arch/mips/include/asm/asm.h @@ -19,6 +19,7 @@ #include #include +#include #ifndef __VDSO__ /* @@ -211,6 +212,8 @@ symbol = value #define LONG_SUB sub #define LONG_SUBU subu #define LONG_L lw +#define LONG_LL ll +#define LONG_SC sc #define LONG_S sw #define LONG_SP swp #define LONG_SLL sll @@ -219,6 +222,8 @@ symbol = value #define LONG_SRLV srlv #define LONG_SRA sra #define LONG_SRAV srav +#define LONG_INS ins +#define LONG_EXT ext #ifdef __ASSEMBLY__ #define LONG .word @@ -236,6 +241,8 @@ symbol = value #define LONG_SUB dsub #define LONG_SUBU dsubu #define LONG_L ld +#define LONG_LL lld +#define LONG_SC scd #define LONG_S sd #define LONG_SP sdp #define LONG_SLL dsll @@ -244,6 +251,8 @@ symbol = value #define LONG_SRLV dsrlv #define LONG_SRA dsra #define LONG_SRAV dsrav +#define LONG_INS dins +#define LONG_EXT dext #ifdef __ASSEMBLY__ #define LONG .dword @@ -320,6 +329,19 @@ symbol = value #define SSNOP sll zero, zero, 1 +/* + * Using a branch-likely instruction to check the result of an sc instruction + * works around a bug present in R10000 CPUs prior to revision 3.0 that could + * cause ll-sc sequences to execute non-atomically. + */ +#ifdef CONFIG_WAR_R10000_LLSC +# define SC_BEQZ beqzl +#elif MIPS_ISA_REV >= 6 +# define SC_BEQZ beqzc +#else +# define SC_BEQZ beqz +#endif + #ifdef CONFIG_SGI_IP28 /* Inhibit speculative stores to volatile (e.g.DMA) or invalid addresses. */ #include diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h index a0b9e7c1e4..712fb5a6a5 100644 --- a/arch/mips/include/asm/atomic.h +++ b/arch/mips/include/asm/atomic.h @@ -16,13 +16,12 @@ #include #include +#include #include #include #include #include -#include #include -#include #define ATOMIC_OPS(pfx, type) \ static __always_inline type arch_##pfx##_read(const pfx##_t *v) \ @@ -74,7 +73,7 @@ static __inline__ void arch_##pfx##_##op(type i, pfx##_t * v) \ "1: " #ll " %0, %1 # " #pfx "_" #op " \n" \ " " #asm_op " %0, %2 \n" \ " " #sc " %0, %1 \n" \ - "\t" __SC_BEQZ "%0, 1b \n" \ + "\t" __stringify(SC_BEQZ) " %0, 1b \n" \ " .set pop \n" \ : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ : "Ir" (i) : __LLSC_CLOBBER); \ @@ -104,7 +103,7 @@ arch_##pfx##_##op##_return_relaxed(type i, pfx##_t * v) \ "1: " #ll " %1, %2 # " #pfx "_" #op "_return\n" \ " " #asm_op " %0, %1, %3 \n" \ " " #sc " %0, %2 \n" \ - "\t" __SC_BEQZ "%0, 1b \n" \ + "\t" __stringify(SC_BEQZ) " %0, 1b \n" \ " " #asm_op " %0, %1, %3 \n" \ " .set pop \n" \ : "=&r" (result), "=&r" (temp), \ @@ -137,7 +136,7 @@ arch_##pfx##_fetch_##op##_relaxed(type i, pfx##_t * v) \ "1: " #ll " %1, %2 # " #pfx "_fetch_" #op "\n" \ " " #asm_op " %0, %1, %3 \n" \ " " #sc " %0, %2 \n" \ - "\t" __SC_BEQZ "%0, 1b \n" \ + "\t" __stringify(SC_BEQZ) " %0, 1b \n" \ " .set pop \n" \ " move %0, %1 \n" \ : "=&r" (result), "=&r" (temp), \ @@ -237,7 +236,7 @@ static __inline__ type arch_##pfx##_sub_if_positive(type i, pfx##_t * v) \ " .set push \n" \ " .set " MIPS_ISA_LEVEL " \n" \ " " #sc " %1, %2 \n" \ - " " __SC_BEQZ "%1, 1b \n" \ + " " __stringify(SC_BEQZ) " %1, 1b \n" \ "2: " __SYNC(full, loongson3_war) " \n" \ " .set pop \n" \ : "=&r" (result), "=&r" (temp), \ diff --git a/arch/mips/include/asm/bitops.h b/arch/mips/include/asm/bitops.h index dc2a6234dd..b4bf754f7d 100644 --- a/arch/mips/include/asm/bitops.h +++ b/arch/mips/include/asm/bitops.h @@ -16,14 +16,12 @@ #include #include #include +#include #include #include /* sigh ... */ #include #include -#include -#include #include -#include #define __bit_op(mem, insn, inputs...) do { \ unsigned long __temp; \ @@ -32,10 +30,10 @@ " .set push \n" \ " .set " MIPS_ISA_LEVEL " \n" \ " " __SYNC(full, loongson3_war) " \n" \ - "1: " __LL "%0, %1 \n" \ + "1: " __stringify(LONG_LL) " %0, %1 \n" \ " " insn " \n" \ - " " __SC "%0, %1 \n" \ - " " __SC_BEQZ "%0, 1b \n" \ + " " __stringify(LONG_SC) " %0, %1 \n" \ + " " __stringify(SC_BEQZ) " %0, 1b \n" \ " .set pop \n" \ : "=&r"(__temp), "+" GCC_OFF_SMALL_ASM()(mem) \ : inputs \ @@ -49,10 +47,10 @@ " .set push \n" \ " .set " MIPS_ISA_LEVEL " \n" \ " " __SYNC(full, loongson3_war) " \n" \ - "1: " __LL ll_dst ", %2 \n" \ + "1: " __stringify(LONG_LL) " " ll_dst ", %2\n" \ " " insn " \n" \ - " " __SC "%1, %2 \n" \ - " " __SC_BEQZ "%1, 1b \n" \ + " " __stringify(LONG_SC) " %1, %2 \n" \ + " " __stringify(SC_BEQZ) " %1, 1b \n" \ " .set pop \n" \ : "=&r"(__orig), "=&r"(__temp), \ "+" GCC_OFF_SMALL_ASM()(mem) \ @@ -98,7 +96,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr) } if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(bit) && (bit >= 16)) { - __bit_op(*m, __INS "%0, %3, %2, 1", "i"(bit), "r"(~0)); + __bit_op(*m, __stringify(LONG_INS) " %0, %3, %2, 1", "i"(bit), "r"(~0)); return; } @@ -126,7 +124,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) } if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(bit)) { - __bit_op(*m, __INS "%0, $0, %2, 1", "i"(bit)); + __bit_op(*m, __stringify(LONG_INS) " %0, $0, %2, 1", "i"(bit)); return; } @@ -234,8 +232,8 @@ static inline int test_and_clear_bit(unsigned long nr, res = __mips_test_and_clear_bit(nr, addr); } else if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(nr)) { res = __test_bit_op(*m, "%1", - __EXT "%0, %1, %3, 1;" - __INS "%1, $0, %3, 1", + __stringify(LONG_EXT) " %0, %1, %3, 1;" + __stringify(LONG_INS) " %1, $0, %3, 1", "i"(bit)); } else { orig = __test_bit_op(*m, "%0", @@ -446,7 +444,6 @@ static inline int ffs(int word) } #include -#include #ifdef __KERNEL__ diff --git a/arch/mips/include/asm/cmpxchg.h b/arch/mips/include/asm/cmpxchg.h index 66a8b293fd..7ec9493b28 100644 --- a/arch/mips/include/asm/cmpxchg.h +++ b/arch/mips/include/asm/cmpxchg.h @@ -10,10 +10,9 @@ #include #include +#include #include -#include #include -#include /* * These functions doesn't exist, so if they are called you'll either: @@ -48,7 +47,7 @@ extern unsigned long __xchg_called_with_bad_pointer(void) " move $1, %z3 \n" \ " .set " MIPS_ISA_ARCH_LEVEL " \n" \ " " st " $1, %1 \n" \ - "\t" __SC_BEQZ "$1, 1b \n" \ + "\t" __stringify(SC_BEQZ) " $1, 1b \n" \ " .set pop \n" \ : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) \ @@ -127,7 +126,7 @@ unsigned long __xchg(volatile void *ptr, unsigned long x, int size) " move $1, %z4 \n" \ " .set "MIPS_ISA_ARCH_LEVEL" \n" \ " " st " $1, %1 \n" \ - "\t" __SC_BEQZ "$1, 1b \n" \ + "\t" __stringify(SC_BEQZ) " $1, 1b \n" \ " .set pop \n" \ "2: " __SYNC(full, loongson3_war) " \n" \ : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ @@ -282,7 +281,7 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, /* Attempt to store new at ptr */ " scd %L1, %2 \n" /* If we failed, loop! */ - "\t" __SC_BEQZ "%L1, 1b \n" + "\t" __stringify(SC_BEQZ) " %L1, 1b \n" "2: " __SYNC(full, loongson3_war) " \n" " .set pop \n" : "=&r"(ret), diff --git a/arch/mips/include/asm/cop2.h b/arch/mips/include/asm/cop2.h index 6b7396a6a1..01b05be23a 100644 --- a/arch/mips/include/asm/cop2.h +++ b/arch/mips/include/asm/cop2.h @@ -22,17 +22,6 @@ extern void octeon_cop2_restore(struct octeon_cop2_state *); #define cop2_present 1 #define cop2_lazy_restore 1 -#elif defined(CONFIG_CPU_XLP) - -extern void nlm_cop2_save(struct nlm_cop2_state *); -extern void nlm_cop2_restore(struct nlm_cop2_state *); - -#define cop2_save(r) nlm_cop2_save(&(r)->thread.cp2) -#define cop2_restore(r) nlm_cop2_restore(&(r)->thread.cp2) - -#define cop2_present 1 -#define cop2_lazy_restore 0 - #elif defined(CONFIG_CPU_LOONGSON64) #define cop2_present 1 diff --git a/arch/mips/include/asm/cpu-type.h b/arch/mips/include/asm/cpu-type.h index 2be5d7b5de..5efe8c8b85 100644 --- a/arch/mips/include/asm/cpu-type.h +++ b/arch/mips/include/asm/cpu-type.h @@ -195,14 +195,6 @@ static inline int __pure __get_cpu_type(const int cpu_type) #ifdef CONFIG_SYS_HAS_CPU_BMIPS5000 case CPU_BMIPS5000: #endif - -#ifdef CONFIG_SYS_HAS_CPU_XLP - case CPU_XLP: -#endif - -#ifdef CONFIG_SYS_HAS_CPU_XLR - case CPU_XLR: -#endif break; default: unreachable(); diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h index d45a52f65b..5c2f8d9cb7 100644 --- a/arch/mips/include/asm/cpu.h +++ b/arch/mips/include/asm/cpu.h @@ -328,7 +328,7 @@ enum cpu_type_enum { */ CPU_5KC, CPU_5KE, CPU_20KC, CPU_25KF, CPU_SB1, CPU_SB1A, CPU_LOONGSON2EF, CPU_LOONGSON64, CPU_CAVIUM_OCTEON, CPU_CAVIUM_OCTEON_PLUS, - CPU_CAVIUM_OCTEON2, CPU_CAVIUM_OCTEON3, CPU_XLR, CPU_XLP, CPU_I6500, + CPU_CAVIUM_OCTEON2, CPU_CAVIUM_OCTEON3, CPU_I6500, CPU_QEMU_GENERIC, diff --git a/arch/mips/include/asm/futex.h b/arch/mips/include/asm/futex.h index d85248404c..8612a7e42d 100644 --- a/arch/mips/include/asm/futex.h +++ b/arch/mips/include/asm/futex.h @@ -19,7 +19,11 @@ #include #include -#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ +#define arch_futex_atomic_op_inuser arch_futex_atomic_op_inuser +#define futex_atomic_cmpxchg_inatomic futex_atomic_cmpxchg_inatomic +#include + +#define __futex_atomic_op(op, insn, ret, oldval, uaddr, oparg) \ { \ if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) { \ __asm__ __volatile__( \ @@ -80,8 +84,10 @@ : "0" (0), GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oparg), \ "i" (-EFAULT) \ : "memory"); \ - } else \ - ret = -ENOSYS; \ + } else { \ + /* fallback for non-SMP */ \ + ret = futex_atomic_op_inuser_local(op, oparg, oval, uaddr); \ + } \ } static inline int @@ -94,23 +100,23 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) switch (op) { case FUTEX_OP_SET: - __futex_atomic_op("move $1, %z5", ret, oldval, uaddr, oparg); + __futex_atomic_op(op, "move $1, %z5", ret, oldval, uaddr, oparg); break; case FUTEX_OP_ADD: - __futex_atomic_op("addu $1, %1, %z5", + __futex_atomic_op(op, "addu $1, %1, %z5", ret, oldval, uaddr, oparg); break; case FUTEX_OP_OR: - __futex_atomic_op("or $1, %1, %z5", + __futex_atomic_op(op, "or $1, %1, %z5", ret, oldval, uaddr, oparg); break; case FUTEX_OP_ANDN: - __futex_atomic_op("and $1, %1, %z5", + __futex_atomic_op(op, "and $1, %1, %z5", ret, oldval, uaddr, ~oparg); break; case FUTEX_OP_XOR: - __futex_atomic_op("xor $1, %1, %z5", + __futex_atomic_op(op, "xor $1, %1, %z5", ret, oldval, uaddr, oparg); break; default: @@ -193,8 +199,9 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, : GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval), "i" (-EFAULT) : "memory"); - } else - return -ENOSYS; + } else { + return futex_atomic_cmpxchg_inatomic_local(uval, uaddr, oldval, newval); + } *uval = val; return ret; diff --git a/arch/mips/include/asm/ginvt.h b/arch/mips/include/asm/ginvt.h index 6eb7c2b94d..87b2974ffc 100644 --- a/arch/mips/include/asm/ginvt.h +++ b/arch/mips/include/asm/ginvt.h @@ -12,11 +12,13 @@ enum ginvt_type { #ifdef TOOLCHAIN_SUPPORTS_GINV # define _ASM_SET_GINV ".set ginv\n" +# define _ASM_UNSET_GINV #else -_ASM_MACRO_1R1I(ginvt, rs, type, - _ASM_INSN_IF_MIPS(0x7c0000bd | (__rs << 21) | (\\type << 8)) - _ASM_INSN32_IF_MM(0x0000717c | (__rs << 16) | (\\type << 9))); -# define _ASM_SET_GINV +# define _ASM_SET_GINV \ + _ASM_MACRO_1R1I(ginvt, rs, type, \ + _ASM_INSN_IF_MIPS(0x7c0000bd | (__rs << 21) | (\\type << 8)) \ + _ASM_INSN32_IF_MM(0x0000717c | (__rs << 16) | (\\type << 9))) +# define _ASM_UNSET_GINV ".purgem ginvt\n" #endif static __always_inline void ginvt(unsigned long addr, enum ginvt_type type) @@ -25,6 +27,7 @@ static __always_inline void ginvt(unsigned long addr, enum ginvt_type type) ".set push\n" _ASM_SET_GINV " ginvt %0, %1\n" + _ASM_UNSET_GINV ".set pop" : /* no outputs */ : "r"(addr), "i"(type) diff --git a/arch/mips/include/asm/hazards.h b/arch/mips/include/asm/hazards.h index f855478d12..cb16be93b0 100644 --- a/arch/mips/include/asm/hazards.h +++ b/arch/mips/include/asm/hazards.h @@ -160,7 +160,7 @@ do { \ #elif defined(CONFIG_MIPS_ALCHEMY) || defined(CONFIG_CPU_CAVIUM_OCTEON) || \ defined(CONFIG_CPU_LOONGSON2EF) || defined(CONFIG_CPU_LOONGSON64) || \ - defined(CONFIG_CPU_R10000) || defined(CONFIG_CPU_R5500) || defined(CONFIG_CPU_XLR) + defined(CONFIG_CPU_R10000) || defined(CONFIG_CPU_R5500) /* * R10000 rocks - all hazards handled in hardware, so this becomes a nobrainer. diff --git a/arch/mips/include/asm/kgdb.h b/arch/mips/include/asm/kgdb.h index 4f2302267d..b4e210d633 100644 --- a/arch/mips/include/asm/kgdb.h +++ b/arch/mips/include/asm/kgdb.h @@ -18,7 +18,7 @@ #ifdef CONFIG_32BIT #define KGDB_GDB_REG_SIZE 32 #define GDB_SIZEOF_REG sizeof(u32) -#else /* CONFIG_CPU_32BIT */ +#else /* CONFIG_32BIT */ #define KGDB_GDB_REG_SIZE 64 #define GDB_SIZEOF_REG sizeof(u64) #endif diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index 696f6b0093..717716cc51 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -379,9 +380,9 @@ static inline void _kvm_atomic_set_c0_guest_reg(unsigned long *reg, __asm__ __volatile__( " .set push \n" " .set "MIPS_ISA_ARCH_LEVEL" \n" - " " __LL "%0, %1 \n" + " "__stringify(LONG_LL) " %0, %1 \n" " or %0, %2 \n" - " " __SC "%0, %1 \n" + " "__stringify(LONG_SC) " %0, %1 \n" " .set pop \n" : "=&r" (temp), "+m" (*reg) : "r" (val)); @@ -396,9 +397,9 @@ static inline void _kvm_atomic_clear_c0_guest_reg(unsigned long *reg, __asm__ __volatile__( " .set push \n" " .set "MIPS_ISA_ARCH_LEVEL" \n" - " " __LL "%0, %1 \n" + " "__stringify(LONG_LL) " %0, %1 \n" " and %0, %2 \n" - " " __SC "%0, %1 \n" + " "__stringify(LONG_SC) " %0, %1 \n" " .set pop \n" : "=&r" (temp), "+m" (*reg) : "r" (~val)); @@ -414,10 +415,10 @@ static inline void _kvm_atomic_change_c0_guest_reg(unsigned long *reg, __asm__ __volatile__( " .set push \n" " .set "MIPS_ISA_ARCH_LEVEL" \n" - " " __LL "%0, %1 \n" + " "__stringify(LONG_LL) " %0, %1 \n" " and %0, %2 \n" " or %0, %3 \n" - " " __SC "%0, %1 \n" + " "__stringify(LONG_SC) " %0, %1 \n" " .set pop \n" : "=&r" (temp), "+m" (*reg) : "r" (~change), "r" (val & change)); @@ -897,7 +898,6 @@ static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} -static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB int kvm_arch_flush_remote_tlb(struct kvm *kvm); diff --git a/arch/mips/include/asm/local.h b/arch/mips/include/asm/local.h index 3fa6340903..08366b1fd2 100644 --- a/arch/mips/include/asm/local.h +++ b/arch/mips/include/asm/local.h @@ -8,7 +8,6 @@ #include #include #include -#include typedef struct { @@ -32,34 +31,18 @@ static __inline__ long local_add_return(long i, local_t * l) { unsigned long result; - if (kernel_uses_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) { - unsigned long temp; - - __asm__ __volatile__( - " .set push \n" - " .set arch=r4000 \n" - __SYNC(full, loongson3_war) " \n" - "1:" __LL "%1, %2 # local_add_return \n" - __stringify(LONG_ADDU) " %0, %1, %3 \n" - __SC "%0, %2 \n" - " beqzl %0, 1b \n" - " addu %0, %1, %3 \n" - " .set pop \n" - : "=&r" (result), "=&r" (temp), "=m" (l->a.counter) - : "Ir" (i), "m" (l->a.counter) - : "memory"); - } else if (kernel_uses_llsc) { + if (kernel_uses_llsc) { unsigned long temp; __asm__ __volatile__( " .set push \n" " .set "MIPS_ISA_ARCH_LEVEL" \n" - __SYNC(full, loongson3_war) " \n" - "1:" __LL "%1, %2 # local_add_return \n" + __SYNC(full, loongson3_war) " \n" + "1:" __stringify(LONG_LL) " %1, %2 \n" + __stringify(LONG_ADDU) " %0, %1, %3 \n" + __stringify(LONG_SC) " %0, %2 \n" + __stringify(SC_BEQZ) " %0, 1b \n" __stringify(LONG_ADDU) " %0, %1, %3 \n" - __SC "%0, %2 \n" - " beqz %0, 1b \n" - " addu %0, %1, %3 \n" " .set pop \n" : "=&r" (result), "=&r" (temp), "=m" (l->a.counter) : "Ir" (i), "m" (l->a.counter) @@ -81,34 +64,19 @@ static __inline__ long local_sub_return(long i, local_t * l) { unsigned long result; - if (kernel_uses_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) { - unsigned long temp; - - __asm__ __volatile__( - " .set push \n" - " .set arch=r4000 \n" - __SYNC(full, loongson3_war) " \n" - "1:" __LL "%1, %2 # local_sub_return \n" - __stringify(LONG_SUBU) " %0, %1, %3 \n" - __SC "%0, %2 \n" - " beqzl %0, 1b \n" - " subu %0, %1, %3 \n" - " .set pop \n" - : "=&r" (result), "=&r" (temp), "=m" (l->a.counter) - : "Ir" (i), "m" (l->a.counter) - : "memory"); - } else if (kernel_uses_llsc) { + if (kernel_uses_llsc) { unsigned long temp; __asm__ __volatile__( " .set push \n" " .set "MIPS_ISA_ARCH_LEVEL" \n" - __SYNC(full, loongson3_war) " \n" - "1:" __LL "%1, %2 # local_sub_return \n" + __SYNC(full, loongson3_war) " \n" + "1:" __stringify(LONG_LL) " %1, %2 \n" + __stringify(LONG_SUBU) " %0, %1, %3 \n" + __stringify(LONG_SUBU) " %0, %1, %3 \n" + __stringify(LONG_SC) " %0, %2 \n" + __stringify(SC_BEQZ) " %0, 1b \n" __stringify(LONG_SUBU) " %0, %1, %3 \n" - __SC "%0, %2 \n" - " beqz %0, 1b \n" - " subu %0, %1, %3 \n" " .set pop \n" : "=&r" (result), "=&r" (temp), "=m" (l->a.counter) : "Ir" (i), "m" (l->a.counter) diff --git a/arch/mips/include/asm/mach-bcm47xx/bcm47xx_board.h b/arch/mips/include/asm/mach-bcm47xx/bcm47xx_board.h index f879be3e80..6583639fe7 100644 --- a/arch/mips/include/asm/mach-bcm47xx/bcm47xx_board.h +++ b/arch/mips/include/asm/mach-bcm47xx/bcm47xx_board.h @@ -72,6 +72,7 @@ enum bcm47xx_board { BCM47XX_BOARD_LINKSYS_WRT300NV11, BCM47XX_BOARD_LINKSYS_WRT310NV1, BCM47XX_BOARD_LINKSYS_WRT310NV2, + BCM47XX_BOARD_LINKSYS_WRT320N_V1, BCM47XX_BOARD_LINKSYS_WRT54G3GV2, BCM47XX_BOARD_LINKSYS_WRT54G_TYPE_0101, BCM47XX_BOARD_LINKSYS_WRT54G_TYPE_0467, @@ -99,9 +100,12 @@ enum bcm47xx_board { BCM47XX_BOARD_MOTOROLA_WR850GV2V3, BCM47XX_BOARD_NETGEAR_R6200_V1, + BCM47XX_BOARD_NETGEAR_R6300_V1, BCM47XX_BOARD_NETGEAR_WGR614V8, BCM47XX_BOARD_NETGEAR_WGR614V9, BCM47XX_BOARD_NETGEAR_WGR614_V10, + BCM47XX_BOARD_NETGEAR_WN2500RP_V1, + BCM47XX_BOARD_NETGEAR_WN2500RP_V2, BCM47XX_BOARD_NETGEAR_WNDR3300, BCM47XX_BOARD_NETGEAR_WNDR3400V1, BCM47XX_BOARD_NETGEAR_WNDR3400V2, diff --git a/arch/mips/include/asm/mach-lantiq/xway/xway_dma.h b/arch/mips/include/asm/mach-lantiq/xway/xway_dma.h index 8218a1356b..3dee15c61c 100644 --- a/arch/mips/include/asm/mach-lantiq/xway/xway_dma.h +++ b/arch/mips/include/asm/mach-lantiq/xway/xway_dma.h @@ -8,7 +8,7 @@ #define LTQ_DMA_H__ #define LTQ_DESC_SIZE 0x08 /* each descriptor is 64bit */ -#define LTQ_DESC_NUM 0x40 /* 64 descriptors / channel */ +#define LTQ_DESC_NUM 0xC0 /* 192 descriptors / channel */ #define LTQ_DMA_OWN BIT(31) /* owner bit */ #define LTQ_DMA_C BIT(30) /* complete bit */ @@ -45,6 +45,6 @@ extern void ltq_dma_close(struct ltq_dma_channel *ch); extern void ltq_dma_alloc_tx(struct ltq_dma_channel *ch); extern void ltq_dma_alloc_rx(struct ltq_dma_channel *ch); extern void ltq_dma_free(struct ltq_dma_channel *ch); -extern void ltq_dma_init_port(int p); +extern void ltq_dma_init_port(int p, int tx_burst, int rx_burst); #endif diff --git a/arch/mips/include/asm/mach-loongson64/loongson_regs.h b/arch/mips/include/asm/mach-loongson64/loongson_regs.h index 1659935147..b5be7511f6 100644 --- a/arch/mips/include/asm/mach-loongson64/loongson_regs.h +++ b/arch/mips/include/asm/mach-loongson64/loongson_regs.h @@ -21,8 +21,10 @@ static inline u32 read_cpucfg(u32 reg) u32 __res; __asm__ __volatile__( + _ASM_SET_PARSE_R "parse_r __res,%0\n\t" "parse_r reg,%1\n\t" + _ASM_UNSET_PARSE_R ".insn \n\t" ".word (0xc8080118 | (reg << 21) | (__res << 11))\n\t" :"=r"(__res) @@ -143,8 +145,10 @@ static inline u32 csr_readl(u32 reg) /* RDCSR reg, val */ __asm__ __volatile__( + _ASM_SET_PARSE_R "parse_r __res,%0\n\t" "parse_r reg,%1\n\t" + _ASM_UNSET_PARSE_R ".insn \n\t" ".word (0xc8000118 | (reg << 21) | (__res << 11))\n\t" :"=r"(__res) @@ -160,8 +164,10 @@ static inline u64 csr_readq(u32 reg) /* DRDCSR reg, val */ __asm__ __volatile__( + _ASM_SET_PARSE_R "parse_r __res,%0\n\t" "parse_r reg,%1\n\t" + _ASM_UNSET_PARSE_R ".insn \n\t" ".word (0xc8020118 | (reg << 21) | (__res << 11))\n\t" :"=r"(__res) @@ -175,8 +181,10 @@ static inline void csr_writel(u32 val, u32 reg) { /* WRCSR reg, val */ __asm__ __volatile__( + _ASM_SET_PARSE_R "parse_r reg,%0\n\t" "parse_r val,%1\n\t" + _ASM_UNSET_PARSE_R ".insn \n\t" ".word (0xc8010118 | (reg << 21) | (val << 11))\n\t" : @@ -189,8 +197,10 @@ static inline void csr_writeq(u64 val, u32 reg) { /* DWRCSR reg, val */ __asm__ __volatile__( + _ASM_SET_PARSE_R "parse_r reg,%0\n\t" "parse_r val,%1\n\t" + _ASM_UNSET_PARSE_R ".insn \n\t" ".word (0xc8030118 | (reg << 21) | (val << 11))\n\t" : @@ -243,8 +253,10 @@ static inline u64 drdtime(void) u64 val = 0; __asm__ __volatile__( + _ASM_SET_PARSE_R "parse_r rID,%0\n\t" "parse_r val,%1\n\t" + _ASM_UNSET_PARSE_R ".insn \n\t" ".word (0xc8090118 | (rID << 21) | (val << 11))\n\t" :"=r"(rID),"=r"(val) diff --git a/arch/mips/include/asm/mach-ralink/spaces.h b/arch/mips/include/asm/mach-ralink/spaces.h index 87d085c9ad..f7af11ea2d 100644 --- a/arch/mips/include/asm/mach-ralink/spaces.h +++ b/arch/mips/include/asm/mach-ralink/spaces.h @@ -2,9 +2,11 @@ #ifndef __ASM_MACH_RALINK_SPACES_H_ #define __ASM_MACH_RALINK_SPACES_H_ -#define PCI_IOBASE _AC(0xa0000000, UL) -#define PCI_IOSIZE SZ_16M +#define PCI_IOBASE mips_io_port_base +#define PCI_IOSIZE SZ_64K #define IO_SPACE_LIMIT (PCI_IOSIZE - 1) +#define pci_remap_iospace pci_remap_iospace + #include #endif diff --git a/arch/mips/include/asm/mach-tx49xx/mangle-port.h b/arch/mips/include/asm/mach-tx49xx/mangle-port.h index 98c7abf448..50b1b8f1e1 100644 --- a/arch/mips/include/asm/mach-tx49xx/mangle-port.h +++ b/arch/mips/include/asm/mach-tx49xx/mangle-port.h @@ -9,16 +9,8 @@ #define ioswabb(a, x) (x) #define __mem_ioswabb(a, x) (x) -#if defined(CONFIG_TOSHIBA_RBTX4939) && \ - IS_ENABLED(CONFIG_SMC91X) && \ - defined(__BIG_ENDIAN) -#define NEEDS_TXX9_IOSWABW -extern u16 (*ioswabw)(volatile u16 *a, u16 x); -extern u16 (*__mem_ioswabw)(volatile u16 *a, u16 x); -#else #define ioswabw(a, x) le16_to_cpu((__force __le16)(x)) #define __mem_ioswabw(a, x) (x) -#endif #define ioswabl(a, x) le32_to_cpu((__force __le32)(x)) #define __mem_ioswabl(a, x) (x) #define ioswabq(a, x) le64_to_cpu((__force __le64)(x)) diff --git a/arch/mips/include/asm/mips-cps.h b/arch/mips/include/asm/mips-cps.h index fd43d87689..c077e8d100 100644 --- a/arch/mips/include/asm/mips-cps.h +++ b/arch/mips/include/asm/mips-cps.h @@ -7,6 +7,7 @@ #ifndef __MIPS_ASM_MIPS_CPS_H__ #define __MIPS_ASM_MIPS_CPS_H__ +#include #include #include @@ -112,14 +113,10 @@ static inline void clear_##unit##_##name(uint##sz##_t val) \ */ static inline unsigned int mips_cps_numclusters(void) { - unsigned int num_clusters; - if (mips_cm_revision() < CM_REV_CM3_5) return 1; - num_clusters = read_gcr_config() & CM_GCR_CONFIG_NUM_CLUSTERS; - num_clusters >>= __ffs(CM_GCR_CONFIG_NUM_CLUSTERS); - return num_clusters; + return FIELD_GET(CM_GCR_CONFIG_NUM_CLUSTERS, read_gcr_config()); } /** @@ -169,7 +166,8 @@ static inline unsigned int mips_cps_numcores(unsigned int cluster) return 0; /* Add one before masking to handle 0xff indicating no cores */ - return (mips_cps_cluster_config(cluster) + 1) & CM_GCR_CONFIG_PCORES; + return FIELD_GET(CM_GCR_CONFIG_PCORES, + mips_cps_cluster_config(cluster) + 1); } /** @@ -181,14 +179,11 @@ static inline unsigned int mips_cps_numcores(unsigned int cluster) */ static inline unsigned int mips_cps_numiocu(unsigned int cluster) { - unsigned int num_iocu; - if (!mips_cm_present()) return 0; - num_iocu = mips_cps_cluster_config(cluster) & CM_GCR_CONFIG_NUMIOCU; - num_iocu >>= __ffs(CM_GCR_CONFIG_NUMIOCU); - return num_iocu; + return FIELD_GET(CM_GCR_CONFIG_NUMIOCU, + mips_cps_cluster_config(cluster)); } /** @@ -230,7 +225,7 @@ static inline unsigned int mips_cps_numvps(unsigned int cluster, unsigned int co mips_cm_unlock_other(); - return (cfg + 1) & CM_GCR_Cx_CONFIG_PVPE; + return FIELD_GET(CM_GCR_Cx_CONFIG_PVPE, cfg + 1); } #endif /* __MIPS_ASM_MIPS_CPS_H__ */ diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index acdf8c6922..2616353b94 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -1297,22 +1297,22 @@ static inline int mm_insn_16bit(u16 insn) "\\var = " #n "\n\t" \ ".endif\n\t" -__asm__(".macro parse_r var r\n\t" - "\\var = -1\n\t" - _IFC_REG(0) _IFC_REG(1) _IFC_REG(2) _IFC_REG(3) - _IFC_REG(4) _IFC_REG(5) _IFC_REG(6) _IFC_REG(7) - _IFC_REG(8) _IFC_REG(9) _IFC_REG(10) _IFC_REG(11) - _IFC_REG(12) _IFC_REG(13) _IFC_REG(14) _IFC_REG(15) - _IFC_REG(16) _IFC_REG(17) _IFC_REG(18) _IFC_REG(19) - _IFC_REG(20) _IFC_REG(21) _IFC_REG(22) _IFC_REG(23) - _IFC_REG(24) _IFC_REG(25) _IFC_REG(26) _IFC_REG(27) - _IFC_REG(28) _IFC_REG(29) _IFC_REG(30) _IFC_REG(31) - ".iflt \\var\n\t" - ".error \"Unable to parse register name \\r\"\n\t" - ".endif\n\t" - ".endm"); - -#undef _IFC_REG +#define _ASM_SET_PARSE_R \ + ".macro parse_r var r\n\t" \ + "\\var = -1\n\t" \ + _IFC_REG(0) _IFC_REG(1) _IFC_REG(2) _IFC_REG(3) \ + _IFC_REG(4) _IFC_REG(5) _IFC_REG(6) _IFC_REG(7) \ + _IFC_REG(8) _IFC_REG(9) _IFC_REG(10) _IFC_REG(11) \ + _IFC_REG(12) _IFC_REG(13) _IFC_REG(14) _IFC_REG(15) \ + _IFC_REG(16) _IFC_REG(17) _IFC_REG(18) _IFC_REG(19) \ + _IFC_REG(20) _IFC_REG(21) _IFC_REG(22) _IFC_REG(23) \ + _IFC_REG(24) _IFC_REG(25) _IFC_REG(26) _IFC_REG(27) \ + _IFC_REG(28) _IFC_REG(29) _IFC_REG(30) _IFC_REG(31) \ + ".iflt \\var\n\t" \ + ".error \"Unable to parse register name \\r\"\n\t" \ + ".endif\n\t" \ + ".endm\n\t" +#define _ASM_UNSET_PARSE_R ".purgem parse_r\n\t" /* * C macros for generating assembler macros for common instruction formats. @@ -1322,43 +1322,45 @@ __asm__(".macro parse_r var r\n\t" * the ENC encodings. */ -/* Instructions with no operands */ -#define _ASM_MACRO_0(OP, ENC) \ - __asm__(".macro " #OP "\n\t" \ - ENC \ - ".endm") - /* Instructions with 1 register operand & 1 immediate operand */ #define _ASM_MACRO_1R1I(OP, R1, I2, ENC) \ - __asm__(".macro " #OP " " #R1 ", " #I2 "\n\t" \ + ".macro " #OP " " #R1 ", " #I2 "\n\t" \ + _ASM_SET_PARSE_R \ "parse_r __" #R1 ", \\" #R1 "\n\t" \ ENC \ - ".endm") + _ASM_UNSET_PARSE_R \ + ".endm\n\t" /* Instructions with 2 register operands */ #define _ASM_MACRO_2R(OP, R1, R2, ENC) \ - __asm__(".macro " #OP " " #R1 ", " #R2 "\n\t" \ + ".macro " #OP " " #R1 ", " #R2 "\n\t" \ + _ASM_SET_PARSE_R \ "parse_r __" #R1 ", \\" #R1 "\n\t" \ "parse_r __" #R2 ", \\" #R2 "\n\t" \ ENC \ - ".endm") + _ASM_UNSET_PARSE_R \ + ".endm\n\t" /* Instructions with 3 register operands */ #define _ASM_MACRO_3R(OP, R1, R2, R3, ENC) \ - __asm__(".macro " #OP " " #R1 ", " #R2 ", " #R3 "\n\t" \ + ".macro " #OP " " #R1 ", " #R2 ", " #R3 "\n\t" \ + _ASM_SET_PARSE_R \ "parse_r __" #R1 ", \\" #R1 "\n\t" \ "parse_r __" #R2 ", \\" #R2 "\n\t" \ "parse_r __" #R3 ", \\" #R3 "\n\t" \ ENC \ - ".endm") + _ASM_UNSET_PARSE_R \ + ".endm\n\t" /* Instructions with 2 register operands and 1 optional select operand */ #define _ASM_MACRO_2R_1S(OP, R1, R2, SEL3, ENC) \ - __asm__(".macro " #OP " " #R1 ", " #R2 ", " #SEL3 " = 0\n\t" \ + ".macro " #OP " " #R1 ", " #R2 ", " #SEL3 " = 0\n\t" \ + _ASM_SET_PARSE_R \ "parse_r __" #R1 ", \\" #R1 "\n\t" \ "parse_r __" #R2 ", \\" #R2 "\n\t" \ ENC \ - ".endm") + _ASM_UNSET_PARSE_R \ + ".endm\n\t" /* * TLB Invalidate Flush @@ -1618,15 +1620,21 @@ do { \ } while (0) #ifndef TOOLCHAIN_SUPPORTS_XPA -_ASM_MACRO_2R_1S(mfhc0, rt, rs, sel, - _ASM_INSN_IF_MIPS(0x40400000 | __rt << 16 | __rs << 11 | \\sel) - _ASM_INSN32_IF_MM(0x000000f4 | __rt << 21 | __rs << 16 | \\sel << 11)); -_ASM_MACRO_2R_1S(mthc0, rt, rd, sel, - _ASM_INSN_IF_MIPS(0x40c00000 | __rt << 16 | __rd << 11 | \\sel) - _ASM_INSN32_IF_MM(0x000002f4 | __rt << 21 | __rd << 16 | \\sel << 11)); -#define _ASM_SET_XPA "" +#define _ASM_SET_MFHC0 \ + _ASM_MACRO_2R_1S(mfhc0, rt, rs, sel, \ + _ASM_INSN_IF_MIPS(0x40400000 | __rt << 16 | __rs << 11 | \\sel) \ + _ASM_INSN32_IF_MM(0x000000f4 | __rt << 21 | __rs << 16 | \\sel << 11)) +#define _ASM_UNSET_MFHC0 ".purgem mfhc0\n\t" +#define _ASM_SET_MTHC0 \ + _ASM_MACRO_2R_1S(mthc0, rt, rd, sel, \ + _ASM_INSN_IF_MIPS(0x40c00000 | __rt << 16 | __rd << 11 | \\sel) \ + _ASM_INSN32_IF_MM(0x000002f4 | __rt << 21 | __rd << 16 | \\sel << 11)) +#define _ASM_UNSET_MTHC0 ".purgem mthc0\n\t" #else /* !TOOLCHAIN_SUPPORTS_XPA */ -#define _ASM_SET_XPA ".set\txpa\n\t" +#define _ASM_SET_MFHC0 ".set\txpa\n\t" +#define _ASM_SET_MTHC0 ".set\txpa\n\t" +#define _ASM_UNSET_MFHC0 +#define _ASM_UNSET_MTHC0 #endif #define __readx_32bit_c0_register(source, sel) \ @@ -1636,8 +1644,9 @@ _ASM_MACRO_2R_1S(mthc0, rt, rd, sel, __asm__ __volatile__( \ " .set push \n" \ " .set mips32r2 \n" \ - _ASM_SET_XPA \ + _ASM_SET_MFHC0 \ " mfhc0 %0, " #source ", %1 \n" \ + _ASM_UNSET_MFHC0 \ " .set pop \n" \ : "=r" (__res) \ : "i" (sel)); \ @@ -1649,8 +1658,9 @@ do { \ __asm__ __volatile__( \ " .set push \n" \ " .set mips32r2 \n" \ - _ASM_SET_XPA \ + _ASM_SET_MTHC0 \ " mthc0 %z0, " #register ", %1 \n" \ + _ASM_UNSET_MTHC0 \ " .set pop \n" \ : \ : "Jr" (value), "i" (sel)); \ @@ -2046,31 +2056,58 @@ do { \ */ #ifndef TOOLCHAIN_SUPPORTS_VIRT -_ASM_MACRO_2R_1S(mfgc0, rt, rs, sel, - _ASM_INSN_IF_MIPS(0x40600000 | __rt << 16 | __rs << 11 | \\sel) - _ASM_INSN32_IF_MM(0x000004fc | __rt << 21 | __rs << 16 | \\sel << 11)); -_ASM_MACRO_2R_1S(dmfgc0, rt, rs, sel, - _ASM_INSN_IF_MIPS(0x40600100 | __rt << 16 | __rs << 11 | \\sel) - _ASM_INSN32_IF_MM(0x580004fc | __rt << 21 | __rs << 16 | \\sel << 11)); -_ASM_MACRO_2R_1S(mtgc0, rt, rd, sel, - _ASM_INSN_IF_MIPS(0x40600200 | __rt << 16 | __rd << 11 | \\sel) - _ASM_INSN32_IF_MM(0x000006fc | __rt << 21 | __rd << 16 | \\sel << 11)); -_ASM_MACRO_2R_1S(dmtgc0, rt, rd, sel, - _ASM_INSN_IF_MIPS(0x40600300 | __rt << 16 | __rd << 11 | \\sel) - _ASM_INSN32_IF_MM(0x580006fc | __rt << 21 | __rd << 16 | \\sel << 11)); -_ASM_MACRO_0(tlbgp, _ASM_INSN_IF_MIPS(0x42000010) - _ASM_INSN32_IF_MM(0x0000017c)); -_ASM_MACRO_0(tlbgr, _ASM_INSN_IF_MIPS(0x42000009) - _ASM_INSN32_IF_MM(0x0000117c)); -_ASM_MACRO_0(tlbgwi, _ASM_INSN_IF_MIPS(0x4200000a) - _ASM_INSN32_IF_MM(0x0000217c)); -_ASM_MACRO_0(tlbgwr, _ASM_INSN_IF_MIPS(0x4200000e) - _ASM_INSN32_IF_MM(0x0000317c)); -_ASM_MACRO_0(tlbginvf, _ASM_INSN_IF_MIPS(0x4200000c) - _ASM_INSN32_IF_MM(0x0000517c)); -#define _ASM_SET_VIRT "" +#define _ASM_SET_MFGC0 \ + _ASM_MACRO_2R_1S(mfgc0, rt, rs, sel, \ + _ASM_INSN_IF_MIPS(0x40600000 | __rt << 16 | __rs << 11 | \\sel) \ + _ASM_INSN32_IF_MM(0x000004fc | __rt << 21 | __rs << 16 | \\sel << 11)) +#define _ASM_UNSET_MFGC0 ".purgem mfgc0\n\t" +#define _ASM_SET_DMFGC0 \ + _ASM_MACRO_2R_1S(dmfgc0, rt, rs, sel, \ + _ASM_INSN_IF_MIPS(0x40600100 | __rt << 16 | __rs << 11 | \\sel) \ + _ASM_INSN32_IF_MM(0x580004fc | __rt << 21 | __rs << 16 | \\sel << 11)) +#define _ASM_UNSET_DMFGC0 ".purgem dmfgc0\n\t" +#define _ASM_SET_MTGC0 \ + _ASM_MACRO_2R_1S(mtgc0, rt, rd, sel, \ + _ASM_INSN_IF_MIPS(0x40600200 | __rt << 16 | __rd << 11 | \\sel) \ + _ASM_INSN32_IF_MM(0x000006fc | __rt << 21 | __rd << 16 | \\sel << 11)) +#define _ASM_UNSET_MTGC0 ".purgem mtgc0\n\t" +#define _ASM_SET_DMTGC0 \ + _ASM_MACRO_2R_1S(dmtgc0, rt, rd, sel, \ + _ASM_INSN_IF_MIPS(0x40600300 | __rt << 16 | __rd << 11 | \\sel) \ + _ASM_INSN32_IF_MM(0x580006fc | __rt << 21 | __rd << 16 | \\sel << 11)) +#define _ASM_UNSET_DMTGC0 ".purgem dmtgc0\n\t" + +#define __tlbgp() \ + _ASM_INSN_IF_MIPS(0x42000010) \ + _ASM_INSN32_IF_MM(0x0000017c) +#define __tlbgr() \ + _ASM_INSN_IF_MIPS(0x42000009) \ + _ASM_INSN32_IF_MM(0x0000117c) +#define __tlbgwi() \ + _ASM_INSN_IF_MIPS(0x4200000a) \ + _ASM_INSN32_IF_MM(0x0000217c) +#define __tlbgwr() \ + _ASM_INSN_IF_MIPS(0x4200000e) \ + _ASM_INSN32_IF_MM(0x0000317c) +#define __tlbginvf() \ + _ASM_INSN_IF_MIPS(0x4200000c) \ + _ASM_INSN32_IF_MM(0x0000517c) #else /* !TOOLCHAIN_SUPPORTS_VIRT */ #define _ASM_SET_VIRT ".set\tvirt\n\t" +#define _ASM_SET_MFGC0 _ASM_SET_VIRT +#define _ASM_SET_DMFGC0 _ASM_SET_VIRT +#define _ASM_SET_MTGC0 _ASM_SET_VIRT +#define _ASM_SET_DMTGC0 _ASM_SET_VIRT +#define _ASM_UNSET_MFGC0 +#define _ASM_UNSET_DMFGC0 +#define _ASM_UNSET_MTGC0 +#define _ASM_UNSET_DMTGC0 + +#define __tlbgp() _ASM_SET_VIRT "tlbgp\n\t" +#define __tlbgr() _ASM_SET_VIRT "tlbgr\n\t" +#define __tlbgwi() _ASM_SET_VIRT "tlbgwi\n\t" +#define __tlbgwr() _ASM_SET_VIRT "tlbgwr\n\t" +#define __tlbginvf() _ASM_SET_VIRT "tlbginvf\n\t" #endif #define __read_32bit_gc0_register(source, sel) \ @@ -2078,8 +2115,9 @@ _ASM_MACRO_0(tlbginvf, _ASM_INSN_IF_MIPS(0x4200000c) __asm__ __volatile__( \ ".set\tpush\n\t" \ ".set\tmips32r5\n\t" \ - _ASM_SET_VIRT \ + _ASM_SET_MFGC0 \ "mfgc0\t%0, " #source ", %1\n\t" \ + _ASM_UNSET_MFGC0 \ ".set\tpop" \ : "=r" (__res) \ : "i" (sel)); \ @@ -2091,8 +2129,9 @@ _ASM_MACRO_0(tlbginvf, _ASM_INSN_IF_MIPS(0x4200000c) __asm__ __volatile__( \ ".set\tpush\n\t" \ ".set\tmips64r5\n\t" \ - _ASM_SET_VIRT \ + _ASM_SET_DMFGC0 \ "dmfgc0\t%0, " #source ", %1\n\t" \ + _ASM_UNSET_DMFGC0 \ ".set\tpop" \ : "=r" (__res) \ : "i" (sel)); \ @@ -2104,8 +2143,9 @@ do { \ __asm__ __volatile__( \ ".set\tpush\n\t" \ ".set\tmips32r5\n\t" \ - _ASM_SET_VIRT \ + _ASM_SET_MTGC0 \ "mtgc0\t%z0, " #register ", %1\n\t" \ + _ASM_UNSET_MTGC0 \ ".set\tpop" \ : : "Jr" ((unsigned int)(value)), \ "i" (sel)); \ @@ -2116,8 +2156,9 @@ do { \ __asm__ __volatile__( \ ".set\tpush\n\t" \ ".set\tmips64r5\n\t" \ - _ASM_SET_VIRT \ + _ASM_SET_DMTGC0 \ "dmtgc0\t%z0, " #register ", %1\n\t" \ + _ASM_UNSET_DMTGC0 \ ".set\tpop" \ : : "Jr" (value), \ "i" (sel)); \ @@ -2788,8 +2829,7 @@ static inline void guest_tlb_probe(void) __asm__ __volatile__( ".set push\n\t" ".set noreorder\n\t" - _ASM_SET_VIRT - "tlbgp\n\t" + __tlbgp() ".set pop"); } @@ -2798,8 +2838,7 @@ static inline void guest_tlb_read(void) __asm__ __volatile__( ".set push\n\t" ".set noreorder\n\t" - _ASM_SET_VIRT - "tlbgr\n\t" + __tlbgr() ".set pop"); } @@ -2808,8 +2847,7 @@ static inline void guest_tlb_write_indexed(void) __asm__ __volatile__( ".set push\n\t" ".set noreorder\n\t" - _ASM_SET_VIRT - "tlbgwi\n\t" + __tlbgwi() ".set pop"); } @@ -2818,8 +2856,7 @@ static inline void guest_tlb_write_random(void) __asm__ __volatile__( ".set push\n\t" ".set noreorder\n\t" - _ASM_SET_VIRT - "tlbgwr\n\t" + __tlbgwr() ".set pop"); } @@ -2831,8 +2868,7 @@ static inline void guest_tlbinvf(void) __asm__ __volatile__( ".set push\n\t" ".set noreorder\n\t" - _ASM_SET_VIRT - "tlbginvf\n\t" + __tlbginvf() ".set pop"); } diff --git a/arch/mips/include/asm/msa.h b/arch/mips/include/asm/msa.h index e0a3dd5233..236a49ee2e 100644 --- a/arch/mips/include/asm/msa.h +++ b/arch/mips/include/asm/msa.h @@ -162,16 +162,26 @@ static inline void init_msa_upper(void) * to allow compilation with toolchains that do not support MSA. Once all * toolchains in use support MSA these can be removed. */ -_ASM_MACRO_2R(cfcmsa, rd, cs, - _ASM_INSN_IF_MIPS(0x787e0019 | __cs << 11 | __rd << 6) - _ASM_INSN32_IF_MM(0x587e0016 | __cs << 11 | __rd << 6)); -_ASM_MACRO_2R(ctcmsa, cd, rs, - _ASM_INSN_IF_MIPS(0x783e0019 | __rs << 11 | __cd << 6) - _ASM_INSN32_IF_MM(0x583e0016 | __rs << 11 | __cd << 6)); -#define _ASM_SET_MSA "" + +#define _ASM_SET_CFCMSA \ + _ASM_MACRO_2R(cfcmsa, rd, cs, \ + _ASM_INSN_IF_MIPS(0x787e0019 | __cs << 11 | __rd << 6) \ + _ASM_INSN32_IF_MM(0x587e0016 | __cs << 11 | __rd << 6)) +#define _ASM_UNSET_CFCMSA ".purgem cfcmsa\n\t" +#define _ASM_SET_CTCMSA \ + _ASM_MACRO_2R(ctcmsa, cd, rs, \ + _ASM_INSN_IF_MIPS(0x783e0019 | __rs << 11 | __cd << 6) \ + _ASM_INSN32_IF_MM(0x583e0016 | __rs << 11 | __cd << 6)) +#define _ASM_UNSET_CTCMSA ".purgem ctcmsa\n\t" #else /* TOOLCHAIN_SUPPORTS_MSA */ -#define _ASM_SET_MSA ".set\tfp=64\n\t" \ - ".set\tmsa\n\t" +#define _ASM_SET_CFCMSA \ + ".set\tfp=64\n\t" \ + ".set\tmsa\n\t" +#define _ASM_UNSET_CFCMSA +#define _ASM_SET_CTCMSA \ + ".set\tfp=64\n\t" \ + ".set\tmsa\n\t" +#define _ASM_UNSET_CTCMSA #endif #define __BUILD_MSA_CTL_REG(name, cs) \ @@ -180,8 +190,9 @@ static inline unsigned int read_msa_##name(void) \ unsigned int reg; \ __asm__ __volatile__( \ " .set push\n" \ - _ASM_SET_MSA \ + _ASM_SET_CFCMSA \ " cfcmsa %0, $" #cs "\n" \ + _ASM_UNSET_CFCMSA \ " .set pop\n" \ : "=r"(reg)); \ return reg; \ @@ -191,8 +202,9 @@ static inline void write_msa_##name(unsigned int val) \ { \ __asm__ __volatile__( \ " .set push\n" \ - _ASM_SET_MSA \ + _ASM_SET_CTCMSA \ " ctcmsa $" #cs ", %0\n" \ + _ASM_UNSET_CTCMSA \ " .set pop\n" \ : : "r"(val)); \ } diff --git a/arch/mips/include/asm/octeon/cvmx-helper.h b/arch/mips/include/asm/octeon/cvmx-helper.h index c6c99e28ee..0cddce3529 100644 --- a/arch/mips/include/asm/octeon/cvmx-helper.h +++ b/arch/mips/include/asm/octeon/cvmx-helper.h @@ -93,13 +93,6 @@ extern int cvmx_helper_ipd_and_packet_input_enable(void); */ extern int cvmx_helper_initialize_packet_io_global(void); -/** - * Does core local initialization for packet io - * - * Returns Zero on success, non-zero on failure - */ -extern int cvmx_helper_initialize_packet_io_local(void); - /** * Returns the number of ports on the given interface. * The interface must be initialized before the port count diff --git a/arch/mips/include/asm/octeon/cvmx-pko.h b/arch/mips/include/asm/octeon/cvmx-pko.h index 03fb64b13f..5fec8476e4 100644 --- a/arch/mips/include/asm/octeon/cvmx-pko.h +++ b/arch/mips/include/asm/octeon/cvmx-pko.h @@ -277,7 +277,6 @@ typedef struct { * output system. */ extern void cvmx_pko_initialize_global(void); -extern int cvmx_pko_initialize_local(void); /** * Enables the packet output hardware. It must already be diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index 804889b709..7b8037f25d 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -86,10 +86,9 @@ extern void paging_init(void); */ #define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd)) -#define __pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) -#ifndef CONFIG_TRANSPARENT_HUGEPAGE -#define pmd_page(pmd) __pmd_page(pmd) -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#ifndef CONFIG_MIPS_HUGE_TLB_SUPPORT +#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) +#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ #define pmd_page_vaddr(pmd) pmd_val(pmd) @@ -416,6 +415,25 @@ static inline pte_t pte_mkhuge(pte_t pte) pte_val(pte) |= _PAGE_HUGE; return pte; } + +#define pmd_write pmd_write +static inline int pmd_write(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_WRITE); +} + +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + return pmd_val(pmd) >> _PFN_SHIFT; +} + +static inline struct page *pmd_page(pmd_t pmd) +{ + if (pmd_val(pmd) & _PAGE_HUGE) + return pfn_to_page(pmd_pfn(pmd)); + + return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT); +} #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY @@ -591,12 +609,6 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd) extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd); -#define pmd_write pmd_write -static inline int pmd_write(pmd_t pmd) -{ - return !!(pmd_val(pmd) & _PAGE_WRITE); -} - static inline pmd_t pmd_wrprotect(pmd_t pmd) { pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_SILENT_WRITE); @@ -677,19 +689,6 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) /* Extern to avoid header file madness */ extern pmd_t mk_pmd(struct page *page, pgprot_t prot); -static inline unsigned long pmd_pfn(pmd_t pmd) -{ - return pmd_val(pmd) >> _PFN_SHIFT; -} - -static inline struct page *pmd_page(pmd_t pmd) -{ - if (pmd_trans_huge(pmd)) - return pfn_to_page(pmd_pfn(pmd)); - - return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT); -} - static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { pmd_val(pmd) = (pmd_val(pmd) & (_PAGE_CHG_MASK | _PAGE_HUGE)) | diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h index 0c3550c82b..4bb24579d1 100644 --- a/arch/mips/include/asm/processor.h +++ b/arch/mips/include/asm/processor.h @@ -207,16 +207,6 @@ struct octeon_cvmseg_state { [cpu_dcache_line_size() / sizeof(unsigned long)]; }; -#elif defined(CONFIG_CPU_XLP) -struct nlm_cop2_state { - u64 rx[4]; - u64 tx[4]; - u32 tx_msg_status; - u32 rx_msg_status; -}; - -#define COP2_INIT \ - .cp2 = {{0}, {0}, 0, 0}, #else #define COP2_INIT #endif @@ -274,9 +264,6 @@ struct thread_struct { #ifdef CONFIG_CPU_CAVIUM_OCTEON struct octeon_cop2_state cp2 __attribute__ ((__aligned__(128))); struct octeon_cvmseg_state cvmseg __attribute__ ((__aligned__(128))); -#endif -#ifdef CONFIG_CPU_XLP - struct nlm_cop2_state cp2; #endif struct mips_abi *abi; }; @@ -369,7 +356,7 @@ static inline void flush_thread(void) { } -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); #define __KSTK_TOS(tsk) ((unsigned long)task_stack_page(tsk) + \ THREAD_SIZE - 32 - sizeof(struct pt_regs)) diff --git a/arch/mips/include/asm/sibyte/sb1250_mc.h b/arch/mips/include/asm/sibyte/sb1250_mc.h index c02fe823ef..61411619df 100644 --- a/arch/mips/include/asm/sibyte/sb1250_mc.h +++ b/arch/mips/include/asm/sibyte/sb1250_mc.h @@ -484,7 +484,7 @@ /* - * Bank Address Address Bits Register (Table 6-22) + * Bank Address Bits Register (Table 6-22) */ #define S_MC_BA_RESERVED 0 diff --git a/arch/mips/include/asm/smp-ops.h b/arch/mips/include/asm/smp-ops.h index 65618ff128..864aea8039 100644 --- a/arch/mips/include/asm/smp-ops.h +++ b/arch/mips/include/asm/smp-ops.h @@ -101,6 +101,9 @@ static inline int register_vsmp_smp_ops(void) #ifdef CONFIG_MIPS_MT_SMP extern const struct plat_smp_ops vsmp_smp_ops; + if (!cpu_has_mipsmt) + return -ENODEV; + register_smp_ops(&vsmp_smp_ops); return 0; diff --git a/arch/mips/include/asm/traps.h b/arch/mips/include/asm/traps.h index b710e76c9c..15cde638b4 100644 --- a/arch/mips/include/asm/traps.h +++ b/arch/mips/include/asm/traps.h @@ -15,7 +15,7 @@ #define MIPS_BE_FATAL 2 /* treat as an unrecoverable error */ extern void (*board_be_init)(void); -extern int (*board_be_handler)(struct pt_regs *regs, int is_fixup); +void mips_set_be_handler(int (*handler)(struct pt_regs *reg, int is_fixup)); extern void (*board_nmi_handler_setup)(void); extern void (*board_ejtag_handler_setup)(void); diff --git a/arch/mips/include/asm/txx9/boards.h b/arch/mips/include/asm/txx9/boards.h index d45237befd..70284e90dc 100644 --- a/arch/mips/include/asm/txx9/boards.h +++ b/arch/mips/include/asm/txx9/boards.h @@ -6,9 +6,3 @@ BOARD_VEC(jmr3927_vec) BOARD_VEC(rbtx4927_vec) BOARD_VEC(rbtx4937_vec) #endif -#ifdef CONFIG_TOSHIBA_RBTX4938 -BOARD_VEC(rbtx4938_vec) -#endif -#ifdef CONFIG_TOSHIBA_RBTX4939 -BOARD_VEC(rbtx4939_vec) -#endif diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index f7effca791..296bcf31ab 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h @@ -145,6 +145,7 @@ Ip_u1(_mtlo); Ip_u3u1u2(_mul); Ip_u1u2(_multu); Ip_u3u1u2(_mulu); +Ip_u3u1u2(_muhu); Ip_u3u1u2(_nor); Ip_u3u1u2(_or); Ip_u2u1u3(_ori); @@ -248,7 +249,11 @@ static inline void uasm_l##lb(struct uasm_label **lab, u32 *addr) \ #define uasm_i_bnezl(buf, rs, off) uasm_i_bnel(buf, rs, 0, off) #define uasm_i_ehb(buf) uasm_i_sll(buf, 0, 0, 3) #define uasm_i_move(buf, a, b) UASM_i_ADDU(buf, a, 0, b) +#ifdef CONFIG_CPU_NOP_WORKAROUNDS +#define uasm_i_nop(buf) uasm_i_or(buf, 1, 1, 0) +#else #define uasm_i_nop(buf) uasm_i_sll(buf, 0, 0, 0) +#endif #define uasm_i_ssnop(buf) uasm_i_sll(buf, 0, 0, 1) static inline void uasm_i_drotr_safe(u32 **p, unsigned int a1, diff --git a/arch/mips/include/asm/vermagic.h b/arch/mips/include/asm/vermagic.h index 371c1873df..0904de0b5e 100644 --- a/arch/mips/include/asm/vermagic.h +++ b/arch/mips/include/asm/vermagic.h @@ -54,10 +54,6 @@ #define MODULE_PROC_FAMILY "OCTEON " #elif defined CONFIG_CPU_P5600 #define MODULE_PROC_FAMILY "P5600 " -#elif defined CONFIG_CPU_XLR -#define MODULE_PROC_FAMILY "XLR " -#elif defined CONFIG_CPU_XLP -#define MODULE_PROC_FAMILY "XLP " #else #error MODULE_PROC_FAMILY undefined for your processor configuration #endif diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h index 1eaf6a1ca5..24e0efb360 100644 --- a/arch/mips/include/uapi/asm/socket.h +++ b/arch/mips/include/uapi/asm/socket.h @@ -142,6 +142,8 @@ #define SO_BUF_LOCK 72 +#define SO_RESERVE_MEM 73 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 7c861e6a89..24a529c6c4 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -1886,87 +1886,6 @@ static inline void cpu_probe_ingenic(struct cpuinfo_mips *c, unsigned int cpu) } } -static inline void cpu_probe_netlogic(struct cpuinfo_mips *c, int cpu) -{ - decode_configs(c); - - if ((c->processor_id & PRID_IMP_MASK) == PRID_IMP_NETLOGIC_AU13XX) { - c->cputype = CPU_ALCHEMY; - __cpu_name[cpu] = "Au1300"; - /* following stuff is not for Alchemy */ - return; - } - - c->options = (MIPS_CPU_TLB | - MIPS_CPU_4KEX | - MIPS_CPU_COUNTER | - MIPS_CPU_DIVEC | - MIPS_CPU_WATCH | - MIPS_CPU_EJTAG | - MIPS_CPU_LLSC); - - switch (c->processor_id & PRID_IMP_MASK) { - case PRID_IMP_NETLOGIC_XLP2XX: - case PRID_IMP_NETLOGIC_XLP9XX: - case PRID_IMP_NETLOGIC_XLP5XX: - c->cputype = CPU_XLP; - __cpu_name[cpu] = "Broadcom XLPII"; - break; - - case PRID_IMP_NETLOGIC_XLP8XX: - case PRID_IMP_NETLOGIC_XLP3XX: - c->cputype = CPU_XLP; - __cpu_name[cpu] = "Netlogic XLP"; - break; - - case PRID_IMP_NETLOGIC_XLR732: - case PRID_IMP_NETLOGIC_XLR716: - case PRID_IMP_NETLOGIC_XLR532: - case PRID_IMP_NETLOGIC_XLR308: - case PRID_IMP_NETLOGIC_XLR532C: - case PRID_IMP_NETLOGIC_XLR516C: - case PRID_IMP_NETLOGIC_XLR508C: - case PRID_IMP_NETLOGIC_XLR308C: - c->cputype = CPU_XLR; - __cpu_name[cpu] = "Netlogic XLR"; - break; - - case PRID_IMP_NETLOGIC_XLS608: - case PRID_IMP_NETLOGIC_XLS408: - case PRID_IMP_NETLOGIC_XLS404: - case PRID_IMP_NETLOGIC_XLS208: - case PRID_IMP_NETLOGIC_XLS204: - case PRID_IMP_NETLOGIC_XLS108: - case PRID_IMP_NETLOGIC_XLS104: - case PRID_IMP_NETLOGIC_XLS616B: - case PRID_IMP_NETLOGIC_XLS608B: - case PRID_IMP_NETLOGIC_XLS416B: - case PRID_IMP_NETLOGIC_XLS412B: - case PRID_IMP_NETLOGIC_XLS408B: - case PRID_IMP_NETLOGIC_XLS404B: - c->cputype = CPU_XLR; - __cpu_name[cpu] = "Netlogic XLS"; - break; - - default: - pr_info("Unknown Netlogic chip id [%02x]!\n", - c->processor_id); - c->cputype = CPU_XLR; - break; - } - - if (c->cputype == CPU_XLP) { - set_isa(c, MIPS_CPU_ISA_M64R2); - c->options |= (MIPS_CPU_FPU | MIPS_CPU_ULRI | MIPS_CPU_MCHECK); - /* This will be updated again after all threads are woken up */ - c->tlbsize = ((read_c0_config6() >> 16) & 0xffff) + 1; - } else { - set_isa(c, MIPS_CPU_ISA_M64R1); - c->tlbsize = ((read_c0_config1() >> 25) & 0x3f) + 1; - } - c->kscratch_mask = 0xf; -} - #ifdef CONFIG_64BIT /* For use by uaccess.h */ u64 __ua_limit; @@ -2031,9 +1950,6 @@ void cpu_probe(void) case PRID_COMP_INGENIC_E1: cpu_probe_ingenic(c, cpu); break; - case PRID_COMP_NETLOGIC: - cpu_probe_netlogic(c, cpu); - break; } BUG_ON(!__cpu_name[cpu]); diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c index 1aca3b4db9..c81b3a0394 100644 --- a/arch/mips/kernel/idle.c +++ b/arch/mips/kernel/idle.c @@ -175,8 +175,6 @@ void __init check_wait(void) case CPU_CAVIUM_OCTEON3: case CPU_XBURST: case CPU_LOONGSON32: - case CPU_XLR: - case CPU_XLP: cpu_wait = r4k_wait; break; case CPU_LOONGSON64: diff --git a/arch/mips/kernel/irq.c b/arch/mips/kernel/irq.c index d20e002b32..5e11582fe3 100644 --- a/arch/mips/kernel/irq.c +++ b/arch/mips/kernel/irq.c @@ -111,15 +111,9 @@ void __irq_entry do_IRQ(unsigned int irq) #ifdef CONFIG_IRQ_DOMAIN void __irq_entry do_domain_IRQ(struct irq_domain *domain, unsigned int hwirq) { - struct irq_desc *desc; - irq_enter(); check_stack_overflow(); - - desc = irq_resolve_mapping(domain, hwirq); - if (likely(desc)) - handle_irq_desc(desc); - + generic_handle_domain_irq(domain, hwirq); irq_exit(); } #endif diff --git a/arch/mips/kernel/kprobes.c b/arch/mips/kernel/kprobes.c index 75bff0f773..6c7f3b143f 100644 --- a/arch/mips/kernel/kprobes.c +++ b/arch/mips/kernel/kprobes.c @@ -11,6 +11,8 @@ * Copyright (C) IBM Corporation, 2002, 2004 */ +#define pr_fmt(fmt) "kprobes: " fmt + #include #include #include @@ -80,8 +82,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) insn = p->addr[0]; if (insn_has_ll_or_sc(insn)) { - pr_notice("Kprobes for ll and sc instructions are not" - "supported\n"); + pr_notice("Kprobes for ll and sc instructions are not supported\n"); ret = -EINVAL; goto out; } @@ -219,7 +220,7 @@ static int evaluate_branch_instruction(struct kprobe *p, struct pt_regs *regs, return 0; unaligned: - pr_notice("%s: unaligned epc - sending SIGBUS.\n", current->comm); + pr_notice("Failed to emulate branch instruction because of unaligned epc - sending SIGBUS to %s.\n", current->comm); force_sig(SIGBUS); return -EFAULT; @@ -238,10 +239,8 @@ static void prepare_singlestep(struct kprobe *p, struct pt_regs *regs, regs->cp0_epc = (unsigned long)p->addr; else if (insn_has_delayslot(p->opcode)) { ret = evaluate_branch_instruction(p, regs, kcb); - if (ret < 0) { - pr_notice("Kprobes: Error in evaluating branch\n"); + if (ret < 0) return; - } } regs->cp0_epc = (unsigned long)&p->ainsn.insn[0]; } @@ -461,14 +460,14 @@ static void __used kretprobe_trampoline_holder(void) /* Keep the assembler from reordering and placing JR here. */ ".set noreorder\n\t" "nop\n\t" - ".global kretprobe_trampoline\n" - "kretprobe_trampoline:\n\t" + ".global __kretprobe_trampoline\n" + "__kretprobe_trampoline:\n\t" "nop\n\t" ".set pop" : : : "memory"); } -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) @@ -477,7 +476,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, ri->fp = NULL; /* Replace the return addr with trampoline addr */ - regs->regs[31] = (unsigned long)kretprobe_trampoline; + regs->regs[31] = (unsigned long)__kretprobe_trampoline; } /* @@ -486,8 +485,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, static int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) { - instruction_pointer(regs) = __kretprobe_trampoline_handler(regs, - kretprobe_trampoline, NULL); + instruction_pointer(regs) = __kretprobe_trampoline_handler(regs, NULL); /* * By returning a non-zero value, we are telling * kprobe_handler() that we don't want the post_handler @@ -498,14 +496,14 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, int __kprobes arch_trampoline_kprobe(struct kprobe *p) { - if (p->addr == (kprobe_opcode_t *)kretprobe_trampoline) + if (p->addr == (kprobe_opcode_t *)__kretprobe_trampoline) return 1; return 0; } static struct kprobe trampoline_p = { - .addr = (kprobe_opcode_t *)kretprobe_trampoline, + .addr = (kprobe_opcode_t *)__kretprobe_trampoline, .pre_handler = trampoline_probe_handler }; diff --git a/arch/mips/kernel/mips-cpc.c b/arch/mips/kernel/mips-cpc.c index 8d2535123f..17aff13cd7 100644 --- a/arch/mips/kernel/mips-cpc.c +++ b/arch/mips/kernel/mips-cpc.c @@ -4,6 +4,7 @@ * Author: Paul Burton */ +#include #include #include #include @@ -97,7 +98,7 @@ void mips_cpc_lock_other(unsigned int core) curr_core = cpu_core(¤t_cpu_data); spin_lock_irqsave(&per_cpu(cpc_core_lock, curr_core), per_cpu(cpc_core_lock_flags, curr_core)); - write_cpc_cl_other(core << __ffs(CPC_Cx_OTHER_CORENUM)); + write_cpc_cl_other(FIELD_PREP(CPC_Cx_OTHER_CORENUM, core)); /* * Ensure the core-other region reflects the appropriate core & diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index 22e22c2de1..1641d274fe 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -1002,15 +1002,6 @@ static const struct mips_perf_event bmips5000_event_map [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T }, }; -static const struct mips_perf_event xlp_event_map[PERF_COUNT_HW_MAX] = { - [PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL }, - [PERF_COUNT_HW_INSTRUCTIONS] = { 0x18, CNTR_ALL }, /* PAPI_TOT_INS */ - [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */ - [PERF_COUNT_HW_CACHE_MISSES] = { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */ - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x1b, CNTR_ALL }, /* PAPI_BR_CN */ - [PERF_COUNT_HW_BRANCH_MISSES] = { 0x1c, CNTR_ALL }, /* PAPI_BR_MSP */ -}; - /* 24K/34K/1004K/interAptiv/loongson1 cores share the same cache event map. */ static const struct mips_perf_event mipsxxcore_cache_map [PERF_COUNT_HW_CACHE_MAX] @@ -1477,63 +1468,6 @@ static const struct mips_perf_event octeon_cache_map }, }; -static const struct mips_perf_event xlp_cache_map - [PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { -[C(L1D)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 0x31, CNTR_ALL }, /* PAPI_L1_DCR */ - [C(RESULT_MISS)] = { 0x30, CNTR_ALL }, /* PAPI_L1_LDM */ - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { 0x2f, CNTR_ALL }, /* PAPI_L1_DCW */ - [C(RESULT_MISS)] = { 0x2e, CNTR_ALL }, /* PAPI_L1_STM */ - }, -}, -[C(L1I)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */ - [C(RESULT_MISS)] = { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */ - }, -}, -[C(LL)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 0x35, CNTR_ALL }, /* PAPI_L2_DCR */ - [C(RESULT_MISS)] = { 0x37, CNTR_ALL }, /* PAPI_L2_LDM */ - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { 0x34, CNTR_ALL }, /* PAPI_L2_DCA */ - [C(RESULT_MISS)] = { 0x36, CNTR_ALL }, /* PAPI_L2_DCM */ - }, -}, -[C(DTLB)] = { - /* - * Only general DTLB misses are counted use the same event for - * read and write. - */ - [C(OP_READ)] = { - [C(RESULT_MISS)] = { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */ - }, - [C(OP_WRITE)] = { - [C(RESULT_MISS)] = { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */ - }, -}, -[C(ITLB)] = { - [C(OP_READ)] = { - [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */ - }, - [C(OP_WRITE)] = { - [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */ - }, -}, -[C(BPU)] = { - [C(OP_READ)] = { - [C(RESULT_MISS)] = { 0x25, CNTR_ALL }, - }, -}, -}; - static int __hw_perf_event_init(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; @@ -1953,20 +1887,6 @@ static const struct mips_perf_event *octeon_pmu_map_raw_event(u64 config) return &raw_event; } -static const struct mips_perf_event *xlp_pmu_map_raw_event(u64 config) -{ - unsigned int raw_id = config & 0xff; - - /* Only 1-63 are defined */ - if ((raw_id < 0x01) || (raw_id > 0x3f)) - return ERR_PTR(-EOPNOTSUPP); - - raw_event.cntr_mask = CNTR_ALL; - raw_event.event_id = raw_id; - - return &raw_event; -} - static int __init init_hw_perf_events(void) { @@ -2091,12 +2011,6 @@ init_hw_perf_events(void) mipspmu.general_event_map = &bmips5000_event_map; mipspmu.cache_event_map = &bmips5000_cache_map; break; - case CPU_XLP: - mipspmu.name = "xlp"; - mipspmu.general_event_map = &xlp_event_map; - mipspmu.cache_event_map = &xlp_cache_map; - mipspmu.map_raw_event = xlp_pmu_map_raw_event; - break; default: pr_cont("Either hardware does not support performance " "counters, or not yet implemented.\n"); diff --git a/arch/mips/kernel/proc.c b/arch/mips/kernel/proc.c index 4184d641f0..9f47a889b0 100644 --- a/arch/mips/kernel/proc.c +++ b/arch/mips/kernel/proc.c @@ -19,8 +19,8 @@ unsigned int vced_count, vcei_count; /* - * * No lock; only written during early bootup by CPU 0. - * */ + * No lock; only written during early bootup by CPU 0. + */ static RAW_NOTIFIER_HEAD(proc_cpuinfo_chain); int __ref register_proc_cpuinfo_notifier(struct notifier_block *nb) @@ -39,7 +39,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) unsigned long n = (unsigned long) v - 1; unsigned int version = cpu_data[n].processor_id; unsigned int fp_vers = cpu_data[n].fpu_id; - char fmt [64]; + char fmt[64]; int i; #ifdef CONFIG_SMP @@ -78,66 +78,207 @@ static int show_cpuinfo(struct seq_file *m, void *v) seq_printf(m, "count: %d, address/irw mask: [", cpu_data[n].watch_reg_count); for (i = 0; i < cpu_data[n].watch_reg_count; i++) - seq_printf(m, "%s0x%04x", i ? ", " : "" , + seq_printf(m, "%s0x%04x", i ? ", " : "", cpu_data[n].watch_reg_masks[i]); - seq_printf(m, "]\n"); + seq_puts(m, "]\n"); } - seq_printf(m, "isa\t\t\t:"); + seq_puts(m, "isa\t\t\t:"); if (cpu_has_mips_1) - seq_printf(m, " mips1"); + seq_puts(m, " mips1"); if (cpu_has_mips_2) - seq_printf(m, "%s", " mips2"); + seq_puts(m, " mips2"); if (cpu_has_mips_3) - seq_printf(m, "%s", " mips3"); + seq_puts(m, " mips3"); if (cpu_has_mips_4) - seq_printf(m, "%s", " mips4"); + seq_puts(m, " mips4"); if (cpu_has_mips_5) - seq_printf(m, "%s", " mips5"); + seq_puts(m, " mips5"); if (cpu_has_mips32r1) - seq_printf(m, "%s", " mips32r1"); + seq_puts(m, " mips32r1"); if (cpu_has_mips32r2) - seq_printf(m, "%s", " mips32r2"); + seq_puts(m, " mips32r2"); if (cpu_has_mips32r5) - seq_printf(m, "%s", " mips32r5"); + seq_puts(m, " mips32r5"); if (cpu_has_mips32r6) - seq_printf(m, "%s", " mips32r6"); + seq_puts(m, " mips32r6"); if (cpu_has_mips64r1) - seq_printf(m, "%s", " mips64r1"); + seq_puts(m, " mips64r1"); if (cpu_has_mips64r2) - seq_printf(m, "%s", " mips64r2"); + seq_puts(m, " mips64r2"); if (cpu_has_mips64r5) - seq_printf(m, "%s", " mips64r5"); + seq_puts(m, " mips64r5"); if (cpu_has_mips64r6) - seq_printf(m, "%s", " mips64r6"); - seq_printf(m, "\n"); + seq_puts(m, " mips64r6"); + seq_puts(m, "\n"); - seq_printf(m, "ASEs implemented\t:"); - if (cpu_has_mips16) seq_printf(m, "%s", " mips16"); - if (cpu_has_mips16e2) seq_printf(m, "%s", " mips16e2"); - if (cpu_has_mdmx) seq_printf(m, "%s", " mdmx"); - if (cpu_has_mips3d) seq_printf(m, "%s", " mips3d"); - if (cpu_has_smartmips) seq_printf(m, "%s", " smartmips"); - if (cpu_has_dsp) seq_printf(m, "%s", " dsp"); - if (cpu_has_dsp2) seq_printf(m, "%s", " dsp2"); - if (cpu_has_dsp3) seq_printf(m, "%s", " dsp3"); - if (cpu_has_mipsmt) seq_printf(m, "%s", " mt"); - if (cpu_has_mmips) seq_printf(m, "%s", " micromips"); - if (cpu_has_vz) seq_printf(m, "%s", " vz"); - if (cpu_has_msa) seq_printf(m, "%s", " msa"); - if (cpu_has_eva) seq_printf(m, "%s", " eva"); - if (cpu_has_htw) seq_printf(m, "%s", " htw"); - if (cpu_has_xpa) seq_printf(m, "%s", " xpa"); - if (cpu_has_loongson_mmi) seq_printf(m, "%s", " loongson-mmi"); - if (cpu_has_loongson_cam) seq_printf(m, "%s", " loongson-cam"); - if (cpu_has_loongson_ext) seq_printf(m, "%s", " loongson-ext"); - if (cpu_has_loongson_ext2) seq_printf(m, "%s", " loongson-ext2"); - seq_printf(m, "\n"); + seq_puts(m, "ASEs implemented\t:"); + if (cpu_has_mips16) + seq_puts(m, " mips16"); + if (cpu_has_mips16e2) + seq_puts(m, " mips16e2"); + if (cpu_has_mdmx) + seq_puts(m, " mdmx"); + if (cpu_has_mips3d) + seq_puts(m, " mips3d"); + if (cpu_has_smartmips) + seq_puts(m, " smartmips"); + if (cpu_has_dsp) + seq_puts(m, " dsp"); + if (cpu_has_dsp2) + seq_puts(m, " dsp2"); + if (cpu_has_dsp3) + seq_puts(m, " dsp3"); + if (cpu_has_mipsmt) + seq_puts(m, " mt"); + if (cpu_has_mmips) + seq_puts(m, " micromips"); + if (cpu_has_vz) + seq_puts(m, " vz"); + if (cpu_has_msa) + seq_puts(m, " msa"); + if (cpu_has_eva) + seq_puts(m, " eva"); + if (cpu_has_htw) + seq_puts(m, " htw"); + if (cpu_has_xpa) + seq_puts(m, " xpa"); + if (cpu_has_loongson_mmi) + seq_puts(m, " loongson-mmi"); + if (cpu_has_loongson_cam) + seq_puts(m, " loongson-cam"); + if (cpu_has_loongson_ext) + seq_puts(m, " loongson-ext"); + if (cpu_has_loongson_ext2) + seq_puts(m, " loongson-ext2"); + seq_puts(m, "\n"); if (cpu_has_mmips) { seq_printf(m, "micromips kernel\t: %s\n", (read_c0_config3() & MIPS_CONF3_ISA_OE) ? "yes" : "no"); } + + seq_puts(m, "Options implemented\t:"); + if (cpu_has_tlb) + seq_puts(m, " tlb"); + if (cpu_has_ftlb) + seq_puts(m, " ftlb"); + if (cpu_has_tlbinv) + seq_puts(m, " tlbinv"); + if (cpu_has_segments) + seq_puts(m, " segments"); + if (cpu_has_rixiex) + seq_puts(m, " rixiex"); + if (cpu_has_ldpte) + seq_puts(m, " ldpte"); + if (cpu_has_maar) + seq_puts(m, " maar"); + if (cpu_has_rw_llb) + seq_puts(m, " rw_llb"); + if (cpu_has_4kex) + seq_puts(m, " 4kex"); + if (cpu_has_3k_cache) + seq_puts(m, " 3k_cache"); + if (cpu_has_4k_cache) + seq_puts(m, " 4k_cache"); + if (cpu_has_tx39_cache) + seq_puts(m, " tx39_cache"); + if (cpu_has_octeon_cache) + seq_puts(m, " octeon_cache"); + if (raw_cpu_has_fpu) + seq_puts(m, " fpu"); + if (cpu_has_32fpr) + seq_puts(m, " 32fpr"); + if (cpu_has_cache_cdex_p) + seq_puts(m, " cache_cdex_p"); + if (cpu_has_cache_cdex_s) + seq_puts(m, " cache_cdex_s"); + if (cpu_has_prefetch) + seq_puts(m, " prefetch"); + if (cpu_has_mcheck) + seq_puts(m, " mcheck"); + if (cpu_has_ejtag) + seq_puts(m, " ejtag"); + if (cpu_has_llsc) + seq_puts(m, " llsc"); + if (cpu_has_guestctl0ext) + seq_puts(m, " guestctl0ext"); + if (cpu_has_guestctl1) + seq_puts(m, " guestctl1"); + if (cpu_has_guestctl2) + seq_puts(m, " guestctl2"); + if (cpu_has_guestid) + seq_puts(m, " guestid"); + if (cpu_has_drg) + seq_puts(m, " drg"); + if (cpu_has_rixi) + seq_puts(m, " rixi"); + if (cpu_has_lpa) + seq_puts(m, " lpa"); + if (cpu_has_mvh) + seq_puts(m, " mvh"); + if (cpu_has_vtag_icache) + seq_puts(m, " vtag_icache"); + if (cpu_has_dc_aliases) + seq_puts(m, " dc_aliases"); + if (cpu_has_ic_fills_f_dc) + seq_puts(m, " ic_fills_f_dc"); + if (cpu_has_pindexed_dcache) + seq_puts(m, " pindexed_dcache"); + if (cpu_has_userlocal) + seq_puts(m, " userlocal"); + if (cpu_has_nofpuex) + seq_puts(m, " nofpuex"); + if (cpu_has_vint) + seq_puts(m, " vint"); + if (cpu_has_veic) + seq_puts(m, " veic"); + if (cpu_has_inclusive_pcaches) + seq_puts(m, " inclusive_pcaches"); + if (cpu_has_perf_cntr_intr_bit) + seq_puts(m, " perf_cntr_intr_bit"); + if (cpu_has_ufr) + seq_puts(m, " ufr"); + if (cpu_has_fre) + seq_puts(m, " fre"); + if (cpu_has_cdmm) + seq_puts(m, " cdmm"); + if (cpu_has_small_pages) + seq_puts(m, " small_pages"); + if (cpu_has_nan_legacy) + seq_puts(m, " nan_legacy"); + if (cpu_has_nan_2008) + seq_puts(m, " nan_2008"); + if (cpu_has_ebase_wg) + seq_puts(m, " ebase_wg"); + if (cpu_has_badinstr) + seq_puts(m, " badinstr"); + if (cpu_has_badinstrp) + seq_puts(m, " badinstrp"); + if (cpu_has_contextconfig) + seq_puts(m, " contextconfig"); + if (cpu_has_perf) + seq_puts(m, " perf"); + if (cpu_has_mac2008_only) + seq_puts(m, " mac2008_only"); + if (cpu_has_ftlbparex) + seq_puts(m, " ftlbparex"); + if (cpu_has_gsexcex) + seq_puts(m, " gsexcex"); + if (cpu_has_shared_ftlb_ram) + seq_puts(m, " shared_ftlb_ram"); + if (cpu_has_shared_ftlb_entries) + seq_puts(m, " shared_ftlb_entries"); + if (cpu_has_mipsmt_pertccounters) + seq_puts(m, " mipsmt_pertccounters"); + if (cpu_has_mmid) + seq_puts(m, " mmid"); + if (cpu_has_mm_sysad) + seq_puts(m, " mm_sysad"); + if (cpu_has_mm_full) + seq_puts(m, " mm_full"); + seq_puts(m, "\n"); + seq_printf(m, "shadow register sets\t: %d\n", cpu_data[n].srsets); seq_printf(m, "kscratch registers\t: %d\n", @@ -163,7 +304,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) raw_notifier_call_chain(&proc_cpuinfo_chain, 0, &proc_cpuinfo_notifier_args); - seq_printf(m, "\n"); + seq_puts(m, "\n"); return 0; } diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 95aa86fa60..cbff1b974f 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -511,7 +511,7 @@ static int __init frame_info_init(void) /* * Without schedule() frame info, result given by - * thread_saved_pc() and get_wchan() are not reliable. + * thread_saved_pc() and __get_wchan() are not reliable. */ if (schedule_mfi.pc_offset < 0) printk("Can't analyze schedule() prologue at %p\n", schedule); @@ -652,9 +652,9 @@ unsigned long unwind_stack(struct task_struct *task, unsigned long *sp, #endif /* - * get_wchan - a maintenance nightmare^W^Wpain in the ass ... + * __get_wchan - a maintenance nightmare^W^Wpain in the ass ... */ -unsigned long get_wchan(struct task_struct *task) +unsigned long __get_wchan(struct task_struct *task) { unsigned long pc = 0; #ifdef CONFIG_KALLSYMS @@ -662,8 +662,6 @@ unsigned long get_wchan(struct task_struct *task) unsigned long ra = 0; #endif - if (!task || task == current || task_is_running(task)) - goto out; if (!task_stack_page(task)) goto out; diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index c9b2a75563..5bce782e69 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -562,6 +562,13 @@ void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, /* Default to using normal stack */ sp = regs->regs[29]; + /* + * If we are on the alternate signal stack and would overflow it, don't. + * Return an always-bogus address instead so we will die with SIGSEGV. + */ + if (on_sig_stack(sp) && !likely(on_sig_stack(sp - frame_size))) + return (void __user __force *)(-1UL); + /* * FPU emulator may have it's own trampoline active just * above the user stack, 16-bytes before the next lowest @@ -747,23 +754,25 @@ static int setup_rt_frame(void *sig_return, struct ksignal *ksig, struct pt_regs *regs, sigset_t *set) { struct rt_sigframe __user *frame; - int err = 0; frame = get_sigframe(ksig, regs, sizeof(*frame)); if (!access_ok(frame, sizeof (*frame))) return -EFAULT; /* Create siginfo. */ - err |= copy_siginfo_to_user(&frame->rs_info, &ksig->info); + if (copy_siginfo_to_user(&frame->rs_info, &ksig->info)) + return -EFAULT; /* Create the ucontext. */ - err |= __put_user(0, &frame->rs_uc.uc_flags); - err |= __put_user(NULL, &frame->rs_uc.uc_link); - err |= __save_altstack(&frame->rs_uc.uc_stack, regs->regs[29]); - err |= setup_sigcontext(regs, &frame->rs_uc.uc_mcontext); - err |= __copy_to_user(&frame->rs_uc.uc_sigmask, set, sizeof(*set)); - - if (err) + if (__put_user(0, &frame->rs_uc.uc_flags)) + return -EFAULT; + if (__put_user(NULL, &frame->rs_uc.uc_link)) + return -EFAULT; + if (__save_altstack(&frame->rs_uc.uc_stack, regs->regs[29])) + return -EFAULT; + if (setup_sigcontext(regs, &frame->rs_uc.uc_mcontext)) + return -EFAULT; + if (__copy_to_user(&frame->rs_uc.uc_sigmask, set, sizeof(*set))) return -EFAULT; /* diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c index b6ef5f7312..f5d7bfa347 100644 --- a/arch/mips/kernel/smp-bmips.c +++ b/arch/mips/kernel/smp-bmips.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -373,7 +374,7 @@ static int bmips_cpu_disable(void) set_cpu_online(cpu, false); calculate_cpu_foreign_map(); - irq_cpu_offline(); + irq_migrate_all_off_this_cpu(); clear_c0_status(IE_IRQ5); local_flush_tlb_all(); diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl index 70e32de2bc..253ff994ed 100644 --- a/arch/mips/kernel/syscalls/syscall_n32.tbl +++ b/arch/mips/kernel/syscalls/syscall_n32.tbl @@ -387,3 +387,5 @@ 446 n32 landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 n32 process_mrelease sys_process_mrelease +449 n32 futex_waitv sys_futex_waitv +450 n32 set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl index 1ca7bc3379..3f1886ad9d 100644 --- a/arch/mips/kernel/syscalls/syscall_n64.tbl +++ b/arch/mips/kernel/syscalls/syscall_n64.tbl @@ -363,3 +363,5 @@ 446 n64 landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 n64 process_mrelease sys_process_mrelease +449 n64 futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl index a61c35edaa..8f243e35a7 100644 --- a/arch/mips/kernel/syscalls/syscall_o32.tbl +++ b/arch/mips/kernel/syscalls/syscall_o32.tbl @@ -436,3 +436,5 @@ 446 o32 landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 o32 process_mrelease sys_process_mrelease +449 o32 futex_waitv sys_futex_waitv +450 o32 set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 6f07362de5..a486486b23 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -103,13 +103,19 @@ extern asmlinkage void handle_reserved(void); extern void tlb_do_page_fault_0(void); void (*board_be_init)(void); -int (*board_be_handler)(struct pt_regs *regs, int is_fixup); +static int (*board_be_handler)(struct pt_regs *regs, int is_fixup); void (*board_nmi_handler_setup)(void); void (*board_ejtag_handler_setup)(void); void (*board_bind_eic_interrupt)(int irq, int regset); void (*board_ebase_setup)(void); void(*board_cache_error_setup)(void); +void mips_set_be_handler(int (*handler)(struct pt_regs *regs, int is_fixup)) +{ + board_be_handler = handler; +} +EXPORT_SYMBOL_GPL(mips_set_be_handler); + static void show_raw_backtrace(unsigned long reg29, const char *loglvl, bool user) { @@ -416,7 +422,7 @@ void __noreturn die(const char *str, struct pt_regs *regs) if (regs && kexec_should_crash(current)) crash_kexec(regs); - do_exit(sig); + make_task_dead(sig); } extern struct exception_table_entry __start___dbe_table[]; diff --git a/arch/mips/kernel/uprobes.c b/arch/mips/kernel/uprobes.c index 9db2a6db5f..6c063aa188 100644 --- a/arch/mips/kernel/uprobes.c +++ b/arch/mips/kernel/uprobes.c @@ -173,6 +173,7 @@ int arch_uprobe_exception_notify(struct notifier_block *self, case DIE_UPROBE_XOL: if (uprobe_post_sstep_notifier(regs)) return NOTIFY_STOP; + break; default: break; } diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index a77297480f..91d197bee9 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -27,6 +27,7 @@ config KVM select KVM_MMIO select MMU_NOTIFIER select SRCU + select INTERVAL_TREE help Support for hosting Guest kernels. diff --git a/arch/mips/kvm/Makefile b/arch/mips/kvm/Makefile index d3710959da..21ff75bcdb 100644 --- a/arch/mips/kvm/Makefile +++ b/arch/mips/kvm/Makefile @@ -2,9 +2,10 @@ # Makefile for KVM support for MIPS # +include $(srctree)/virt/kvm/Makefile.kvm + ccflags-y += -Ivirt/kvm -Iarch/mips/kvm -kvm-y := $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o eventfd.o binary_stats.o) kvm-$(CONFIG_CPU_HAS_MSA) += msa.o kvm-y += mips.o emulate.o entry.o \ diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c index 22e745e49b..b494d8d392 100644 --- a/arch/mips/kvm/emulate.c +++ b/arch/mips/kvm/emulate.c @@ -952,7 +952,7 @@ enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu) if (!vcpu->arch.pending_exceptions) { kvm_vz_lose_htimer(vcpu); vcpu->arch.wait = 1; - kvm_vcpu_block(vcpu); + kvm_vcpu_halt(vcpu); /* * We we are runnable, then definitely go off to user space to diff --git a/arch/mips/kvm/entry.c b/arch/mips/kvm/entry.c index 8131fb2bdf..aceed14aa1 100644 --- a/arch/mips/kvm/entry.c +++ b/arch/mips/kvm/entry.c @@ -104,13 +104,7 @@ static void *kvm_mips_build_ret_to_host(void *addr); */ static int c0_kscratch(void) { - switch (boot_cpu_type()) { - case CPU_XLP: - case CPU_XLR: - return 22; - default: - return 31; - } + return 31; } /** diff --git a/arch/mips/kvm/loongson_ipi.c b/arch/mips/kvm/loongson_ipi.c index 3681fc8fba..5d53f32d83 100644 --- a/arch/mips/kvm/loongson_ipi.c +++ b/arch/mips/kvm/loongson_ipi.c @@ -120,7 +120,7 @@ static int loongson_vipi_write(struct loongson_kvm_ipi *ipi, s->status |= data; irq.cpu = id; irq.irq = 6; - kvm_vcpu_ioctl_interrupt(kvm->vcpus[id], &irq); + kvm_vcpu_ioctl_interrupt(kvm_get_vcpu(kvm, id), &irq); break; case CORE0_CLEAR_OFF: @@ -128,7 +128,7 @@ static int loongson_vipi_write(struct loongson_kvm_ipi *ipi, if (!s->status) { irq.cpu = id; irq.irq = -6; - kvm_vcpu_ioctl_interrupt(kvm->vcpus[id], &irq); + kvm_vcpu_ioctl_interrupt(kvm_get_vcpu(kvm, id), &irq); } break; diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 75c6f264c6..a25e0b73ee 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -171,25 +171,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) return 0; } -void kvm_mips_free_vcpus(struct kvm *kvm) -{ - unsigned int i; - struct kvm_vcpu *vcpu; - - kvm_for_each_vcpu(i, vcpu, kvm) { - kvm_vcpu_destroy(vcpu); - } - - mutex_lock(&kvm->lock); - - for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) - kvm->vcpus[i] = NULL; - - atomic_set(&kvm->online_vcpus, 0); - - mutex_unlock(&kvm->lock); -} - static void kvm_mips_free_gpa_pt(struct kvm *kvm) { /* It should always be safe to remove after flushing the whole range */ @@ -199,7 +180,7 @@ static void kvm_mips_free_gpa_pt(struct kvm *kvm) void kvm_arch_destroy_vm(struct kvm *kvm) { - kvm_mips_free_vcpus(kvm); + kvm_destroy_vcpus(kvm); kvm_mips_free_gpa_pt(kvm); } @@ -233,25 +214,20 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, } int kvm_arch_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, enum kvm_mr_change change) { return 0; } void kvm_arch_commit_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { int needs_flush; - kvm_debug("%s: kvm: %p slot: %d, GPA: %llx, size: %llx, QVA: %llx\n", - __func__, kvm, mem->slot, mem->guest_phys_addr, - mem->memory_size, mem->userspace_addr); - /* * If dirty page logging is enabled, write protect all pages in the slot * ready for dirty logging. @@ -438,6 +414,24 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, return -ENOIOCTLCMD; } +/* + * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while + * the vCPU is running. + * + * This must be noinstr as instrumentation may make use of RCU, and this is not + * safe during the EQS. + */ +static int noinstr kvm_mips_vcpu_enter_exit(struct kvm_vcpu *vcpu) +{ + int ret; + + guest_state_enter_irqoff(); + ret = kvm_mips_callbacks->vcpu_run(vcpu); + guest_state_exit_irqoff(); + + return ret; +} + int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { int r = -EINTR; @@ -458,7 +452,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) lose_fpu(1); local_irq_disable(); - guest_enter_irqoff(); + guest_timing_enter_irqoff(); trace_kvm_enter(vcpu); /* @@ -469,10 +463,23 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) */ smp_store_mb(vcpu->mode, IN_GUEST_MODE); - r = kvm_mips_callbacks->vcpu_run(vcpu); + r = kvm_mips_vcpu_enter_exit(vcpu); + + /* + * We must ensure that any pending interrupts are taken before + * we exit guest timing so that timer ticks are accounted as + * guest time. Transiently unmask interrupts so that any + * pending interrupts are taken. + * + * TODO: is there a barrier which ensures that pending interrupts are + * recognised? Currently this just hopes that the CPU takes any pending + * interrupts between the enable and disable. + */ + local_irq_enable(); + local_irq_disable(); trace_kvm_out(vcpu); - guest_exit_irqoff(); + guest_timing_exit_irqoff(); local_irq_enable(); out: @@ -498,7 +505,7 @@ int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, if (irq->cpu == -1) dvcpu = vcpu; else - dvcpu = vcpu->kvm->vcpus[irq->cpu]; + dvcpu = kvm_get_vcpu(vcpu->kvm, irq->cpu); if (intr == 2 || intr == 3 || intr == 4 || intr == 6) { kvm_mips_callbacks->queue_io_int(dvcpu, irq); @@ -1067,13 +1074,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = 1; break; case KVM_CAP_NR_VCPUS: - r = num_online_cpus(); + r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS); break; case KVM_CAP_MAX_VCPUS: r = KVM_MAX_VCPUS; break; case KVM_CAP_MAX_VCPU_ID: - r = KVM_MAX_VCPU_ID; + r = KVM_MAX_VCPU_IDS; break; case KVM_CAP_MIPS_FPU: /* We don't handle systems with inconsistent cpu_has_fpu */ @@ -1192,7 +1199,7 @@ static void kvm_mips_set_c0_status(void) /* * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV) */ -int kvm_mips_handle_exit(struct kvm_vcpu *vcpu) +static int __kvm_mips_handle_exit(struct kvm_vcpu *vcpu) { struct kvm_run *run = vcpu->run; u32 cause = vcpu->arch.host_cp0_cause; @@ -1381,6 +1388,17 @@ int kvm_mips_handle_exit(struct kvm_vcpu *vcpu) return ret; } +int noinstr kvm_mips_handle_exit(struct kvm_vcpu *vcpu) +{ + int ret; + + guest_state_exit_irqoff(); + ret = __kvm_mips_handle_exit(vcpu); + guest_state_enter_irqoff(); + + return ret; +} + /* Enable FPU for guest and restore context */ void kvm_own_fpu(struct kvm_vcpu *vcpu) { diff --git a/arch/mips/kvm/vz.c b/arch/mips/kvm/vz.c index 4adca5abbc..c706f5890a 100644 --- a/arch/mips/kvm/vz.c +++ b/arch/mips/kvm/vz.c @@ -458,8 +458,8 @@ void kvm_vz_acquire_htimer(struct kvm_vcpu *vcpu) /** * _kvm_vz_save_htimer() - Switch to software emulation of guest timer. * @vcpu: Virtual CPU. - * @compare: Pointer to write compare value to. - * @cause: Pointer to write cause value to. + * @out_compare: Pointer to write compare value to. + * @out_cause: Pointer to write cause value to. * * Save VZ guest timer state and switch to software emulation of guest CP0 * timer. The hard timer must already be in use, so preemption should be @@ -1541,11 +1541,14 @@ static int kvm_trap_vz_handle_guest_exit(struct kvm_vcpu *vcpu) } /** - * kvm_trap_vz_handle_cop_unusuable() - Guest used unusable coprocessor. + * kvm_trap_vz_handle_cop_unusable() - Guest used unusable coprocessor. * @vcpu: Virtual CPU context. * * Handle when the guest attempts to use a coprocessor which hasn't been allowed * by the root context. + * + * Return: value indicating whether to resume the host or the guest + * (RESUME_HOST or RESUME_GUEST) */ static int kvm_trap_vz_handle_cop_unusable(struct kvm_vcpu *vcpu) { @@ -1592,6 +1595,9 @@ static int kvm_trap_vz_handle_cop_unusable(struct kvm_vcpu *vcpu) * * Handle when the guest attempts to use MSA when it is disabled in the root * context. + * + * Return: value indicating whether to resume the host or the guest + * (RESUME_HOST or RESUME_GUEST) */ static int kvm_trap_vz_handle_msa_disabled(struct kvm_vcpu *vcpu) { diff --git a/arch/mips/lantiq/falcon/sysctrl.c b/arch/mips/lantiq/falcon/sysctrl.c index 42222f849b..64726c670c 100644 --- a/arch/mips/lantiq/falcon/sysctrl.c +++ b/arch/mips/lantiq/falcon/sysctrl.c @@ -141,7 +141,7 @@ static void falcon_gpe_enable(void) unsigned int freq; unsigned int status; - /* if if the clock is already enabled */ + /* if the clock is already enabled */ status = sysctl_r32(SYSCTL_SYS1, SYS1_INFRAC); if (status & (1 << (GPPC_OFFSET + 1))) return; diff --git a/arch/mips/lantiq/xway/dma.c b/arch/mips/lantiq/xway/dma.c index 53fcc672a2..f8eedeb15f 100644 --- a/arch/mips/lantiq/xway/dma.c +++ b/arch/mips/lantiq/xway/dma.c @@ -182,7 +182,7 @@ ltq_dma_free(struct ltq_dma_channel *ch) EXPORT_SYMBOL_GPL(ltq_dma_free); void -ltq_dma_init_port(int p) +ltq_dma_init_port(int p, int tx_burst, int rx_burst) { ltq_dma_w32(p, LTQ_DMA_PS); switch (p) { @@ -191,16 +191,44 @@ ltq_dma_init_port(int p) * Tell the DMA engine to swap the endianness of data frames and * drop packets if the channel arbitration fails. */ - ltq_dma_w32_mask(0, DMA_ETOP_ENDIANNESS | DMA_PDEN, + ltq_dma_w32_mask(0, (DMA_ETOP_ENDIANNESS | DMA_PDEN), LTQ_DMA_PCTRL); break; - case DMA_PORT_DEU: - ltq_dma_w32((DMA_PCTRL_2W_BURST << DMA_TX_BURST_SHIFT) | - (DMA_PCTRL_2W_BURST << DMA_RX_BURST_SHIFT), + default: + break; + } + + switch (rx_burst) { + case 8: + ltq_dma_w32_mask(0x0c, (DMA_PCTRL_8W_BURST << DMA_RX_BURST_SHIFT), LTQ_DMA_PCTRL); break; + case 4: + ltq_dma_w32_mask(0x0c, (DMA_PCTRL_4W_BURST << DMA_RX_BURST_SHIFT), + LTQ_DMA_PCTRL); + break; + case 2: + ltq_dma_w32_mask(0x0c, (DMA_PCTRL_2W_BURST << DMA_RX_BURST_SHIFT), + LTQ_DMA_PCTRL); + break; + default: + break; + } + switch (tx_burst) { + case 8: + ltq_dma_w32_mask(0x30, (DMA_PCTRL_8W_BURST << DMA_TX_BURST_SHIFT), + LTQ_DMA_PCTRL); + break; + case 4: + ltq_dma_w32_mask(0x30, (DMA_PCTRL_4W_BURST << DMA_TX_BURST_SHIFT), + LTQ_DMA_PCTRL); + break; + case 2: + ltq_dma_w32_mask(0x30, (DMA_PCTRL_2W_BURST << DMA_TX_BURST_SHIFT), + LTQ_DMA_PCTRL); + break; default: break; } diff --git a/arch/mips/loongson2ef/Platform b/arch/mips/loongson2ef/Platform index ae023b9a1c..50e659aca5 100644 --- a/arch/mips/loongson2ef/Platform +++ b/arch/mips/loongson2ef/Platform @@ -2,12 +2,9 @@ # Loongson Processors' Support # -# Only gcc >= 4.4 have Loongson specific support cflags-$(CONFIG_CPU_LOONGSON2EF) += -Wa,--trap -cflags-$(CONFIG_CPU_LOONGSON2E) += \ - $(call cc-option,-march=loongson2e,-march=r4600) -cflags-$(CONFIG_CPU_LOONGSON2F) += \ - $(call cc-option,-march=loongson2f,-march=r4600) +cflags-$(CONFIG_CPU_LOONGSON2E) += -march=loongson2e +cflags-$(CONFIG_CPU_LOONGSON2F) += -march=loongson2f # # Some versions of binutils, not currently mainline as of 2019/02/04, support # an -mfix-loongson3-llsc flag which emits a sync prior to each ll instruction @@ -32,16 +29,8 @@ cflags-$(CONFIG_CPU_LOONGSON2EF) += $(call as-option,-Wa$(comma)-mno-fix-loongso # Enable the workarounds for Loongson2f ifdef CONFIG_CPU_LOONGSON2F_WORKAROUNDS - ifeq ($(call as-option,-Wa$(comma)-mfix-loongson2f-nop,),) - $(error only binutils >= 2.20.2 have needed option -mfix-loongson2f-nop) - else - cflags-$(CONFIG_CPU_NOP_WORKAROUNDS) += -Wa$(comma)-mfix-loongson2f-nop - endif - ifeq ($(call as-option,-Wa$(comma)-mfix-loongson2f-jump,),) - $(error only binutils >= 2.20.2 have needed option -mfix-loongson2f-jump) - else - cflags-$(CONFIG_CPU_JUMP_WORKAROUNDS) += -Wa$(comma)-mfix-loongson2f-jump - endif +cflags-$(CONFIG_CPU_NOP_WORKAROUNDS) += -Wa,-mfix-loongson2f-nop +cflags-$(CONFIG_CPU_JUMP_WORKAROUNDS) += -Wa,-mfix-loongson2f-jump endif # Some -march= flags enable MMI instructions, and GCC complains about that diff --git a/arch/mips/loongson64/init.c b/arch/mips/loongson64/init.c index 76e0a9636a..ee8de1735b 100644 --- a/arch/mips/loongson64/init.c +++ b/arch/mips/loongson64/init.c @@ -77,7 +77,9 @@ void __init szmem(unsigned int node) (u32)node_id, mem_type, mem_start, mem_size); pr_info(" start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n", start_pfn, end_pfn, num_physpages); - memblock_add_node(PFN_PHYS(start_pfn), PFN_PHYS(node_psize), node); + memblock_add_node(PFN_PHYS(start_pfn), + PFN_PHYS(node_psize), node, + MEMBLOCK_NONE); break; case SYSTEM_RAM_RESERVED: pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx MB\n", @@ -177,6 +179,7 @@ static __init void reserve_pio_range(void) if (of_range_parser_init(&parser, np)) { pr_info("Failed to parse resources.\n"); + of_node_put(np); break; } diff --git a/arch/mips/loongson64/smp.c b/arch/mips/loongson64/smp.c index 09ebe84a17..660e1de441 100644 --- a/arch/mips/loongson64/smp.c +++ b/arch/mips/loongson64/smp.c @@ -550,7 +550,6 @@ static int loongson3_cpu_disable(void) set_cpu_online(cpu, false); calculate_cpu_foreign_map(); local_irq_save(flags); - irq_cpu_offline(); clear_c0_status(ST0_IM); local_irq_restore(flags); local_flush_tlb_all(); diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c index ec2ae50153..737870d8fd 100644 --- a/arch/mips/mm/c-octeon.c +++ b/arch/mips/mm/c-octeon.c @@ -332,7 +332,7 @@ static void co_cache_error_call_notifiers(unsigned long val) } /* - * Called when the the exception is recoverable + * Called when the exception is recoverable */ asmlinkage void cache_parity_error_octeon_recoverable(void) @@ -341,7 +341,7 @@ asmlinkage void cache_parity_error_octeon_recoverable(void) } /* - * Called when the the exception is not recoverable + * Called when the exception is not recoverable */ asmlinkage void cache_parity_error_octeon_non_recoverable(void) diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 74b09e801c..50261fd8eb 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1410,7 +1410,6 @@ static void probe_pcache(void) case CPU_I6500: case CPU_SB1: case CPU_SB1A: - case CPU_XLR: c->dcache.flags |= MIPS_CACHE_PINDEX; break; @@ -1699,7 +1698,6 @@ static void setup_scache(void) return; case CPU_CAVIUM_OCTEON3: - case CPU_XLP: /* don't need to worry about L2, fully coherent */ return; diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index e7abda9c01..44f98100e8 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -171,18 +171,17 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, goto do_sigbus; BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; - /* - * No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - goto retry; - } + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; } mmap_read_unlock(mm); diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 19347dc6bb..5a80028395 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -519,17 +519,9 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) return node_distance(cpu_to_node(from), cpu_to_node(to)); } -static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, - size_t align) +static int __init pcpu_cpu_to_node(int cpu) { - return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS), - MEMBLOCK_ALLOC_ACCESSIBLE, - cpu_to_node(cpu)); -} - -static void __init pcpu_fc_free(void *ptr, size_t size) -{ - memblock_free_early(__pa(ptr), size); + return cpu_to_node(cpu); } void __init setup_per_cpu_areas(void) @@ -545,7 +537,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, pcpu_cpu_distance, - pcpu_fc_alloc, pcpu_fc_free); + pcpu_cpu_to_node); if (rc < 0) panic("Failed to initialize percpu areas."); diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 9adad24c2e..b131e6a773 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -325,13 +325,7 @@ static unsigned int kscratch_used_mask; static inline int __maybe_unused c0_kscratch(void) { - switch (current_cpu_type()) { - case CPU_XLP: - case CPU_XLR: - return 22; - default: - return 31; - } + return 31; } static int allocate_kscratch(void) @@ -553,7 +547,6 @@ void build_tlb_write_entry(u32 **p, struct uasm_label **l, case CPU_5KC: case CPU_TX49XX: case CPU_PR4450: - case CPU_XLR: uasm_i_nop(p); tlbw(p); break; diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c index 7154a1d99a..e15c6700cd 100644 --- a/arch/mips/mm/uasm-mips.c +++ b/arch/mips/mm/uasm-mips.c @@ -90,7 +90,7 @@ static const struct insn insn_table[insn_invalid] = { RS | RT | RD}, [insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, [insn_dmultu] = {M(spec_op, 0, 0, 0, 0, dmultu_op), RS | RT}, - [insn_dmulu] = {M(spec_op, 0, 0, 0, dmult_dmul_op, dmultu_op), + [insn_dmulu] = {M(spec_op, 0, 0, 0, dmultu_dmulu_op, dmultu_op), RS | RT | RD}, [insn_drotr] = {M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE}, [insn_drotr32] = {M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE}, @@ -150,6 +150,8 @@ static const struct insn insn_table[insn_invalid] = { [insn_mtlo] = {M(spec_op, 0, 0, 0, 0, mtlo_op), RS}, [insn_mulu] = {M(spec_op, 0, 0, 0, multu_mulu_op, multu_op), RS | RT | RD}, + [insn_muhu] = {M(spec_op, 0, 0, 0, multu_muhu_op, multu_op), + RS | RT | RD}, #ifndef CONFIG_CPU_MIPSR6 [insn_mul] = {M(spec2_op, 0, 0, 0, 0, mul_op), RS | RT | RD}, #else diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c index 81dd226d6b..125140979d 100644 --- a/arch/mips/mm/uasm.c +++ b/arch/mips/mm/uasm.c @@ -59,7 +59,7 @@ enum opcode { insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld, insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi, insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0, - insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_nor, + insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor, insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc, insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, @@ -344,6 +344,7 @@ I_u1(_mtlo) I_u3u1u2(_mul) I_u1u2(_multu) I_u3u1u2(_mulu) +I_u3u1u2(_muhu) I_u3u1u2(_nor) I_u3u1u2(_or) I_u2u1u3(_ori) diff --git a/arch/mips/net/Makefile b/arch/mips/net/Makefile index d559123490..e3e6ae6514 100644 --- a/arch/mips/net/Makefile +++ b/arch/mips/net/Makefile @@ -1,5 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only # MIPS networking code -obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o -obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o +obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o + +ifeq ($(CONFIG_32BIT),y) + obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o +else + obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o +endif diff --git a/arch/mips/pci/Makefile b/arch/mips/pci/Makefile index f3eecc065e..9a6bc70260 100644 --- a/arch/mips/pci/Makefile +++ b/arch/mips/pci/Makefile @@ -49,17 +49,12 @@ obj-$(CONFIG_TANBAC_TB0287) += fixup-tb0287.o obj-$(CONFIG_TOSHIBA_JMR3927) += fixup-jmr3927.o obj-$(CONFIG_SOC_TX4927) += pci-tx4927.o obj-$(CONFIG_SOC_TX4938) += pci-tx4938.o -obj-$(CONFIG_SOC_TX4939) += pci-tx4939.o obj-$(CONFIG_TOSHIBA_RBTX4927) += fixup-rbtx4927.o -obj-$(CONFIG_TOSHIBA_RBTX4938) += fixup-rbtx4938.o obj-$(CONFIG_VICTOR_MPC30X) += fixup-mpc30x.o obj-$(CONFIG_ZAO_CAPCELLA) += fixup-capcella.o obj-$(CONFIG_MIKROTIK_RB532) += pci-rc32434.o ops-rc32434.o fixup-rc32434.o obj-$(CONFIG_CAVIUM_OCTEON_SOC) += pci-octeon.o pcie-octeon.o -obj-$(CONFIG_CPU_XLR) += pci-xlr.o -obj-$(CONFIG_CPU_XLP) += pci-xlp.o ifdef CONFIG_PCI_MSI obj-$(CONFIG_CAVIUM_OCTEON_SOC) += msi-octeon.o -obj-$(CONFIG_CPU_XLP) += msi-xlp.o endif diff --git a/arch/mips/pci/fixup-cobalt.c b/arch/mips/pci/fixup-cobalt.c index 44be65c3e6..00206ff529 100644 --- a/arch/mips/pci/fixup-cobalt.c +++ b/arch/mips/pci/fixup-cobalt.c @@ -36,6 +36,21 @@ #define VIA_COBALT_BRD_ID_REG 0x94 #define VIA_COBALT_BRD_REG_to_ID(reg) ((unsigned char)(reg) >> 4) +/* + * Default value of PCI Class Code on GT64111 is PCI_CLASS_MEMORY_OTHER (0x0580) + * instead of PCI_CLASS_BRIDGE_HOST (0x0600). Galileo explained this choice in + * document "GT-64111 System Controller for RC4640, RM523X and VR4300 CPUs", + * section "6.5.3 PCI Autoconfiguration at RESET": + * + * Some PCs refuse to configure host bridges if they are found plugged into + * a PCI slot (ask the BIOS vendors why...). The "Memory Controller" Class + * Code does not cause a problem for these non-compliant BIOSes, so we used + * this as the default in the GT-64111. + * + * So fix the incorrect default value of PCI Class Code. More details are on: + * https://lore.kernel.org/r/20211102154831.xtrlgrmrizl5eidl@pali/ + * https://lore.kernel.org/r/20211102150201.GA11675@alpha.franken.de/ + */ static void qube_raq_galileo_early_fixup(struct pci_dev *dev) { if (dev->devfn == PCI_DEVFN(0, 0) && diff --git a/arch/mips/pci/msi-octeon.c b/arch/mips/pci/msi-octeon.c index 288b58b00d..c2860ebbd8 100644 --- a/arch/mips/pci/msi-octeon.c +++ b/arch/mips/pci/msi-octeon.c @@ -68,6 +68,9 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) u64 search_mask; int index; + if (desc->pci.msi_attrib.is_msix) + return -EINVAL; + /* * Read the MSI config to figure out how many IRQs this device * wants. Most devices only want 1, which will give @@ -182,35 +185,6 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) return 0; } -int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - struct msi_desc *entry; - int ret; - - /* - * MSI-X is not supported. - */ - if (type == PCI_CAP_ID_MSIX) - return -EINVAL; - - /* - * If an architecture wants to support multiple MSI, it needs to - * override arch_setup_msi_irqs() - */ - if (type == PCI_CAP_ID_MSI && nvec > 1) - return 1; - - for_each_pci_msi_entry(entry, dev) { - ret = arch_setup_msi_irq(dev, entry); - if (ret < 0) - return ret; - if (ret > 0) - return -ENOSPC; - } - - return 0; -} - /** * Called when a device no longer needs its MSI interrupts. All * MSI interrupts for the device are freed. diff --git a/arch/mips/pci/pci-bcm47xx.c b/arch/mips/pci/pci-bcm47xx.c index 230d7dd273..ffac06a5ca 100644 --- a/arch/mips/pci/pci-bcm47xx.c +++ b/arch/mips/pci/pci-bcm47xx.c @@ -41,8 +41,7 @@ static int bcm47xx_pcibios_plat_dev_init_ssb(struct pci_dev *dev) res = ssb_pcibios_plat_dev_init(dev); if (res < 0) { - printk(KERN_ALERT "PCI: Failed to init device %s\n", - pci_name(dev)); + pci_alert(dev, "PCI: Failed to init device\n"); return res; } @@ -52,8 +51,7 @@ static int bcm47xx_pcibios_plat_dev_init_ssb(struct pci_dev *dev) /* IRQ-0 and IRQ-1 are software interrupts. */ if (res < 2) { - printk(KERN_ALERT "PCI: Failed to map IRQ of device %s\n", - pci_name(dev)); + pci_alert(dev, "PCI: Failed to map IRQ of device\n"); return res; } @@ -69,8 +67,7 @@ static int bcm47xx_pcibios_plat_dev_init_bcma(struct pci_dev *dev) res = bcma_core_pci_plat_dev_init(dev); if (res < 0) { - printk(KERN_ALERT "PCI: Failed to init device %s\n", - pci_name(dev)); + pci_alert(dev, "PCI: Failed to init device\n"); return res; } @@ -78,8 +75,7 @@ static int bcm47xx_pcibios_plat_dev_init_bcma(struct pci_dev *dev) /* IRQ-0 and IRQ-1 are software interrupts. */ if (res < 2) { - printk(KERN_ALERT "PCI: Failed to map IRQ of device %s\n", - pci_name(dev)); + pci_alert(dev, "PCI: Failed to map IRQ of device\n"); return res; } @@ -93,12 +89,10 @@ int pcibios_plat_dev_init(struct pci_dev *dev) #ifdef CONFIG_BCM47XX_SSB if (bcm47xx_bus_type == BCM47XX_BUS_TYPE_SSB) return bcm47xx_pcibios_plat_dev_init_ssb(dev); - else #endif #ifdef CONFIG_BCM47XX_BCMA if (bcm47xx_bus_type == BCM47XX_BUS_TYPE_BCMA) return bcm47xx_pcibios_plat_dev_init_bcma(dev); - else #endif - return 0; + return 0; } diff --git a/arch/mips/pci/pci-generic.c b/arch/mips/pci/pci-generic.c index 95b0001788..d2d68bac3d 100644 --- a/arch/mips/pci/pci-generic.c +++ b/arch/mips/pci/pci-generic.c @@ -46,3 +46,19 @@ void pcibios_fixup_bus(struct pci_bus *bus) { pci_read_bridge_bases(bus); } + +#ifdef pci_remap_iospace +int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) +{ + unsigned long vaddr; + + if (res->start != 0) { + WARN_ONCE(1, "resource start address is not zero\n"); + return -ENODEV; + } + + vaddr = (unsigned long)ioremap(phys_addr, resource_size(res)); + set_io_port_base(vaddr); + return 0; +} +#endif diff --git a/arch/mips/pci/pci-rt3883.c b/arch/mips/pci/pci-rt3883.c index d3c947fa29..e07ae098bd 100644 --- a/arch/mips/pci/pci-rt3883.c +++ b/arch/mips/pci/pci-rt3883.c @@ -102,14 +102,12 @@ static u32 rt3883_pci_read_cfg32(struct rt3883_pci_controller *rpc, unsigned func, unsigned reg) { u32 address; - u32 ret; address = rt3883_pci_get_cfgaddr(bus, slot, func, reg); rt3883_pci_w32(rpc, address, RT3883_PCI_REG_CFGADDR); - ret = rt3883_pci_r32(rpc, RT3883_PCI_REG_CFGDATA); - return ret; + return rt3883_pci_r32(rpc, RT3883_PCI_REG_CFGDATA); } static void rt3883_pci_write_cfg32(struct rt3883_pci_controller *rpc, diff --git a/arch/mips/ralink/Kconfig b/arch/mips/ralink/Kconfig index c800bf5559..120adad51d 100644 --- a/arch/mips/ralink/Kconfig +++ b/arch/mips/ralink/Kconfig @@ -51,7 +51,8 @@ choice select SYS_SUPPORTS_HIGHMEM select MIPS_GIC select CLKSRC_MIPS_GIC - select HAVE_PCI if PCI_MT7621 + select HAVE_PCI + select PCI_DRIVERS_GENERIC select SOC_BUS endchoice diff --git a/arch/mips/ralink/ill_acc.c b/arch/mips/ralink/ill_acc.c index bdf53807d7..115a69fc20 100644 --- a/arch/mips/ralink/ill_acc.c +++ b/arch/mips/ralink/ill_acc.c @@ -65,6 +65,7 @@ static int __init ill_acc_of_setup(void) } irq = irq_of_parse_and_map(np, 0); + of_node_put(np); if (!irq) { dev_err(&pdev->dev, "failed to get irq\n"); put_device(&pdev->dev); diff --git a/arch/mips/ralink/mt7621.c b/arch/mips/ralink/mt7621.c index 4c83786612..fb0565bc34 100644 --- a/arch/mips/ralink/mt7621.c +++ b/arch/mips/ralink/mt7621.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include @@ -24,6 +26,35 @@ static u32 detect_magic __initdata; +int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge) +{ + struct resource_entry *entry; + resource_size_t mask; + + entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); + if (!entry) { + pr_err("Cannot get memory resource\n"); + return -EINVAL; + } + + if (mips_cps_numiocu(0)) { + /* + * Hardware doesn't accept mask values with 1s after + * 0s (e.g. 0xffef), so warn if that's happen + */ + mask = ~(entry->res->end - entry->res->start) & CM_GCR_REGn_MASK_ADDRMASK; + WARN_ON(mask && BIT(ffz(~mask)) - 1 != ~mask); + + write_gcr_reg1_base(entry->res->start); + write_gcr_reg1_mask(mask | CM_GCR_REGn_MASK_CMTGT_IOCU0); + pr_info("PCI coherence region base: 0x%08llx, mask/settings: 0x%08llx\n", + (unsigned long long)read_gcr_reg1_base(), + (unsigned long long)read_gcr_reg1_mask()); + } + + return 0; +} + phys_addr_t mips_cpc_default_phys_base(void) { panic("Cannot detect cpc address"); diff --git a/arch/mips/ralink/of.c b/arch/mips/ralink/of.c index 0135376c5d..35a87a2da1 100644 --- a/arch/mips/ralink/of.c +++ b/arch/mips/ralink/of.c @@ -53,17 +53,6 @@ void __init device_tree_init(void) unflatten_and_copy_device_tree(); } -static int memory_dtb; - -static int __init early_init_dt_find_memory(unsigned long node, - const char *uname, int depth, void *data) -{ - if (depth == 1 && !strcmp(uname, "memory@0")) - memory_dtb = 1; - - return 0; -} - void __init plat_mem_setup(void) { void *dtb; @@ -77,10 +66,10 @@ void __init plat_mem_setup(void) dtb = get_fdt(); __dt_setup_arch(dtb); - of_scan_flat_dt(early_init_dt_find_memory, NULL); - if (memory_dtb) - of_scan_flat_dt(early_init_dt_scan_memory, NULL); - else if (soc_info.mem_detect) + if (!early_init_dt_scan_memory()) + return; + + if (soc_info.mem_detect) soc_info.mem_detect(); else if (soc_info.mem_size) memblock_add(soc_info.mem_base, soc_info.mem_size * SZ_1M); diff --git a/arch/mips/rb532/prom.c b/arch/mips/rb532/prom.c index 23ad8dd9aa..b116937155 100644 --- a/arch/mips/rb532/prom.c +++ b/arch/mips/rb532/prom.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include diff --git a/arch/mips/sgi-ip22/Platform b/arch/mips/sgi-ip22/Platform index 62fa30bb95..a4c46e3356 100644 --- a/arch/mips/sgi-ip22/Platform +++ b/arch/mips/sgi-ip22/Platform @@ -23,10 +23,5 @@ endif # be 16kb aligned or the handling of the current variable will break. # Simplified: what IP22 does at 128MB+ in ksegN, IP28 does at 512MB+ in xkphys # -ifdef CONFIG_SGI_IP28 - ifeq ($(call cc-option-yn,-march=r10000 -mr10k-cache-barrier=store), n) - $(error gcc doesn't support needed option -mr10k-cache-barrier=store) - endif -endif cflags-$(CONFIG_SGI_IP28) += -mr10k-cache-barrier=store -I$(srctree)/arch/mips/include/asm/mach-ip28 load-$(CONFIG_SGI_IP28) += 0xa800000020004000 diff --git a/arch/mips/sgi-ip22/ip22-berr.c b/arch/mips/sgi-ip22/ip22-berr.c index dc0110a607..afe8a61078 100644 --- a/arch/mips/sgi-ip22/ip22-berr.c +++ b/arch/mips/sgi-ip22/ip22-berr.c @@ -112,5 +112,5 @@ static int ip22_be_handler(struct pt_regs *regs, int is_fixup) void __init ip22_be_init(void) { - board_be_handler = ip22_be_handler; + mips_set_be_handler(ip22_be_handler); } diff --git a/arch/mips/sgi-ip22/ip28-berr.c b/arch/mips/sgi-ip22/ip28-berr.c index c61362d9ea..16ca470deb 100644 --- a/arch/mips/sgi-ip22/ip28-berr.c +++ b/arch/mips/sgi-ip22/ip28-berr.c @@ -468,7 +468,7 @@ static int ip28_be_handler(struct pt_regs *regs, int is_fixup) void __init ip22_be_init(void) { - board_be_handler = ip28_be_handler; + mips_set_be_handler(ip28_be_handler); } int ip28_show_be_info(struct seq_file *m) diff --git a/arch/mips/sgi-ip27/ip27-berr.c b/arch/mips/sgi-ip27/ip27-berr.c index 5a38ae6bdf..923a63a51c 100644 --- a/arch/mips/sgi-ip27/ip27-berr.c +++ b/arch/mips/sgi-ip27/ip27-berr.c @@ -85,7 +85,7 @@ void __init ip27_be_init(void) int cpu = LOCAL_HUB_L(PI_CPU_NUM); int cpuoff = cpu << 8; - board_be_handler = ip27_be_handler; + mips_set_be_handler(ip27_be_handler); LOCAL_HUB_S(PI_ERR_INT_PEND, cpu ? PI_ERR_CLEAR_ALL_B : PI_ERR_CLEAR_ALL_A); diff --git a/arch/mips/sgi-ip27/ip27-memory.c b/arch/mips/sgi-ip27/ip27-memory.c index 6173684b5a..adc2faeecf 100644 --- a/arch/mips/sgi-ip27/ip27-memory.c +++ b/arch/mips/sgi-ip27/ip27-memory.c @@ -341,7 +341,8 @@ static void __init szmem(void) continue; } memblock_add_node(PFN_PHYS(slot_getbasepfn(node, slot)), - PFN_PHYS(slot_psize), node); + PFN_PHYS(slot_psize), node, + MEMBLOCK_NONE); } } } diff --git a/arch/mips/sgi-ip30/ip30-setup.c b/arch/mips/sgi-ip30/ip30-setup.c index 44b1607e96..75a34684e7 100644 --- a/arch/mips/sgi-ip30/ip30-setup.c +++ b/arch/mips/sgi-ip30/ip30-setup.c @@ -69,10 +69,10 @@ static void __init ip30_mem_init(void) total_mem += size; if (addr >= IP30_REAL_MEMORY_START) - memblock_free(addr, size); + memblock_phys_free(addr, size); else if ((addr + size) > IP30_REAL_MEMORY_START) - memblock_free(IP30_REAL_MEMORY_START, - size - IP30_MAX_PROM_MEMORY); + memblock_phys_free(IP30_REAL_MEMORY_START, + size - IP30_MAX_PROM_MEMORY); } pr_info("Detected %luMB of physical memory.\n", MEM_SHIFT(total_mem)); } diff --git a/arch/mips/sgi-ip32/ip32-berr.c b/arch/mips/sgi-ip32/ip32-berr.c index c860f95ab7..478b63b4c8 100644 --- a/arch/mips/sgi-ip32/ip32-berr.c +++ b/arch/mips/sgi-ip32/ip32-berr.c @@ -34,5 +34,5 @@ static int ip32_be_handler(struct pt_regs *regs, int is_fixup) void __init ip32_be_init(void) { - board_be_handler = ip32_be_handler; + mips_set_be_handler(ip32_be_handler); } diff --git a/arch/mips/sibyte/common/cfe.c b/arch/mips/sibyte/common/cfe.c index a3323f8dcc..1a504294d8 100644 --- a/arch/mips/sibyte/common/cfe.c +++ b/arch/mips/sibyte/common/cfe.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/mips/sibyte/swarm/setup.c b/arch/mips/sibyte/swarm/setup.c index 538a2791b4..72a31eeeeb 100644 --- a/arch/mips/sibyte/swarm/setup.c +++ b/arch/mips/sibyte/swarm/setup.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -123,7 +122,7 @@ void __init plat_mem_setup(void) #error invalid SiByte board configuration #endif - board_be_handler = swarm_be_handler; + mips_set_be_handler(swarm_be_handler); if (xicor_probe()) swarm_rtc_type = RTC_XICOR; diff --git a/arch/mips/txx9/Kconfig b/arch/mips/txx9/Kconfig index 85c4c121c7..6c61feee6d 100644 --- a/arch/mips/txx9/Kconfig +++ b/arch/mips/txx9/Kconfig @@ -6,6 +6,7 @@ config MACH_TX39XX config MACH_TX49XX bool + select BOOT_ELF32 select MACH_TXX9 select CEVT_R4K select CSRC_R4K @@ -38,23 +39,6 @@ config TOSHIBA_RBTX4927 This Toshiba board is based on the TX4927 processor. Say Y here to support this machine type -config TOSHIBA_RBTX4938 - bool "Toshiba RBTX4938 board" - depends on MACH_TX49XX - select SOC_TX4938 - help - This Toshiba board is based on the TX4938 processor. Say Y here to - support this machine type - -config TOSHIBA_RBTX4939 - bool "Toshiba RBTX4939 board" - depends on MACH_TX49XX - select SOC_TX4939 - select TXX9_7SEGLED - help - This Toshiba board is based on the TX4939 processor. Say Y here to - support this machine type - config SOC_TX3927 bool select CEVT_TXX9 @@ -71,7 +55,6 @@ config SOC_TX4927 select IRQ_TXX9 select PCI_TX4927 select GPIO_TXX9 - imply HAS_TXX9_ACLC config SOC_TX4938 bool @@ -81,18 +64,6 @@ config SOC_TX4938 select IRQ_TXX9 select PCI_TX4927 select GPIO_TXX9 - imply HAS_TXX9_ACLC - -config SOC_TX4939 - bool - select CEVT_TXX9 - imply HAS_TXX9_SERIAL - select HAVE_PCI - select PCI_TX4927 - imply HAS_TXX9_ACLC - -config TXX9_7SEGLED - bool config TOSHIBA_FPCIB0 bool "FPCIB0 Backplane Support" @@ -104,25 +75,5 @@ config PICMG_PCI_BACKPLANE_DEFAULT depends on PCI && MACH_TXX9 default y if !TOSHIBA_FPCIB0 -if TOSHIBA_RBTX4938 - -comment "Multiplex Pin Select" -choice - prompt "PIO[58:61]" - default TOSHIBA_RBTX4938_MPLEX_PIO58_61 - -config TOSHIBA_RBTX4938_MPLEX_PIO58_61 - bool "PIO" -config TOSHIBA_RBTX4938_MPLEX_NAND - bool "NAND" -config TOSHIBA_RBTX4938_MPLEX_ATA - bool "ATA" -config TOSHIBA_RBTX4938_MPLEX_KEEP - bool "Keep firmware settings" - -endchoice - -endif - config PCI_TX4927 bool diff --git a/arch/mips/txx9/Makefile b/arch/mips/txx9/Makefile index 1952959372..53269910a4 100644 --- a/arch/mips/txx9/Makefile +++ b/arch/mips/txx9/Makefile @@ -14,5 +14,3 @@ obj-$(CONFIG_TOSHIBA_JMR3927) += jmr3927/ # Toshiba RBTX49XX boards # obj-$(CONFIG_TOSHIBA_RBTX4927) += rbtx4927/ -obj-$(CONFIG_TOSHIBA_RBTX4938) += rbtx4938/ -obj-$(CONFIG_TOSHIBA_RBTX4939) += rbtx4939/ diff --git a/arch/mips/txx9/generic/Makefile b/arch/mips/txx9/generic/Makefile index 6d00580fc8..be5af9fe7c 100644 --- a/arch/mips/txx9/generic/Makefile +++ b/arch/mips/txx9/generic/Makefile @@ -8,7 +8,4 @@ obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_SOC_TX3927) += setup_tx3927.o irq_tx3927.o obj-$(CONFIG_SOC_TX4927) += mem_tx4927.o setup_tx4927.o irq_tx4927.o obj-$(CONFIG_SOC_TX4938) += mem_tx4927.o setup_tx4938.o irq_tx4938.o -obj-$(CONFIG_SOC_TX4939) += setup_tx4939.o irq_tx4939.o obj-$(CONFIG_TOSHIBA_FPCIB0) += smsc_fdc37m81x.o -obj-$(CONFIG_SPI) += spi_eeprom.o -obj-$(CONFIG_TXX9_7SEGLED) += 7segled.o diff --git a/arch/mips/txx9/generic/setup.c b/arch/mips/txx9/generic/setup.c index 42ba1e97df..39cd1edf9d 100644 --- a/arch/mips/txx9/generic/setup.c +++ b/arch/mips/txx9/generic/setup.c @@ -314,16 +314,6 @@ static void __init select_board(void) case 0x4937: txx9_board_vec = &rbtx4937_vec; break; -#endif -#ifdef CONFIG_TOSHIBA_RBTX4938 - case 0x4938: - txx9_board_vec = &rbtx4938_vec; - break; -#endif -#ifdef CONFIG_TOSHIBA_RBTX4939 - case 0x4939: - txx9_board_vec = &rbtx4939_vec; - break; #endif } #endif @@ -590,21 +580,6 @@ unsigned long (*__swizzle_addr_b)(unsigned long port) = __swizzle_addr_none; EXPORT_SYMBOL(__swizzle_addr_b); #endif -#ifdef NEEDS_TXX9_IOSWABW -static u16 ioswabw_default(volatile u16 *a, u16 x) -{ - return le16_to_cpu(x); -} -static u16 __mem_ioswabw_default(volatile u16 *a, u16 x) -{ - return x; -} -u16 (*ioswabw)(volatile u16 *a, u16 x) = ioswabw_default; -EXPORT_SYMBOL(ioswabw); -u16 (*__mem_ioswabw)(volatile u16 *a, u16 x) = __mem_ioswabw_default; -EXPORT_SYMBOL(__mem_ioswabw); -#endif - void __init txx9_physmap_flash_init(int no, unsigned long addr, unsigned long size, const struct physmap_flash_data *pdata) @@ -840,34 +815,6 @@ void __init txx9_aclc_init(unsigned long baseaddr, int irq, unsigned int dma_chan_out, unsigned int dma_chan_in) { -#if IS_ENABLED(CONFIG_SND_SOC_TXX9ACLC) - unsigned int dma_base = dmac_id * TXX9_DMA_MAX_NR_CHANNELS; - struct resource res[] = { - { - .start = baseaddr, - .end = baseaddr + 0x100 - 1, - .flags = IORESOURCE_MEM, - }, { - .start = irq, - .flags = IORESOURCE_IRQ, - }, { - .name = "txx9dmac-chan", - .start = dma_base + dma_chan_out, - .flags = IORESOURCE_DMA, - }, { - .name = "txx9dmac-chan", - .start = dma_base + dma_chan_in, - .flags = IORESOURCE_DMA, - } - }; - struct platform_device *pdev = - platform_device_alloc("txx9aclc-ac97", -1); - - if (!pdev || - platform_device_add_resources(pdev, res, ARRAY_SIZE(res)) || - platform_device_add(pdev)) - platform_device_put(pdev); -#endif } static struct bus_type txx9_sramc_subsys = { diff --git a/arch/mips/txx9/generic/setup_tx4927.c b/arch/mips/txx9/generic/setup_tx4927.c index 46e9c41013..63f9725b2e 100644 --- a/arch/mips/txx9/generic/setup_tx4927.c +++ b/arch/mips/txx9/generic/setup_tx4927.c @@ -80,7 +80,7 @@ static int tx4927_be_handler(struct pt_regs *regs, int is_fixup) } static void __init tx4927_be_init(void) { - board_be_handler = tx4927_be_handler; + mips_set_be_handler(tx4927_be_handler); } static struct resource tx4927_sdram_resource[4]; diff --git a/arch/mips/txx9/generic/setup_tx4938.c b/arch/mips/txx9/generic/setup_tx4938.c index 17395d5d15..ba646548c5 100644 --- a/arch/mips/txx9/generic/setup_tx4938.c +++ b/arch/mips/txx9/generic/setup_tx4938.c @@ -82,7 +82,7 @@ static int tx4938_be_handler(struct pt_regs *regs, int is_fixup) } static void __init tx4938_be_init(void) { - board_be_handler = tx4938_be_handler; + mips_set_be_handler(tx4938_be_handler); } static struct resource tx4938_sdram_resource[4]; diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile index 1b2ea34c3d..d65f55f67e 100644 --- a/arch/mips/vdso/Makefile +++ b/arch/mips/vdso/Makefile @@ -57,7 +57,7 @@ endif # VDSO linker flags. ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ - $(filter -E%,$(KBUILD_CFLAGS)) -nostdlib -shared \ + $(filter -E%,$(KBUILD_CFLAGS)) -shared \ -G 0 --eh-frame-hdr --hash-style=sysv --build-id=sha1 -T CFLAGS_REMOVE_vdso.o = $(CC_FLAGS_FTRACE) diff --git a/arch/nios2/Kbuild b/arch/nios2/Kbuild index a4e40e534e..4e39f7abde 100644 --- a/arch/nios2/Kbuild +++ b/arch/nios2/Kbuild @@ -1 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only + +# for cleaning +subdir- += boot diff --git a/arch/nios2/Makefile b/arch/nios2/Makefile index 52c03e60b1..02d6785590 100644 --- a/arch/nios2/Makefile +++ b/arch/nios2/Makefile @@ -8,8 +8,7 @@ # Written by Fredrik Markstrom # # This file is included by the global makefile so that you can add your own -# architecture-specific flags and dependencies. Remember to do have actions -# for "archclean" cleaning up for this architecture. +# architecture-specific flags and dependencies. # # Nios2 port by Wind River Systems Inc trough: # fredrik.markstrom@gmail.com and ivarholmqvist@gmail.com @@ -53,14 +52,12 @@ core-y += $(nios2-boot)/dts/ all: vmImage -archclean: - $(Q)$(MAKE) $(clean)=$(nios2-boot) - $(BOOT_TARGETS): vmlinux $(Q)$(MAKE) $(build)=$(nios2-boot) $(nios2-boot)/$@ install: - $(Q)$(MAKE) $(build)=$(nios2-boot) BOOTIMAGE=$(KBUILD_IMAGE) install + sh $(srctree)/$(nios2-boot)/install.sh $(KERNELRELEASE) \ + $(KBUILD_IMAGE) System.map "$(INSTALL_PATH)" define archhelp echo '* vmImage - Kernel-only image for U-Boot ($(KBUILD_IMAGE))' diff --git a/arch/nios2/boot/Makefile b/arch/nios2/boot/Makefile index 37dfc7e584..8c3ad76602 100644 --- a/arch/nios2/boot/Makefile +++ b/arch/nios2/boot/Makefile @@ -30,6 +30,3 @@ $(obj)/zImage: $(obj)/compressed/vmlinux FORCE $(obj)/compressed/vmlinux: $(obj)/vmlinux.gz FORCE $(Q)$(MAKE) $(build)=$(obj)/compressed $@ - -install: - sh $(srctree)/$(src)/install.sh $(KERNELRELEASE) $(BOOTIMAGE) System.map "$(INSTALL_PATH)" diff --git a/arch/nios2/boot/dts/Makefile b/arch/nios2/boot/dts/Makefile index a91a0b09be..e9e31bb40d 100644 --- a/arch/nios2/boot/dts/Makefile +++ b/arch/nios2/boot/dts/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y := $(patsubst "%.dts",%.dtb.o,$(CONFIG_NIOS2_DTB_SOURCE)) +obj-y := $(patsubst %.dts,%.dtb.o,$(CONFIG_NIOS2_DTB_SOURCE)) dtstree := $(srctree)/$(src) dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) diff --git a/arch/nios2/include/asm/cacheflush.h b/arch/nios2/include/asm/cacheflush.h index 18eb9f69f8..d0b71dd712 100644 --- a/arch/nios2/include/asm/cacheflush.h +++ b/arch/nios2/include/asm/cacheflush.h @@ -28,7 +28,7 @@ extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, extern void flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long pfn); #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -extern void flush_dcache_page(struct page *page); +void flush_dcache_page(struct page *page); extern void flush_icache_range(unsigned long start, unsigned long end); extern void flush_icache_page(struct vm_area_struct *vma, struct page *page); diff --git a/arch/nios2/include/asm/processor.h b/arch/nios2/include/asm/processor.h index 94bcb86f67..b8125dfbca 100644 --- a/arch/nios2/include/asm/processor.h +++ b/arch/nios2/include/asm/processor.h @@ -69,7 +69,7 @@ static inline void release_thread(struct task_struct *dead_task) { } -extern unsigned long get_wchan(struct task_struct *p); +extern unsigned long __get_wchan(struct task_struct *p); #define task_pt_regs(p) \ ((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1) diff --git a/arch/nios2/include/asm/syscall.h b/arch/nios2/include/asm/syscall.h index 526449edd7..fff52205fb 100644 --- a/arch/nios2/include/asm/syscall.h +++ b/arch/nios2/include/asm/syscall.h @@ -58,17 +58,6 @@ static inline void syscall_get_arguments(struct task_struct *task, *args = regs->r9; } -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, const unsigned long *args) -{ - regs->r4 = *args++; - regs->r5 = *args++; - regs->r6 = *args++; - regs->r7 = *args++; - regs->r8 = *args++; - regs->r9 = *args; -} - static inline int syscall_get_arch(struct task_struct *task) { return AUDIT_ARCH_NIOS2; diff --git a/arch/nios2/kernel/process.c b/arch/nios2/kernel/process.c index 9ff37ba2bb..f8ea522a15 100644 --- a/arch/nios2/kernel/process.c +++ b/arch/nios2/kernel/process.c @@ -217,15 +217,12 @@ void dump(struct pt_regs *fp) pr_emerg("\n\n"); } -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { unsigned long fp, pc; unsigned long stack_page; int count = 0; - if (!p || p == current || task_is_running(p)) - return 0; - stack_page = (unsigned long)p; fp = ((struct switch_stack *)p->thread.ksp)->fp; /* ;dgt2 */ do { diff --git a/arch/nios2/kernel/traps.c b/arch/nios2/kernel/traps.c index 596986a74a..85ac49d64c 100644 --- a/arch/nios2/kernel/traps.c +++ b/arch/nios2/kernel/traps.c @@ -37,10 +37,10 @@ void die(const char *str, struct pt_regs *regs, long err) show_regs(regs); spin_unlock_irq(&die_lock); /* - * do_exit() should take care of panic'ing from an interrupt + * make_task_dead() should take care of panic'ing from an interrupt * context so we don't handle it here */ - do_exit(err); + make_task_dead(err); } void _exception(int signo, struct pt_regs *regs, int code, unsigned long addr) diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c index 9476feecf5..a32f14cd72 100644 --- a/arch/nios2/mm/fault.c +++ b/arch/nios2/mm/fault.c @@ -149,18 +149,16 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* - * No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ - goto retry; - } + goto retry; } mmap_read_unlock(mm); diff --git a/arch/openrisc/Kbuild b/arch/openrisc/Kbuild index 4234b4c03e..b0b0f2b03f 100644 --- a/arch/openrisc/Kbuild +++ b/arch/openrisc/Kbuild @@ -1,3 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-y += lib/ kernel/ mm/ obj-y += boot/dts/ + +# for cleaning +subdir- += boot diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index e804026b47..f724b3f1ae 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -10,10 +10,10 @@ config OPENRISC select ARCH_HAS_DMA_SET_UNCACHED select ARCH_HAS_DMA_CLEAR_UNCACHED select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select COMMON_CLK select OF select OF_EARLY_FLATTREE select IRQ_DOMAIN - select HANDLE_DOMAIN_IRQ select GPIOLIB select HAVE_ARCH_TRACEHOOK select SPARSE_IRQ diff --git a/arch/openrisc/Makefile b/arch/openrisc/Makefile index c52de526e5..760b734fb8 100644 --- a/arch/openrisc/Makefile +++ b/arch/openrisc/Makefile @@ -1,9 +1,7 @@ # BK Id: %F% %I% %G% %U% %#% # # This file is included by the global makefile so that you can add your own -# architecture-specific flags and dependencies. Remember to do have actions -# for "archclean" and "archdep" for cleaning up and making dependencies for -# this architecture +# architecture-specific flags and dependencies. # # This file is subject to the terms and conditions of the GNU General Public # License. See the file "COPYING" in the main directory of this archive @@ -48,6 +46,3 @@ PHONY += vmlinux.bin vmlinux.bin: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ - -archclean: - $(Q)$(MAKE) $(clean)=$(boot) diff --git a/arch/openrisc/boot/dts/Makefile b/arch/openrisc/boot/dts/Makefile index 17dd791a83..13db5a2aab 100644 --- a/arch/openrisc/boot/dts/Makefile +++ b/arch/openrisc/boot/dts/Makefile @@ -1,9 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -ifneq '$(CONFIG_OPENRISC_BUILTIN_DTB)' '""' -BUILTIN_DTB := $(patsubst "%",%,$(CONFIG_OPENRISC_BUILTIN_DTB)).dtb.o -else -BUILTIN_DTB := -endif -obj-y += $(BUILTIN_DTB) +obj-y += $(addsuffix .dtb.o, $(CONFIG_OPENRISC_BUILTIN_DTB)) #DTC_FLAGS ?= -p 1024 diff --git a/arch/openrisc/include/asm/bitops.h b/arch/openrisc/include/asm/bitops.h index 7f1ca35213..d773ed938a 100644 --- a/arch/openrisc/include/asm/bitops.h +++ b/arch/openrisc/include/asm/bitops.h @@ -30,7 +30,6 @@ #include #include #include -#include #ifndef _LINUX_BITOPS_H #error only can be included directly diff --git a/arch/openrisc/include/asm/processor.h b/arch/openrisc/include/asm/processor.h index ad53b31848..aa1699c18a 100644 --- a/arch/openrisc/include/asm/processor.h +++ b/arch/openrisc/include/asm/processor.h @@ -73,7 +73,7 @@ struct thread_struct { void start_thread(struct pt_regs *regs, unsigned long nip, unsigned long sp); void release_thread(struct task_struct *); -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); #define cpu_relax() barrier() diff --git a/arch/openrisc/include/asm/spinlock.h b/arch/openrisc/include/asm/spinlock.h index a8940bdfcb..264944a715 100644 --- a/arch/openrisc/include/asm/spinlock.h +++ b/arch/openrisc/include/asm/spinlock.h @@ -19,9 +19,6 @@ #include -#define arch_read_lock_flags(lock, flags) arch_read_lock(lock) -#define arch_write_lock_flags(lock, flags) arch_write_lock(lock) - #define arch_spin_relax(lock) cpu_relax() #define arch_read_relax(lock) cpu_relax() #define arch_write_relax(lock) cpu_relax() diff --git a/arch/openrisc/include/asm/syscall.h b/arch/openrisc/include/asm/syscall.h index e6383be2a1..903ed882bd 100644 --- a/arch/openrisc/include/asm/syscall.h +++ b/arch/openrisc/include/asm/syscall.h @@ -57,13 +57,6 @@ syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, memcpy(args, ®s->gpr[3], 6 * sizeof(args[0])); } -static inline void -syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, - const unsigned long *args) -{ - memcpy(®s->gpr[3], args, 6 * sizeof(args[0])); -} - static inline int syscall_get_arch(struct task_struct *task) { return AUDIT_ARCH_OPENRISC; diff --git a/arch/openrisc/kernel/entry.S b/arch/openrisc/kernel/entry.S index c68f3349c1..3ca1b1f490 100644 --- a/arch/openrisc/kernel/entry.S +++ b/arch/openrisc/kernel/entry.S @@ -569,8 +569,8 @@ EXCEPTION_ENTRY(_external_irq_handler) #endif CLEAR_LWA_FLAG(r3) l.addi r3,r1,0 - l.movhi r8,hi(do_IRQ) - l.ori r8,r8,lo(do_IRQ) + l.movhi r8,hi(generic_handle_arch_irq) + l.ori r8,r8,lo(generic_handle_arch_irq) l.jalr r8 l.nop l.j _ret_from_intr @@ -1001,11 +1001,10 @@ ENTRY(ret_from_fork) l.lwz r11,PT_GPR11(r1) /* The syscall fast path return expects call-saved registers - * r12-r28 to be untouched, so we restore them here as they + * r14-r28 to be untouched, so we restore them here as they * will have been effectively clobbered when arriving here * via the call to switch() */ - l.lwz r12,PT_GPR12(r1) l.lwz r14,PT_GPR14(r1) l.lwz r16,PT_GPR16(r1) l.lwz r18,PT_GPR18(r1) @@ -1037,10 +1036,10 @@ ENTRY(ret_from_fork) /* _switch MUST never lay on page boundry, cause it runs from * effective addresses and beeing interrupted by iTLB miss would kill it. - * dTLB miss seams to never accour in the bad place since data accesses + * dTLB miss seems to never accour in the bad place since data accesses * are from task structures which are always page aligned. * - * The problem happens in RESTORE_ALL_NO_R11 where we first set the EPCR + * The problem happens in RESTORE_ALL where we first set the EPCR * register, then load the previous register values and only at the end call * the l.rfe instruction. If get TLB miss in beetwen the EPCR register gets * garbled and we end up calling l.rfe with the wrong EPCR. (same probably @@ -1068,9 +1067,8 @@ ENTRY(_switch) /* No need to store r1/PT_SP as it goes into KSP below */ l.sw PT_GPR2(r1),r2 l.sw PT_GPR9(r1),r9 - /* This is wrong, r12 shouldn't be here... but GCC is broken for the time being - * and expects r12 to be callee-saved... */ - l.sw PT_GPR12(r1),r12 + + /* Save callee-saved registers to the new pt_regs */ l.sw PT_GPR14(r1),r14 l.sw PT_GPR16(r1),r16 l.sw PT_GPR18(r1),r18 @@ -1111,9 +1109,7 @@ ENTRY(_switch) /* No need to restore r10 */ /* ...and do not restore r11 */ - /* This is wrong, r12 shouldn't be here... but GCC is broken for the time being - * and expects r12 to be callee-saved... */ - l.lwz r12,PT_GPR12(r1) + /* Restore callee-saved registers */ l.lwz r14,PT_GPR14(r1) l.lwz r16,PT_GPR16(r1) l.lwz r18,PT_GPR18(r1) @@ -1166,9 +1162,8 @@ _fork_save_extra_regs_and_call: ENTRY(__sys_clone) l.movhi r29,hi(sys_clone) - l.ori r29,r29,lo(sys_clone) l.j _fork_save_extra_regs_and_call - l.nop + l.ori r29,r29,lo(sys_clone) ENTRY(__sys_clone3) l.movhi r29,hi(sys_clone3) @@ -1177,9 +1172,8 @@ ENTRY(__sys_clone3) ENTRY(__sys_fork) l.movhi r29,hi(sys_fork) - l.ori r29,r29,lo(sys_fork) l.j _fork_save_extra_regs_and_call - l.nop + l.ori r29,r29,lo(sys_fork) ENTRY(sys_rt_sigreturn) l.jal _sys_rt_sigreturn diff --git a/arch/openrisc/kernel/irq.c b/arch/openrisc/kernel/irq.c index c38fa863af..f38e10962a 100644 --- a/arch/openrisc/kernel/irq.c +++ b/arch/openrisc/kernel/irq.c @@ -36,8 +36,3 @@ void __init init_IRQ(void) { irqchip_init(); } - -void __irq_entry do_IRQ(struct pt_regs *regs) -{ - handle_arch_irq(regs); -} diff --git a/arch/openrisc/kernel/process.c b/arch/openrisc/kernel/process.c index b0698d9ce1..3c0c91bcdc 100644 --- a/arch/openrisc/kernel/process.c +++ b/arch/openrisc/kernel/process.c @@ -263,7 +263,7 @@ void dump_elf_thread(elf_greg_t *dest, struct pt_regs* regs) dest[35] = 0; } -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { /* TODO */ diff --git a/arch/openrisc/kernel/signal.c b/arch/openrisc/kernel/signal.c index 1ebcff2710..92c5b70740 100644 --- a/arch/openrisc/kernel/signal.c +++ b/arch/openrisc/kernel/signal.c @@ -28,8 +28,6 @@ #include #include -#define DEBUG_SIG 0 - struct rt_sigframe { struct siginfo info; struct ucontext uc; @@ -315,7 +313,7 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) } } local_irq_disable(); - thread_flags = current_thread_info()->flags; + thread_flags = read_thread_flags(); } while (thread_flags & _TIF_WORK_MASK); return 0; } diff --git a/arch/openrisc/kernel/smp.c b/arch/openrisc/kernel/smp.c index ba78766cf0..27041db2c8 100644 --- a/arch/openrisc/kernel/smp.c +++ b/arch/openrisc/kernel/smp.c @@ -65,11 +65,7 @@ void __init smp_init_cpus(void) u32 cpu_id; for_each_of_cpu_node(cpu) { - if (of_property_read_u32(cpu, "reg", &cpu_id)) { - pr_warn("%s missing reg property", cpu->full_name); - continue; - } - + cpu_id = of_get_cpu_hwid(cpu, 0); if (cpu_id < NR_CPUS) set_cpu_possible(cpu_id, true); } diff --git a/arch/openrisc/kernel/time.c b/arch/openrisc/kernel/time.c index b828660619..6d18989d63 100644 --- a/arch/openrisc/kernel/time.c +++ b/arch/openrisc/kernel/time.c @@ -20,6 +20,7 @@ #include #include #include +#include #include @@ -127,7 +128,7 @@ irqreturn_t __irq_entry timer_interrupt(struct pt_regs *regs) return IRQ_HANDLED; } -/** +/* * Clocksource: Based on OpenRISC timer/counter * * This sets up the OpenRISC Tick Timer as a clock source. The tick timer @@ -169,4 +170,7 @@ void __init time_init(void) openrisc_timer_init(); openrisc_clockevent_init(); + + of_clk_init(NULL); + timer_probe(); } diff --git a/arch/openrisc/kernel/traps.c b/arch/openrisc/kernel/traps.c index aa1e709405..0446a3c343 100644 --- a/arch/openrisc/kernel/traps.c +++ b/arch/openrisc/kernel/traps.c @@ -197,7 +197,7 @@ void nommu_dump_state(struct pt_regs *regs, } /* This is normally the 'Oops' routine */ -void die(const char *str, struct pt_regs *regs, long err) +void __noreturn die(const char *str, struct pt_regs *regs, long err) { console_verbose(); @@ -212,7 +212,7 @@ void die(const char *str, struct pt_regs *regs, long err) __asm__ __volatile__("l.nop 1"); do {} while (1); #endif - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } /* This is normally the 'Oops' routine */ diff --git a/arch/openrisc/mm/fault.c b/arch/openrisc/mm/fault.c index c730d1a516..80bb66ad42 100644 --- a/arch/openrisc/mm/fault.c +++ b/arch/openrisc/mm/fault.c @@ -32,7 +32,7 @@ unsigned long pte_errors; /* updated by do_page_fault() */ */ volatile pgd_t *current_pgd[NR_CPUS]; -extern void die(char *, struct pt_regs *, long); +extern void __noreturn die(char *, struct pt_regs *, long); /* * This routine handles page faults. It determines the address, @@ -177,18 +177,16 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address, BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - /*RGD modeled on Cris */ - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; + /*RGD modeled on Cris */ + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + /* No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ - goto retry; - } + goto retry; } mmap_read_unlock(mm); @@ -248,8 +246,6 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address, die("Oops", regs, write_acc); - do_exit(SIGKILL); - /* * We ran out of memory, or some other thing happened to us that made * us unable to handle the page fault gracefully. diff --git a/arch/openrisc/mm/init.c b/arch/openrisc/mm/init.c index cfef61a7b6..97305bde1b 100644 --- a/arch/openrisc/mm/init.c +++ b/arch/openrisc/mm/init.c @@ -25,7 +25,6 @@ #include #include #include -#include /* for initrd_* */ #include #include diff --git a/arch/parisc/Kbuild b/arch/parisc/Kbuild index 3c068b700a..a6d3b280ba 100644 --- a/arch/parisc/Kbuild +++ b/arch/parisc/Kbuild @@ -1,2 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y += mm/ kernel/ math-emu/ + +# for cleaning +subdir- += boot diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 27a8b49af1..43c1c880de 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -13,6 +13,8 @@ config PARISC select ARCH_NO_SG_CHAIN select ARCH_SUPPORTS_HUGETLBFS if PA20 select ARCH_SUPPORTS_MEMORY_FAILURE + select ARCH_STACKWALK + select HAVE_RELIABLE_STACKTRACE select DMA_OPS select RTC_CLASS select RTC_DRV_GENERIC @@ -46,6 +48,7 @@ config PARISC select HAVE_ARCH_HASH select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE + select HAVE_ARCH_KFENCE select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_REGS_AND_STACK_ACCESS_API @@ -53,6 +56,7 @@ config PARISC select HAVE_UNSTABLE_SCHED_CLOCK if SMP select LEGACY_TIMER_TICK select CPU_NO_EFFICIENT_FFS + select THREAD_INFO_IN_TASK select NEED_DMA_MAP_STATE select NEED_SG_DMA_LENGTH select HAVE_ARCH_KGDB @@ -81,11 +85,6 @@ config MMU config STACK_GROWSUP def_bool y -config ARCH_DEFCONFIG - string - default "arch/parisc/configs/generic-32bit_defconfig" if !64BIT - default "arch/parisc/configs/generic-64bit_defconfig" if 64BIT - config GENERIC_LOCKBREAK bool default y @@ -252,11 +251,11 @@ config PARISC_PAGE_SIZE_4KB config PARISC_PAGE_SIZE_16KB bool "16KB" - depends on PA8X00 && BROKEN + depends on PA8X00 && BROKEN && !KFENCE config PARISC_PAGE_SIZE_64KB bool "64KB" - depends on PA8X00 && BROKEN + depends on PA8X00 && BROKEN && !KFENCE endchoice @@ -357,7 +356,8 @@ config NR_CPUS int "Maximum number of CPUs (2-32)" range 2 32 depends on SMP - default "4" + default "4" if 64BIT + default "16" config KEXEC bool "Kexec system call" diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile index fadb098de1..82d77f4b0d 100644 --- a/arch/parisc/Makefile +++ b/arch/parisc/Makefile @@ -2,9 +2,7 @@ # parisc/Makefile # # This file is included by the global makefile so that you can add your own -# architecture-specific flags and dependencies. Remember to do have actions -# for "archclean" and "archdep" for cleaning up and making dependencies for -# this architecture +# architecture-specific flags and dependencies. # # This file is subject to the terms and conditions of the GNU General Public # License. See the file "COPYING" in the main directory of this archive @@ -186,8 +184,5 @@ define archhelp @echo ' zinstall - Install compressed vmlinuz kernel' endef -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - archheaders: $(Q)$(MAKE) $(build)=arch/parisc/kernel/syscalls all diff --git a/arch/parisc/boot/compressed/.gitignore b/arch/parisc/boot/compressed/.gitignore index b9853a356a..a5839aa167 100644 --- a/arch/parisc/boot/compressed/.gitignore +++ b/arch/parisc/boot/compressed/.gitignore @@ -1,6 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -firmware.c -real2.S sizes.h vmlinux vmlinux.lds diff --git a/arch/parisc/boot/compressed/Makefile b/arch/parisc/boot/compressed/Makefile index 9fe5487816..a294a1b58e 100644 --- a/arch/parisc/boot/compressed/Makefile +++ b/arch/parisc/boot/compressed/Makefile @@ -9,10 +9,10 @@ KCOV_INSTRUMENT := n GCOV_PROFILE := n UBSAN_SANITIZE := n +OBJECTS := head.o real2.o firmware.o misc.o piggy.o targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4 -targets += misc.o piggy.o sizes.h head.o real2.o firmware.o -targets += real2.S firmware.c +targets += $(OBJECTS) sizes.h KBUILD_CFLAGS := -D__KERNEL__ -O2 -DBOOTLOADER KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING @@ -23,10 +23,8 @@ ifndef CONFIG_64BIT KBUILD_CFLAGS += -mfast-indirect-calls endif -OBJECTS += $(obj)/head.o $(obj)/real2.o $(obj)/firmware.o $(obj)/misc.o $(obj)/piggy.o - LDFLAGS_vmlinux := -X -e startup --as-needed -T -$(obj)/vmlinux: $(obj)/vmlinux.lds $(OBJECTS) $(LIBGCC) FORCE +$(obj)/vmlinux: $(obj)/vmlinux.lds $(addprefix $(obj)/, $(OBJECTS)) $(LIBGCC) FORCE $(call if_changed,ld) sed-sizes := -e 's/^\([0-9a-fA-F]*\) . \(__bss_start\|_end\|parisc_kernel_start\)$$/\#define SZ\2 0x\1/p' @@ -43,16 +41,7 @@ $(obj)/head.o: $(obj)/sizes.h CFLAGS_misc.o += -I$(objtree)/$(obj) $(obj)/misc.o: $(obj)/sizes.h -$(obj)/firmware.o: $(obj)/firmware.c -$(obj)/firmware.c: $(srctree)/arch/$(SRCARCH)/kernel/firmware.c - $(call cmd,shipped) - AFLAGS_real2.o += -DBOOTLOADER -$(obj)/real2.o: $(obj)/real2.S -$(obj)/real2.S: $(srctree)/arch/$(SRCARCH)/kernel/real2.S - $(call cmd,shipped) - -$(obj)/misc.o: $(obj)/sizes.h CPPFLAGS_vmlinux.lds += -I$(objtree)/$(obj) -DBOOTLOADER $(obj)/vmlinux.lds: $(obj)/sizes.h @@ -61,8 +50,6 @@ OBJCOPYFLAGS_vmlinux.bin := -R .comment -R .note -S $(obj)/vmlinux.bin: vmlinux FORCE $(call if_changed,objcopy) -vmlinux.bin.all-y := $(obj)/vmlinux.bin - suffix-$(CONFIG_KERNEL_GZIP) := gz suffix-$(CONFIG_KERNEL_BZIP2) := bz2 suffix-$(CONFIG_KERNEL_LZ4) := lz4 @@ -70,18 +57,18 @@ suffix-$(CONFIG_KERNEL_LZMA) := lzma suffix-$(CONFIG_KERNEL_LZO) := lzo suffix-$(CONFIG_KERNEL_XZ) := xz -$(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE +$(obj)/vmlinux.bin.gz: $(obj)/vmlinux.bin FORCE $(call if_changed,gzip) -$(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE - $(call if_changed,bzip2) -$(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lz4) -$(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzma) -$(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzo) -$(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,xzkern) +$(obj)/vmlinux.bin.bz2: $(obj)/vmlinux.bin FORCE + $(call if_changed,bzip2_with_size) +$(obj)/vmlinux.bin.lz4: $(obj)/vmlinux.bin FORCE + $(call if_changed,lz4_with_size) +$(obj)/vmlinux.bin.lzma: $(obj)/vmlinux.bin FORCE + $(call if_changed,lzma_with_size) +$(obj)/vmlinux.bin.lzo: $(obj)/vmlinux.bin FORCE + $(call if_changed,lzo_with_size) +$(obj)/vmlinux.bin.xz: $(obj)/vmlinux.bin FORCE + $(call if_changed,xzkern_with_size) LDFLAGS_piggy.o := -r --format binary --oformat $(LD_BFD) -T $(obj)/piggy.o: $(obj)/vmlinux.scr $(obj)/vmlinux.bin.$(suffix-y) FORCE diff --git a/arch/parisc/configs/generic-32bit_defconfig b/arch/parisc/configs/generic-32bit_defconfig index dd14e31313..53061cb2cf 100644 --- a/arch/parisc/configs/generic-32bit_defconfig +++ b/arch/parisc/configs/generic-32bit_defconfig @@ -22,7 +22,6 @@ CONFIG_PCI_LBA=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y -# CONFIG_BLK_DEV_BSG is not set # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m CONFIG_NET=y @@ -55,13 +54,14 @@ CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_ST=y CONFIG_BLK_DEV_SR=y CONFIG_CHR_DEV_SG=y +# CONFIG_BLK_DEV_BSG is not set CONFIG_SCSI_LASI700=y CONFIG_SCSI_SYM53C8XX_2=y CONFIG_SCSI_ZALON=y CONFIG_SCSI_DH=y CONFIG_ATA=y -CONFIG_ATA_GENERIC=y CONFIG_PATA_NS87415=y +CONFIG_ATA_GENERIC=y CONFIG_MD=y CONFIG_BLK_DEV_MD=m CONFIG_MD_LINEAR=m @@ -212,7 +212,6 @@ CONFIG_NFS_FS=m CONFIG_NFSD=m CONFIG_NFSD_V3=y CONFIG_CIFS=m -CONFIG_CIFS_WEAK_PW_HASH=y CONFIG_CIFS_XATTR=y CONFIG_CIFS_POSIX=y # CONFIG_CIFS_DEBUG is not set @@ -221,21 +220,18 @@ CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA1=y -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m -CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_DES=y -CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SERPENT=m -CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_DEFLATE=y CONFIG_CRC_CCITT=m CONFIG_CRC_T10DIF=y CONFIG_FONTS=y +CONFIG_PRINTK_TIME=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y CONFIG_DEBUG_MEMORY_INIT=y diff --git a/arch/parisc/configs/generic-64bit_defconfig b/arch/parisc/configs/generic-64bit_defconfig index 53054b8146..1b8fd80cbe 100644 --- a/arch/parisc/configs/generic-64bit_defconfig +++ b/arch/parisc/configs/generic-64bit_defconfig @@ -1,16 +1,19 @@ CONFIG_LOCALVERSION="-64bit" # CONFIG_LOCALVERSION_AUTO is not set +CONFIG_KERNEL_LZ4=y CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y +CONFIG_AUDIT=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_BSD_PROCESS_ACCT_V3=y CONFIG_TASKSTATS=y CONFIG_TASK_DELAY_ACCT=y CONFIG_TASK_XACCT=y CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y CONFIG_CGROUPS=y CONFIG_MEMCG=y -CONFIG_MEMCG_SWAP=y CONFIG_CGROUP_PIDS=y CONFIG_CPUSETS=y CONFIG_RELAY=y @@ -34,6 +37,7 @@ CONFIG_MODVERSIONS=y CONFIG_BLK_DEV_INTEGRITY=y CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set +CONFIG_MEMORY_FAILURE=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -64,12 +68,15 @@ CONFIG_SCSI_ISCSI_ATTRS=y CONFIG_SCSI_SRP_ATTRS=y CONFIG_ISCSI_BOOT_SYSFS=y CONFIG_SCSI_MPT2SAS=y -CONFIG_SCSI_LASI700=m +CONFIG_SCSI_LASI700=y CONFIG_SCSI_SYM53C8XX_2=y CONFIG_SCSI_ZALON=y CONFIG_SCSI_QLA_ISCSI=m CONFIG_SCSI_DH=y CONFIG_ATA=y +CONFIG_SATA_SIL=y +CONFIG_SATA_SIS=y +CONFIG_SATA_VIA=y CONFIG_PATA_NS87415=y CONFIG_PATA_SIL680=y CONFIG_ATA_GENERIC=y @@ -78,6 +85,7 @@ CONFIG_MD_LINEAR=m CONFIG_BLK_DEV_DM=m CONFIG_DM_RAID=m CONFIG_DM_UEVENT=y +CONFIG_DM_AUDIT=y CONFIG_FUSION=y CONFIG_FUSION_SPI=y CONFIG_FUSION_SAS=y @@ -124,14 +132,13 @@ CONFIG_QLCNIC=m # CONFIG_NET_VENDOR_TI is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -CONFIG_MDIO_BITBANG=m CONFIG_PHYLIB=y CONFIG_BROADCOM_PHY=m CONFIG_CICADA_PHY=m CONFIG_DAVICOM_PHY=m CONFIG_ICPLUS_PHY=m -CONFIG_LSI_ET1011C_PHY=m CONFIG_LXT_PHY=m +CONFIG_LSI_ET1011C_PHY=m CONFIG_MARVELL_PHY=m CONFIG_NATIONAL_PHY=m CONFIG_QSEMI_PHY=m @@ -139,6 +146,7 @@ CONFIG_REALTEK_PHY=m CONFIG_SMSC_PHY=m CONFIG_STE10XP=m CONFIG_VITESSE_PHY=m +CONFIG_MDIO_BITBANG=m CONFIG_SLIP=m # CONFIG_WLAN is not set CONFIG_INPUT_EVDEV=y @@ -150,7 +158,6 @@ CONFIG_SERIO_SERPORT=m # CONFIG_HP_SDC is not set CONFIG_SERIO_RAW=m # CONFIG_LEGACY_PTYS is not set -CONFIG_NOZOMI=m CONFIG_SERIAL_8250=y # CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set CONFIG_SERIAL_8250_CONSOLE=y @@ -160,6 +167,7 @@ CONFIG_SERIAL_8250_EXTENDED=y CONFIG_SERIAL_8250_MANY_PORTS=y CONFIG_SERIAL_8250_SHARE_IRQ=y CONFIG_SERIAL_JSM=m +CONFIG_NOZOMI=m CONFIG_IPMI_HANDLER=y CONFIG_IPMI_DEVICE_INTERFACE=y CONFIG_IPMI_SI=y @@ -188,13 +196,22 @@ CONFIG_AGP=y CONFIG_AGP_PARISC=y CONFIG_DRM=y CONFIG_DRM_RADEON=y -CONFIG_FIRMWARE_EDID=y -CONFIG_FB_MODE_HELPERS=y -CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_FB=y +CONFIG_FB_MATROX=y +CONFIG_FB_MATROX_MYSTIQUE=y +CONFIG_FB_MATROX_G=y +CONFIG_FB_MATROX_I2C=y +CONFIG_FB_MATROX_MAVEN=y +CONFIG_FB_RADEON=y +CONFIG_LOGO=y +# CONFIG_LOGO_LINUX_CLUT224 is not set CONFIG_HIDRAW=y CONFIG_HID_PID=y CONFIG_USB_HIDDEV=y CONFIG_USB=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_OHCI_HCD_PLATFORM=y CONFIG_UIO=y CONFIG_UIO_PDRV_GENIRQ=m CONFIG_UIO_AEC=m @@ -202,7 +219,6 @@ CONFIG_UIO_SERCOS3=m CONFIG_UIO_PCI_GENERIC=m CONFIG_STAGING=y CONFIG_QLGE=m -CONFIG_HP100=m CONFIG_EXT2_FS=y CONFIG_EXT2_FS_XATTR=y CONFIG_EXT2_FS_SECURITY=y @@ -245,7 +261,6 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_ARC4=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_DEFLATE=m # CONFIG_CRYPTO_HW is not set diff --git a/arch/parisc/include/asm/assembly.h b/arch/parisc/include/asm/assembly.h index a39250cb7d..6369082c6c 100644 --- a/arch/parisc/include/asm/assembly.h +++ b/arch/parisc/include/asm/assembly.h @@ -3,38 +3,19 @@ * Copyright (C) 1999 Hewlett-Packard (Frank Rowand) * Copyright (C) 1999 Philipp Rumpf * Copyright (C) 1999 SuSE GmbH + * Copyright (C) 2021 Helge Deller */ #ifndef _PARISC_ASSEMBLY_H #define _PARISC_ASSEMBLY_H -#define CALLEE_FLOAT_FRAME_SIZE 80 - #ifdef CONFIG_64BIT -#define LDREG ldd -#define STREG std -#define LDREGX ldd,s -#define LDREGM ldd,mb -#define STREGM std,ma -#define SHRREG shrd -#define SHLREG shld -#define ANDCM andcm,* -#define COND(x) * ## x #define RP_OFFSET 16 #define FRAME_SIZE 128 #define CALLEE_REG_FRAME_SIZE 144 #define REG_SZ 8 #define ASM_ULONG_INSN .dword #else /* CONFIG_64BIT */ -#define LDREG ldw -#define STREG stw -#define LDREGX ldwx,s -#define LDREGM ldwm -#define STREGM stwm -#define SHRREG shr -#define SHLREG shlw -#define ANDCM andcm -#define COND(x) x #define RP_OFFSET 20 #define FRAME_SIZE 64 #define CALLEE_REG_FRAME_SIZE 128 @@ -42,6 +23,10 @@ #define ASM_ULONG_INSN .word #endif +/* Frame alignment for 32- and 64-bit */ +#define FRAME_ALIGN 64 + +#define CALLEE_FLOAT_FRAME_SIZE 80 #define CALLEE_SAVE_FRAME_SIZE (CALLEE_REG_FRAME_SIZE + CALLEE_FLOAT_FRAME_SIZE) #ifdef CONFIG_PA20 @@ -58,8 +43,34 @@ #define PA_ASM_LEVEL 1.1 #endif +/* Privilege level field in the rightmost two bits of the IA queues */ +#define PRIV_USER 3 +#define PRIV_KERNEL 0 + #ifdef __ASSEMBLY__ +#ifdef CONFIG_64BIT +#define LDREG ldd +#define STREG std +#define LDREGX ldd,s +#define LDREGM ldd,mb +#define STREGM std,ma +#define SHRREG shrd +#define SHLREG shld +#define ANDCM andcm,* +#define COND(x) * ## x +#else /* CONFIG_64BIT */ +#define LDREG ldw +#define STREG stw +#define LDREGX ldwx,s +#define LDREGM ldwm +#define STREGM stwm +#define SHRREG shr +#define SHLREG shlw +#define ANDCM andcm +#define COND(x) x +#endif + #ifdef CONFIG_64BIT /* the 64-bit pa gnu assembler unfortunately defaults to .level 1.1 or 2.0 so * work around that for now... */ @@ -71,6 +82,7 @@ #include #include +#include sp = 30 gp = 27 @@ -135,6 +147,27 @@ extrd,u \r, 63-(\sa), 64-(\sa), \t .endm + /* Extract unsigned for 32- and 64-bit + * The extru instruction leaves the most significant 32 bits of the + * target register in an undefined state on PA 2.0 systems. */ + .macro extru_safe r, p, len, t +#ifdef CONFIG_64BIT + extrd,u \r, 32+(\p), \len, \t +#else + extru \r, \p, \len, \t +#endif + .endm + + /* The depi instruction leaves the most significant 32 bits of the + * target register in an undefined state on PA 2.0 systems. */ + .macro depi_safe i, p, len, t +#ifdef CONFIG_64BIT + depdi \i, 32+(\p), \len, \t +#else + depi \i, \p, \len, \t +#endif + .endm + /* load 32-bit 'value' into 'reg' compensating for the ldil * sign-extension when running in wide mode. * WARNING!! neither 'value' nor 'reg' can be expressions @@ -497,6 +530,30 @@ nop /* 7 */ .endm + /* Switch to virtual mapping, trashing only %r1 */ + .macro virt_map + /* pcxt_ssm_bug */ + rsm PSW_SM_I, %r0 /* barrier for "Relied upon Translation */ + mtsp %r0, %sr4 + mtsp %r0, %sr5 + mtsp %r0, %sr6 + tovirt_r1 %r29 + load32 KERNEL_PSW, %r1 + + rsm PSW_SM_QUIET,%r0 /* second "heavy weight" ctl op */ + mtctl %r0, %cr17 /* Clear IIASQ tail */ + mtctl %r0, %cr17 /* Clear IIASQ head */ + mtctl %r1, %ipsw + load32 4f, %r1 + mtctl %r1, %cr18 /* Set IIAOQ tail */ + ldo 4(%r1), %r1 + mtctl %r1, %cr18 /* Set IIAOQ head */ + rfir + nop +4: + .endm + + /* * ASM_EXCEPTIONTABLE_ENTRY * diff --git a/arch/parisc/include/asm/bitops.h b/arch/parisc/include/asm/bitops.h index 5779d463b3..56ffd260c6 100644 --- a/arch/parisc/include/asm/bitops.h +++ b/arch/parisc/include/asm/bitops.h @@ -112,8 +112,6 @@ static __inline__ int test_and_change_bit(int nr, volatile unsigned long * addr) #include -#ifdef __KERNEL__ - /** * __ffs - find first bit in word. returns 0 to "BITS_PER_LONG-1". * @word: The word to search @@ -213,16 +211,7 @@ static __inline__ int fls(unsigned int x) #include #include #include - -#endif /* __KERNEL__ */ - -#include - -#ifdef __KERNEL__ - #include #include -#endif /* __KERNEL__ */ - #endif /* _PARISC_BITOPS_H */ diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h index eef0096db5..859b8a34ad 100644 --- a/arch/parisc/include/asm/cacheflush.h +++ b/arch/parisc/include/asm/cacheflush.h @@ -49,7 +49,7 @@ void invalidate_kernel_vmap_range(void *vaddr, int size); #define flush_cache_vunmap(start, end) flush_cache_all() #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -extern void flush_dcache_page(struct page *page); +void flush_dcache_page(struct page *page); #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) diff --git a/arch/parisc/include/asm/futex.h b/arch/parisc/include/asm/futex.h index 71aa0921d6..b5835325d4 100644 --- a/arch/parisc/include/asm/futex.h +++ b/arch/parisc/include/asm/futex.h @@ -2,32 +2,31 @@ #ifndef _ASM_PARISC_FUTEX_H #define _ASM_PARISC_FUTEX_H -#ifdef __KERNEL__ - #include #include #include #include /* The following has to match the LWS code in syscall.S. We have - sixteen four-word locks. */ + * 256 four-word locks. We use bits 20-27 of the futex virtual + * address for the hash index. + */ + +static inline unsigned long _futex_hash_index(unsigned long ua) +{ + return (ua >> 2) & 0x3fc; +} static inline void -_futex_spin_lock_irqsave(u32 __user *uaddr, unsigned long int *flags) +_futex_spin_lock_irqsave(arch_spinlock_t *s, unsigned long *flags) { - extern u32 lws_lock_start[]; - long index = ((long)uaddr & 0x7f8) >> 1; - arch_spinlock_t *s = (arch_spinlock_t *)&lws_lock_start[index]; local_irq_save(*flags); arch_spin_lock(s); } static inline void -_futex_spin_unlock_irqrestore(u32 __user *uaddr, unsigned long int *flags) +_futex_spin_unlock_irqrestore(arch_spinlock_t *s, unsigned long *flags) { - extern u32 lws_lock_start[]; - long index = ((long)uaddr & 0x7f8) >> 1; - arch_spinlock_t *s = (arch_spinlock_t *)&lws_lock_start[index]; arch_spin_unlock(s); local_irq_restore(*flags); } @@ -35,15 +34,21 @@ _futex_spin_unlock_irqrestore(u32 __user *uaddr, unsigned long int *flags) static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { - unsigned long int flags; + extern u32 lws_lock_start[]; + unsigned long ua = (unsigned long)uaddr; + arch_spinlock_t *s; + unsigned long flags; int oldval, ret; u32 tmp; - _futex_spin_lock_irqsave(uaddr, &flags); + s = (arch_spinlock_t *)&lws_lock_start[_futex_hash_index(ua)]; + _futex_spin_lock_irqsave(s, &flags); - ret = -EFAULT; - if (unlikely(get_user(oldval, uaddr) != 0)) + /* Return -EFAULT if we encounter a page fault or COW break */ + if (unlikely(get_user(oldval, uaddr) != 0)) { + ret = -EFAULT; goto out_pagefault_enable; + } ret = 0; tmp = oldval; @@ -66,13 +71,14 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) break; default: ret = -ENOSYS; + goto out_pagefault_enable; } - if (ret == 0 && unlikely(put_user(tmp, uaddr) != 0)) + if (unlikely(put_user(tmp, uaddr) != 0)) ret = -EFAULT; out_pagefault_enable: - _futex_spin_unlock_irqrestore(uaddr, &flags); + _futex_spin_unlock_irqrestore(s, &flags); if (!ret) *oval = oldval; @@ -84,6 +90,9 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { + extern u32 lws_lock_start[]; + unsigned long ua = (unsigned long)uaddr; + arch_spinlock_t *s; u32 val; unsigned long flags; @@ -98,26 +107,27 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, /* HPPA has no cmpxchg in hardware and therefore the * best we can do here is use an array of locks. The - * lock selected is based on a hash of the userspace - * address. This should scale to a couple of CPUs. + * lock selected is based on a hash of the virtual + * address of the futex. This should scale to a couple + * of CPUs. */ - _futex_spin_lock_irqsave(uaddr, &flags); + s = (arch_spinlock_t *)&lws_lock_start[_futex_hash_index(ua)]; + _futex_spin_lock_irqsave(s, &flags); if (unlikely(get_user(val, uaddr) != 0)) { - _futex_spin_unlock_irqrestore(uaddr, &flags); + _futex_spin_unlock_irqrestore(s, &flags); return -EFAULT; } if (val == oldval && unlikely(put_user(newval, uaddr) != 0)) { - _futex_spin_unlock_irqrestore(uaddr, &flags); + _futex_spin_unlock_irqrestore(s, &flags); return -EFAULT; } *uval = val; - _futex_spin_unlock_irqrestore(uaddr, &flags); + _futex_spin_unlock_irqrestore(s, &flags); return 0; } -#endif /*__KERNEL__*/ #endif /*_ASM_PARISC_FUTEX_H*/ diff --git a/arch/parisc/include/asm/ide.h b/arch/parisc/include/asm/ide.h index 34cdac01ed..7aa75b93a1 100644 --- a/arch/parisc/include/asm/ide.h +++ b/arch/parisc/include/asm/ide.h @@ -12,8 +12,6 @@ #ifndef __ASM_PARISC_IDE_H #define __ASM_PARISC_IDE_H -#ifdef __KERNEL__ - /* Generic I/O and MEMIO string operations. */ #define __ide_insw insw @@ -53,6 +51,4 @@ static __inline__ void __ide_mm_outsl(void __iomem *port, void *addr, u32 count) } } -#endif /* __KERNEL__ */ - #endif /* __ASM_PARISC_IDE_H */ diff --git a/arch/parisc/include/asm/io.h b/arch/parisc/include/asm/io.h index 0b52591023..837ddddbac 100644 --- a/arch/parisc/include/asm/io.h +++ b/arch/parisc/include/asm/io.h @@ -273,9 +273,9 @@ static inline int inl(unsigned long addr) return -1; } -#define outb(x, y) BUG() -#define outw(x, y) BUG() -#define outl(x, y) BUG() +#define outb(x, y) ({(void)(x); (void)(y); BUG(); 0;}) +#define outw(x, y) ({(void)(x); (void)(y); BUG(); 0;}) +#define outl(x, y) ({(void)(x); (void)(y); BUG(); 0;}) #endif /* diff --git a/arch/parisc/include/asm/jump_label.h b/arch/parisc/include/asm/jump_label.h index 7efb1aa2f7..af2a598bc0 100644 --- a/arch/parisc/include/asm/jump_label.h +++ b/arch/parisc/include/asm/jump_label.h @@ -5,6 +5,7 @@ #ifndef __ASSEMBLY__ #include +#include #include #define JUMP_LABEL_NOP_SIZE 4 diff --git a/arch/parisc/include/asm/mckinley.h b/arch/parisc/include/asm/mckinley.h index eb84dbeb7f..1314390b90 100644 --- a/arch/parisc/include/asm/mckinley.h +++ b/arch/parisc/include/asm/mckinley.h @@ -1,10 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef ASM_PARISC_MCKINLEY_H #define ASM_PARISC_MCKINLEY_H -#ifdef __KERNEL__ /* declared in arch/parisc/kernel/setup.c */ extern struct proc_dir_entry * proc_mckinley_root; -#endif /*__KERNEL__*/ #endif /*ASM_PARISC_MCKINLEY_H*/ diff --git a/arch/parisc/include/asm/pdc.h b/arch/parisc/include/asm/pdc.h index b388d81765..18b957a863 100644 --- a/arch/parisc/include/asm/pdc.h +++ b/arch/parisc/include/asm/pdc.h @@ -51,6 +51,8 @@ int pdc_spaceid_bits(unsigned long *space_bits); int pdc_btlb_info(struct pdc_btlb_info *btlb); int pdc_mem_map_hpa(struct pdc_memory_map *r_addr, struct pdc_module_path *mod_path); #endif /* !CONFIG_PA20 */ +int pdc_pim_toc11(struct pdc_toc_pim_11 *ret); +int pdc_pim_toc20(struct pdc_toc_pim_20 *ret); int pdc_lan_station_id(char *lan_addr, unsigned long net_hpa); int pdc_stable_read(unsigned long staddr, void *memaddr, unsigned long count); diff --git a/arch/parisc/include/asm/processor.h b/arch/parisc/include/asm/processor.h index eeb7da0642..3a3d054384 100644 --- a/arch/parisc/include/asm/processor.h +++ b/arch/parisc/include/asm/processor.h @@ -12,6 +12,7 @@ #ifndef __ASSEMBLY__ #include +#include #include #include #include @@ -37,16 +38,12 @@ #define DEFAULT_MAP_BASE DEFAULT_MAP_BASE32 #endif -#ifdef __KERNEL__ - /* XXX: STACK_TOP actually should be STACK_BOTTOM for parisc. * prumpf */ #define STACK_TOP TASK_SIZE #define STACK_TOP_MAX DEFAULT_TASK_SIZE -#endif - #ifndef __ASSEMBLY__ unsigned long calc_max_stack_size(unsigned long stack_max); @@ -101,8 +98,6 @@ DECLARE_PER_CPU(struct cpuinfo_parisc, cpu_data); #define CPU_HVERSION ((boot_cpu_data.hversion >> 4) & 0x0FFF) -#define ARCH_MIN_TASKALIGN 8 - struct thread_struct { struct pt_regs regs; unsigned long task_size; @@ -273,7 +268,7 @@ struct mm_struct; /* Free all resources held by a thread. */ extern void release_thread(struct task_struct *); -extern unsigned long get_wchan(struct task_struct *p); +extern unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) ((tsk)->thread.regs.iaoq[0]) #define KSTK_ESP(tsk) ((tsk)->thread.regs.gr[30]) @@ -294,6 +289,11 @@ extern int _parisc_requires_coherency; extern int running_on_qemu; +extern void __noreturn toc_intr(struct pt_regs *regs); +extern void toc_handler(void); +extern unsigned int toc_handler_size; +extern unsigned int toc_handler_csum; + #endif /* __ASSEMBLY__ */ #endif /* __ASM_PARISC_PROCESSOR_H */ diff --git a/arch/parisc/include/asm/ptrace.h b/arch/parisc/include/asm/ptrace.h index 143fb2a89d..eea3f3df08 100644 --- a/arch/parisc/include/asm/ptrace.h +++ b/arch/parisc/include/asm/ptrace.h @@ -5,17 +5,17 @@ #ifndef _PARISC_PTRACE_H #define _PARISC_PTRACE_H +#include #include - #define task_regs(task) ((struct pt_regs *) ((char *)(task) + TASK_REGS)) #define arch_has_single_step() 1 #define arch_has_block_step() 1 /* XXX should we use iaoq[1] or iaoq[0] ? */ -#define user_mode(regs) (((regs)->iaoq[0] & 3) ? 1 : 0) -#define user_space(regs) (((regs)->iasq[1] != 0) ? 1 : 0) +#define user_mode(regs) (((regs)->iaoq[0] & 3) != PRIV_KERNEL) +#define user_space(regs) ((regs)->iasq[1] != PRIV_KERNEL) #define instruction_pointer(regs) ((regs)->iaoq[0] & ~3) #define user_stack_pointer(regs) ((regs)->gr[30]) unsigned long profile_pc(struct pt_regs *); diff --git a/arch/parisc/include/asm/runway.h b/arch/parisc/include/asm/runway.h index f3cfe69439..5cf061376d 100644 --- a/arch/parisc/include/asm/runway.h +++ b/arch/parisc/include/asm/runway.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef ASM_PARISC_RUNWAY_H #define ASM_PARISC_RUNWAY_H -#ifdef __KERNEL__ /* declared in arch/parisc/kernel/setup.c */ extern struct proc_dir_entry * proc_runway_root; @@ -9,5 +8,4 @@ extern struct proc_dir_entry * proc_runway_root; #define RUNWAY_STATUS 0x10 #define RUNWAY_DEBUG 0x40 -#endif /* __KERNEL__ */ #endif /* ASM_PARISC_RUNWAY_H */ diff --git a/arch/parisc/include/asm/smp.h b/arch/parisc/include/asm/smp.h index b9a18db4b0..2279ebe5e2 100644 --- a/arch/parisc/include/asm/smp.h +++ b/arch/parisc/include/asm/smp.h @@ -32,9 +32,9 @@ extern void smp_send_all_nop(void); extern void arch_send_call_function_single_ipi(int cpu); extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); -#endif /* !ASSEMBLY */ +#define raw_smp_processor_id() (current_thread_info()->cpu) -#define raw_smp_processor_id() (current_thread_info()->cpu) +#endif /* !ASSEMBLY */ #else /* CONFIG_SMP */ diff --git a/arch/parisc/include/asm/spinlock.h b/arch/parisc/include/asm/spinlock.h index fa5ee8a45d..a6e5d66a76 100644 --- a/arch/parisc/include/asm/spinlock.h +++ b/arch/parisc/include/asm/spinlock.h @@ -23,21 +23,6 @@ static inline void arch_spin_lock(arch_spinlock_t *x) continue; } -static inline void arch_spin_lock_flags(arch_spinlock_t *x, - unsigned long flags) -{ - volatile unsigned int *a; - - a = __ldcw_align(x); - while (__ldcw(a) == 0) - while (*a == 0) - if (flags & PSW_SM_I) { - local_irq_enable(); - local_irq_disable(); - } -} -#define arch_spin_lock_flags arch_spin_lock_flags - static inline void arch_spin_unlock(arch_spinlock_t *x) { volatile unsigned int *a; diff --git a/arch/parisc/include/asm/thread_info.h b/arch/parisc/include/asm/thread_info.h index 00ad50fef7..1a58795f78 100644 --- a/arch/parisc/include/asm/thread_info.h +++ b/arch/parisc/include/asm/thread_info.h @@ -2,30 +2,24 @@ #ifndef _ASM_PARISC_THREAD_INFO_H #define _ASM_PARISC_THREAD_INFO_H -#ifdef __KERNEL__ - #ifndef __ASSEMBLY__ #include #include struct thread_info { - struct task_struct *task; /* main task structure */ unsigned long flags; /* thread_info flags (see TIF_*) */ - __u32 cpu; /* current CPU */ int preempt_count; /* 0=premptable, <0=BUG; will also serve as bh-counter */ +#ifdef CONFIG_SMP + unsigned int cpu; +#endif }; #define INIT_THREAD_INFO(tsk) \ { \ - .task = &tsk, \ .flags = 0, \ - .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ } -/* how to get the thread information struct from C */ -#define current_thread_info() ((struct thread_info *)mfctl(30)) - #endif /* !__ASSEMBLY */ /* thread information allocation */ @@ -57,6 +51,7 @@ struct thread_info { #define TIF_BLOCKSTEP 10 /* branch stepping? */ #define TIF_SECCOMP 11 /* secure computing */ #define TIF_SYSCALL_TRACEPOINT 12 /* syscall tracepoint instrumentation */ +#define TIF_NONBLOCK_WARNING 13 /* warned about wrong O_NONBLOCK usage */ #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) @@ -87,6 +82,4 @@ struct thread_info { # define is_32bit_task() (1) #endif -#endif /* __KERNEL__ */ - #endif /* _ASM_PARISC_THREAD_INFO_H */ diff --git a/arch/parisc/include/asm/traps.h b/arch/parisc/include/asm/traps.h index 8ecc1f0c04..34619f010c 100644 --- a/arch/parisc/include/asm/traps.h +++ b/arch/parisc/include/asm/traps.h @@ -14,6 +14,7 @@ void parisc_terminate(char *msg, struct pt_regs *regs, void die_if_kernel(char *str, struct pt_regs *regs, long err); /* mm/fault.c */ +unsigned long parisc_acctyp(unsigned long code, unsigned int inst); const char *trap_name(unsigned long code); void do_page_fault(struct pt_regs *regs, unsigned long code, unsigned long address); diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index 192ad9e11b..123d5f16cd 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -53,15 +53,18 @@ struct exception_table_entry { /* * ASM_EXCEPTIONTABLE_ENTRY_EFAULT() creates a special exception table entry * (with lowest bit set) for which the fault handler in fixup_exception() will - * load -EFAULT into %r8 for a read or write fault, and zeroes the target + * load -EFAULT into %r29 for a read or write fault, and zeroes the target * register in case of a read fault in get_user(). */ +#define ASM_EXCEPTIONTABLE_REG 29 +#define ASM_EXCEPTIONTABLE_VAR(__variable) \ + register long __variable __asm__ ("r29") = 0 #define ASM_EXCEPTIONTABLE_ENTRY_EFAULT( fault_addr, except_addr )\ ASM_EXCEPTIONTABLE_ENTRY( fault_addr, except_addr + 1) #define __get_user_internal(sr, val, ptr) \ ({ \ - register long __gu_err __asm__ ("r8") = 0; \ + ASM_EXCEPTIONTABLE_VAR(__gu_err); \ \ switch (sizeof(*(ptr))) { \ case 1: __get_user_asm(sr, val, "ldb", ptr); break; \ @@ -86,8 +89,8 @@ struct exception_table_entry { __asm__("1: " ldx " 0(" sr "%2),%0\n" \ "9:\n" \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \ - : "=r"(__gu_val), "=r"(__gu_err) \ - : "r"(ptr), "1"(__gu_err)); \ + : "=r"(__gu_val), "+r"(__gu_err) \ + : "r"(ptr)); \ \ (val) = (__force __typeof__(*(ptr))) __gu_val; \ } @@ -120,8 +123,8 @@ struct exception_table_entry { "9:\n" \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 9b) \ - : "=&r"(__gu_tmp.l), "=r"(__gu_err) \ - : "r"(ptr), "1"(__gu_err)); \ + : "=&r"(__gu_tmp.l), "+r"(__gu_err) \ + : "r"(ptr)); \ \ (val) = __gu_tmp.t; \ } @@ -131,14 +134,13 @@ struct exception_table_entry { #define __put_user_internal(sr, x, ptr) \ ({ \ - register long __pu_err __asm__ ("r8") = 0; \ - __typeof__(*(ptr)) __x = (__typeof__(*(ptr)))(x); \ + ASM_EXCEPTIONTABLE_VAR(__pu_err); \ \ switch (sizeof(*(ptr))) { \ - case 1: __put_user_asm(sr, "stb", __x, ptr); break; \ - case 2: __put_user_asm(sr, "sth", __x, ptr); break; \ - case 4: __put_user_asm(sr, "stw", __x, ptr); break; \ - case 8: STD_USER(sr, __x, ptr); break; \ + case 1: __put_user_asm(sr, "stb", x, ptr); break; \ + case 2: __put_user_asm(sr, "sth", x, ptr); break; \ + case 4: __put_user_asm(sr, "stw", x, ptr); break; \ + case 8: STD_USER(sr, x, ptr); break; \ default: BUILD_BUG(); \ } \ \ @@ -147,7 +149,9 @@ struct exception_table_entry { #define __put_user(x, ptr) \ ({ \ - __put_user_internal("%%sr3,", x, ptr); \ + __typeof__(&*(ptr)) __ptr = ptr; \ + __typeof__(*(__ptr)) __x = (__typeof__(*(__ptr)))(x); \ + __put_user_internal("%%sr3,", __x, __ptr); \ }) #define __put_kernel_nofault(dst, src, type, err_label) \ @@ -168,7 +172,8 @@ struct exception_table_entry { * gcc knows about, so there are no aliasing issues. These macros must * also be aware that fixups are executed in the context of the fault, * and any registers used there must be listed as clobbers. - * r8 is already listed as err. + * The register holding the possible EFAULT error (ASM_EXCEPTIONTABLE_REG) + * is already listed as input and output register. */ #define __put_user_asm(sr, stx, x, ptr) \ @@ -176,8 +181,8 @@ struct exception_table_entry { "1: " stx " %2,0(" sr "%1)\n" \ "9:\n" \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \ - : "=r"(__pu_err) \ - : "r"(ptr), "r"(x), "0"(__pu_err)) + : "+r"(__pu_err) \ + : "r"(ptr), "r"(x)) #if !defined(CONFIG_64BIT) @@ -189,8 +194,8 @@ struct exception_table_entry { "9:\n" \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 9b) \ ASM_EXCEPTIONTABLE_ENTRY_EFAULT(2b, 9b) \ - : "=r"(__pu_err) \ - : "r"(ptr), "r"(__val), "0"(__pu_err)); \ + : "+r"(__pu_err) \ + : "r"(ptr), "r"(__val)); \ } while (0) #endif /* !defined(CONFIG_64BIT) */ diff --git a/arch/parisc/include/asm/unaligned.h b/arch/parisc/include/asm/unaligned.h index 3bda16773b..c062129510 100644 --- a/arch/parisc/include/asm/unaligned.h +++ b/arch/parisc/include/asm/unaligned.h @@ -4,10 +4,8 @@ #include -#ifdef __KERNEL__ struct pt_regs; void handle_unaligned(struct pt_regs *regs); int check_unaligned(struct pt_regs *regs); -#endif #endif /* _ASM_PARISC_UNALIGNED_H */ diff --git a/arch/parisc/include/uapi/asm/pdc.h b/arch/parisc/include/uapi/asm/pdc.h index 15211723eb..e794e143ec 100644 --- a/arch/parisc/include/uapi/asm/pdc.h +++ b/arch/parisc/include/uapi/asm/pdc.h @@ -4,7 +4,7 @@ /* * PDC return values ... - * All PDC calls return a subset of these errors. + * All PDC calls return a subset of these errors. */ #define PDC_WARN 3 /* Call completed with a warning */ @@ -165,7 +165,7 @@ #define PDC_PSW_GET_DEFAULTS 1 /* Return defaults */ #define PDC_PSW_SET_DEFAULTS 2 /* Set default */ #define PDC_PSW_ENDIAN_BIT 1 /* set for big endian */ -#define PDC_PSW_WIDE_BIT 2 /* set for wide mode */ +#define PDC_PSW_WIDE_BIT 2 /* set for wide mode */ #define PDC_SYSTEM_MAP 22 /* find system modules */ #define PDC_FIND_MODULE 0 @@ -274,7 +274,7 @@ #define PDC_PCI_PCI_INT_ROUTE_SIZE 13 #define PDC_PCI_GET_INT_TBL_SIZE PDC_PCI_PCI_INT_ROUTE_SIZE #define PDC_PCI_PCI_INT_ROUTE 14 -#define PDC_PCI_GET_INT_TBL PDC_PCI_PCI_INT_ROUTE +#define PDC_PCI_GET_INT_TBL PDC_PCI_PCI_INT_ROUTE #define PDC_PCI_READ_MON_TYPE 15 #define PDC_PCI_WRITE_MON_TYPE 16 @@ -345,7 +345,7 @@ /* constants for PDC_CHASSIS */ #define OSTAT_OFF 0 -#define OSTAT_FLT 1 +#define OSTAT_FLT 1 #define OSTAT_TEST 2 #define OSTAT_INIT 3 #define OSTAT_SHUT 4 @@ -398,10 +398,12 @@ struct zeropage { /* int (*vec_rendz)(void); */ unsigned int vec_rendz; int vec_pow_fail_flen; - int vec_pad[10]; - + int vec_pad0[3]; + unsigned int vec_toc_hi; + int vec_pad1[6]; + /* [0x040] reserved processor dependent */ - int pad0[112]; + int pad0[112]; /* in QEMU pad0[0] holds "SeaBIOS\0" */ /* [0x200] reserved */ int pad1[84]; @@ -689,6 +691,42 @@ struct pdc_hpmc_pim_20 { /* PDC_PIM */ unsigned long long fr[32]; }; +struct pim_cpu_state_cf { + union { + unsigned int + iqv : 1, /* IIA queue Valid */ + iqf : 1, /* IIA queue Failure */ + ipv : 1, /* IPRs Valid */ + grv : 1, /* GRs Valid */ + crv : 1, /* CRs Valid */ + srv : 1, /* SRs Valid */ + trv : 1, /* CR24 through CR31 valid */ + pad : 24, /* reserved */ + td : 1; /* TOC did not cause any damage to the system state */ + unsigned int val; + }; +}; + +struct pdc_toc_pim_11 { + unsigned int gr[32]; + unsigned int cr[32]; + unsigned int sr[8]; + unsigned int iasq_back; + unsigned int iaoq_back; + unsigned int check_type; + struct pim_cpu_state_cf cpu_state; +}; + +struct pdc_toc_pim_20 { + unsigned long long gr[32]; + unsigned long long cr[32]; + unsigned long long sr[8]; + unsigned long long iasq_back; + unsigned long long iaoq_back; + unsigned int check_type; + struct pim_cpu_state_cf cpu_state; +}; + #endif /* !defined(__ASSEMBLY__) */ #endif /* _UAPI_PARISC_PDC_H */ diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h index 8baaad52d7..845ddc63c8 100644 --- a/arch/parisc/include/uapi/asm/socket.h +++ b/arch/parisc/include/uapi/asm/socket.h @@ -123,6 +123,8 @@ #define SO_BUF_LOCK 0x4046 +#define SO_RESERVE_MEM 0x4047 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/parisc/kernel/Makefile b/arch/parisc/kernel/Makefile index 068d90950d..8fb819bbbb 100644 --- a/arch/parisc/kernel/Makefile +++ b/arch/parisc/kernel/Makefile @@ -10,7 +10,7 @@ obj-y := cache.o pacache.o setup.o pdt.o traps.o time.o irq.o \ ptrace.o hardware.o inventory.o drivers.o alternative.o \ signal.o hpmc.o real2.o parisc_ksyms.o unaligned.o \ process.o processor.o pdc_cons.o pdc_chassis.o unwind.o \ - patch.o + patch.o toc.o toc_asm.o ifdef CONFIG_FUNCTION_TRACER # Do not profile debug and lowlevel utilities diff --git a/arch/parisc/kernel/asm-offsets.c b/arch/parisc/kernel/asm-offsets.c index 22924a3f17..2a83ef36d2 100644 --- a/arch/parisc/kernel/asm-offsets.c +++ b/arch/parisc/kernel/asm-offsets.c @@ -22,18 +22,12 @@ #include #include +#include #include #include #include #include -#ifdef CONFIG_64BIT -#define FRAME_SIZE 128 -#else -#define FRAME_SIZE 64 -#endif -#define FRAME_ALIGN 64 - /* Add FRAME_SIZE to the size x and align it to y. All definitions * that use align_frame will include space for a frame. */ @@ -41,13 +35,12 @@ int main(void) { - DEFINE(TASK_THREAD_INFO, offsetof(struct task_struct, stack)); - DEFINE(TASK_FLAGS, offsetof(struct task_struct, flags)); - DEFINE(TASK_SIGPENDING, offsetof(struct task_struct, pending)); - DEFINE(TASK_PTRACE, offsetof(struct task_struct, ptrace)); - DEFINE(TASK_MM, offsetof(struct task_struct, mm)); - DEFINE(TASK_PERSONALITY, offsetof(struct task_struct, personality)); - DEFINE(TASK_PID, offsetof(struct task_struct, pid)); + DEFINE(TASK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); +#ifdef CONFIG_SMP + DEFINE(TASK_TI_CPU, offsetof(struct task_struct, thread_info.cpu)); +#endif + DEFINE(TASK_STACK, offsetof(struct task_struct, stack)); + DEFINE(TASK_PAGEFAULT_DISABLED, offsetof(struct task_struct, pagefault_disabled)); BLANK(); DEFINE(TASK_REGS, offsetof(struct task_struct, thread.regs)); DEFINE(TASK_PT_PSW, offsetof(struct task_struct, thread.regs.gr[ 0])); @@ -135,10 +128,6 @@ int main(void) DEFINE(TASK_PT_ISR, offsetof(struct task_struct, thread.regs.isr)); DEFINE(TASK_PT_IOR, offsetof(struct task_struct, thread.regs.ior)); BLANK(); - DEFINE(TASK_SZ, sizeof(struct task_struct)); - /* TASK_SZ_ALGN includes space for a stack frame. */ - DEFINE(TASK_SZ_ALGN, align_frame(sizeof(struct task_struct), FRAME_ALIGN)); - BLANK(); DEFINE(PT_PSW, offsetof(struct pt_regs, gr[ 0])); DEFINE(PT_GR1, offsetof(struct pt_regs, gr[ 1])); DEFINE(PT_GR2, offsetof(struct pt_regs, gr[ 2])); @@ -223,17 +212,11 @@ int main(void) DEFINE(PT_IIR, offsetof(struct pt_regs, iir)); DEFINE(PT_ISR, offsetof(struct pt_regs, isr)); DEFINE(PT_IOR, offsetof(struct pt_regs, ior)); - DEFINE(PT_SIZE, sizeof(struct pt_regs)); /* PT_SZ_ALGN includes space for a stack frame. */ DEFINE(PT_SZ_ALGN, align_frame(sizeof(struct pt_regs), FRAME_ALIGN)); BLANK(); - DEFINE(TI_TASK, offsetof(struct thread_info, task)); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); - DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); - DEFINE(TI_PRE_COUNT, offsetof(struct thread_info, preempt_count)); - DEFINE(THREAD_SZ, sizeof(struct thread_info)); - /* THREAD_SZ_ALGN includes space for a stack frame. */ - DEFINE(THREAD_SZ_ALGN, align_frame(sizeof(struct thread_info), FRAME_ALIGN)); + DEFINE(TI_PRE_COUNT, offsetof(struct task_struct, thread_info.preempt_count)); BLANK(); DEFINE(ICACHE_BASE, offsetof(struct pdc_cache_info, ic_base)); DEFINE(ICACHE_STRIDE, offsetof(struct pdc_cache_info, ic_stride)); diff --git a/arch/parisc/kernel/audit.c b/arch/parisc/kernel/audit.c index 9eb47b2225..f420b55521 100644 --- a/arch/parisc/kernel/audit.c +++ b/arch/parisc/kernel/audit.c @@ -47,13 +47,15 @@ int audit_classify_syscall(int abi, unsigned syscall) #endif switch (syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_execve: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 4ed4942d83..94150b91c9 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -543,10 +543,33 @@ static inline pte_t *get_ptep(pgd_t *pgd, unsigned long addr) return ptep; } +static void flush_cache_pages(struct vm_area_struct *vma, struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + unsigned long addr, pfn; + pte_t *ptep; + + for (addr = start; addr < end; addr += PAGE_SIZE) { + ptep = get_ptep(mm->pgd, addr); + if (ptep) { + pfn = pte_pfn(*ptep); + flush_cache_page(vma, addr, pfn); + } + } +} + +static void flush_user_cache_tlb(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + flush_user_dcache_range_asm(start, end); + if (vma->vm_flags & VM_EXEC) + flush_user_icache_range_asm(start, end); + flush_tlb_range(vma, start, end); +} + void flush_cache_mm(struct mm_struct *mm) { struct vm_area_struct *vma; - pgd_t *pgd; /* Flushing the whole cache on each cpu takes forever on rp3440, etc. So, avoid it if the mm isn't too big. */ @@ -558,45 +581,22 @@ void flush_cache_mm(struct mm_struct *mm) return; } + preempt_disable(); if (mm->context == mfsp(3)) { - for (vma = mm->mmap; vma; vma = vma->vm_next) { - flush_user_dcache_range_asm(vma->vm_start, vma->vm_end); - if (vma->vm_flags & VM_EXEC) - flush_user_icache_range_asm(vma->vm_start, vma->vm_end); - flush_tlb_range(vma, vma->vm_start, vma->vm_end); - } + for (vma = mm->mmap; vma; vma = vma->vm_next) + flush_user_cache_tlb(vma, vma->vm_start, vma->vm_end); + preempt_enable(); return; } - pgd = mm->pgd; - for (vma = mm->mmap; vma; vma = vma->vm_next) { - unsigned long addr; - - for (addr = vma->vm_start; addr < vma->vm_end; - addr += PAGE_SIZE) { - unsigned long pfn; - pte_t *ptep = get_ptep(pgd, addr); - if (!ptep) - continue; - pfn = pte_pfn(*ptep); - if (!pfn_valid(pfn)) - continue; - if (unlikely(mm->context)) { - flush_tlb_page(vma, addr); - __flush_cache_page(vma, addr, PFN_PHYS(pfn)); - } else { - __purge_cache_page(vma, addr, PFN_PHYS(pfn)); - } - } - } + for (vma = mm->mmap; vma; vma = vma->vm_next) + flush_cache_pages(vma, mm, vma->vm_start, vma->vm_end); + preempt_enable(); } void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - pgd_t *pgd; - unsigned long addr; - if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) && end - start >= parisc_cache_flush_threshold) { if (vma->vm_mm->context) @@ -605,30 +605,15 @@ void flush_cache_range(struct vm_area_struct *vma, return; } + preempt_disable(); if (vma->vm_mm->context == mfsp(3)) { - flush_user_dcache_range_asm(start, end); - if (vma->vm_flags & VM_EXEC) - flush_user_icache_range_asm(start, end); - flush_tlb_range(vma, start, end); + flush_user_cache_tlb(vma, start, end); + preempt_enable(); return; } - pgd = vma->vm_mm->pgd; - for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { - unsigned long pfn; - pte_t *ptep = get_ptep(pgd, addr); - if (!ptep) - continue; - pfn = pte_pfn(*ptep); - if (pfn_valid(pfn)) { - if (unlikely(vma->vm_mm->context)) { - flush_tlb_page(vma, addr); - __flush_cache_page(vma, addr, PFN_PHYS(pfn)); - } else { - __purge_cache_page(vma, addr, PFN_PHYS(pfn)); - } - } - } + flush_cache_pages(vma, vma->vm_mm, vma->vm_start, vma->vm_end); + preempt_enable(); } void diff --git a/arch/parisc/kernel/compat_audit.c b/arch/parisc/kernel/compat_audit.c index 20c39c9d86..539b16891b 100644 --- a/arch/parisc/kernel/compat_audit.c +++ b/arch/parisc/kernel/compat_audit.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include unsigned int parisc32_dir_class[] = { @@ -30,12 +31,14 @@ int parisc32_classify_syscall(unsigned syscall) { switch (syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_execve: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index 437c8d31f3..6e9cdb2698 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -51,30 +51,6 @@ extrd,u \spc,(64 - (SPACEID_SHIFT)),32,\prot .endm #endif - - /* Switch to virtual mapping, trashing only %r1 */ - .macro virt_map - /* pcxt_ssm_bug */ - rsm PSW_SM_I, %r0 /* barrier for "Relied upon Translation */ - mtsp %r0, %sr4 - mtsp %r0, %sr5 - mtsp %r0, %sr6 - tovirt_r1 %r29 - load32 KERNEL_PSW, %r1 - - rsm PSW_SM_QUIET,%r0 /* second "heavy weight" ctl op */ - mtctl %r0, %cr17 /* Clear IIASQ tail */ - mtctl %r0, %cr17 /* Clear IIASQ head */ - mtctl %r1, %ipsw - load32 4f, %r1 - mtctl %r1, %cr18 /* Set IIAOQ tail */ - ldo 4(%r1), %r1 - mtctl %r1, %cr18 /* Set IIAOQ head */ - rfir - nop -4: - .endm - /* * The "get_stack" macros are responsible for determining the * kernel stack value. @@ -87,8 +63,8 @@ * Need to set up a kernel stack, so call the * get_stack_use_cr30 macro to set up a pointer * to the pt_regs structure contained within the - * task pointer pointed to by cr30. Set the stack - * pointer to point to the end of the task structure. + * task pointer pointed to by cr30. Load the stack + * pointer from the task structure. * * Note that we use shadowed registers for temps until * we can save %r26 and %r29. %r26 is used to preserve @@ -100,8 +76,6 @@ * or handle_interruption. %r29 is used to hold a pointer * the register save area, and once again, it needs to * be a non-shadowed register so that it survives the rfir. - * - * N.B. TASK_SZ_ALGN and PT_SZ_ALGN include space for a stack frame. */ .macro get_stack_use_cr30 @@ -110,12 +84,11 @@ copy %r30, %r17 mfctl %cr30, %r1 - ldo THREAD_SZ_ALGN(%r1), %r30 - mtsp %r0,%sr7 + tophys %r1,%r9 /* task_struct */ + LDREG TASK_STACK(%r9),%r30 + ldo PT_SZ_ALGN(%r30),%r30 + mtsp %r0,%sr7 /* clear sr7 after kernel stack was set! */ mtsp %r16,%sr3 - tophys %r1,%r9 - LDREG TI_TASK(%r9), %r1 /* thread_info -> task_struct */ - tophys %r1,%r9 ldo TASK_REGS(%r9),%r9 STREG %r17,PT_GR30(%r9) STREG %r29,PT_GR29(%r9) @@ -393,17 +366,9 @@ */ .macro L2_ptep pmd,pte,index,va,fault #if CONFIG_PGTABLE_LEVELS == 3 - extru \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index + extru_safe \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index #else -# if defined(CONFIG_64BIT) - extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index - #else - # if PAGE_SIZE > 4096 - extru \va,31-ASM_PGDIR_SHIFT,32-ASM_PGDIR_SHIFT,\index - # else - extru \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index - # endif -# endif + extru_safe \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index #endif dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */ #if CONFIG_PGTABLE_LEVELS < 3 @@ -413,7 +378,7 @@ bb,>=,n \pmd,_PxD_PRESENT_BIT,\fault dep %r0,31,PxD_FLAG_SHIFT,\pmd /* clear flags */ SHLREG \pmd,PxD_VALUE_SHIFT,\pmd - extru \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index + extru_safe \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */ shladd \index,BITS_PER_PTE_ENTRY,\pmd,\pmd /* pmd is now pte */ .endm @@ -757,7 +722,7 @@ ENTRY(ret_from_kernel_thread) BL schedule_tail, %r2 nop - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30), %r1 + mfctl %cr30,%r1 /* task_struct */ LDREG TASK_PT_GR25(%r1), %r26 #ifdef CONFIG_64BIT LDREG TASK_PT_GR27(%r1), %r27 @@ -788,7 +753,6 @@ ENTRY_CFI(_switch_to) STREG %r30, TASK_PT_KSP(%r26) LDREG TASK_PT_KSP(%r25), %r30 - LDREG TASK_THREAD_INFO(%r25), %r25 bv %r0(%r2) mtctl %r25,%cr30 @@ -819,17 +783,16 @@ ENDPROC_CFI(_switch_to) .align PAGE_SIZE ENTRY_CFI(syscall_exit_rfi) - mfctl %cr30,%r16 - LDREG TI_TASK(%r16), %r16 /* thread_info -> task_struct */ + mfctl %cr30,%r16 /* task_struct */ ldo TASK_REGS(%r16),%r16 /* Force iaoq to userspace, as the user has had access to our current * context via sigcontext. Also Filter the PSW for the same reason. */ LDREG PT_IAOQ0(%r16),%r19 - depi 3,31,2,%r19 + depi PRIV_USER,31,2,%r19 STREG %r19,PT_IAOQ0(%r16) LDREG PT_IAOQ1(%r16),%r19 - depi 3,31,2,%r19 + depi PRIV_USER,31,2,%r19 STREG %r19,PT_IAOQ1(%r16) LDREG PT_PSW(%r16),%r19 load32 USER_PSW_MASK,%r1 @@ -865,14 +828,14 @@ ENTRY_CFI(syscall_exit_rfi) ENTRY(intr_return) /* check for reschedule */ mfctl %cr30,%r1 - LDREG TI_FLAGS(%r1),%r19 /* sched.h: TIF_NEED_RESCHED */ + LDREG TASK_TI_FLAGS(%r1),%r19 /* sched.h: TIF_NEED_RESCHED */ bb,<,n %r19,31-TIF_NEED_RESCHED,intr_do_resched /* forward */ .import do_notify_resume,code intr_check_sig: /* As above */ mfctl %cr30,%r1 - LDREG TI_FLAGS(%r1),%r19 + LDREG TASK_TI_FLAGS(%r1),%r19 ldi (_TIF_USER_WORK_MASK & ~_TIF_NEED_RESCHED), %r20 and,COND(<>) %r19, %r20, %r0 b,n intr_restore /* skip past if we've nothing to do */ @@ -974,8 +937,8 @@ intr_do_preempt: /* current_thread_info()->preempt_count */ mfctl %cr30, %r1 - LDREG TI_PRE_COUNT(%r1), %r19 - cmpib,COND(<>) 0, %r19, intr_restore /* if preempt_count > 0 */ + ldw TI_PRE_COUNT(%r1), %r19 + cmpib,<> 0, %r19, intr_restore /* if preempt_count > 0 */ nop /* prev insn branched backwards */ /* check if we interrupted a critical path */ @@ -1716,7 +1679,7 @@ dtlb_fault: .macro fork_like name ENTRY_CFI(sys_\name\()_wrapper) - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30), %r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1),%r1 reg_save %r1 mfctl %cr27, %r28 @@ -1736,7 +1699,7 @@ ENTRY(child_return) BL schedule_tail, %r2 nop finish_child_return: - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30), %r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1),%r1 /* get pt regs */ LDREG PT_CR27(%r1), %r3 @@ -1747,7 +1710,7 @@ finish_child_return: END(child_return) ENTRY_CFI(sys_rt_sigreturn_wrapper) - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r26 + mfctl %cr30,%r26 ldo TASK_REGS(%r26),%r26 /* get pt regs */ /* Don't save regs, we are going to restore them from sigcontext. */ STREG %r2, -RP_OFFSET(%r30) @@ -1764,7 +1727,7 @@ ENTRY_CFI(sys_rt_sigreturn_wrapper) LDREG -RP_OFFSET(%r30), %r2 /* FIXME: I think we need to restore a few more things here. */ - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1),%r1 /* get pt regs */ reg_restore %r1 @@ -1783,9 +1746,7 @@ ENTRY(syscall_exit) */ /* save return value now */ - mfctl %cr30, %r1 - LDREG TI_TASK(%r1),%r1 STREG %r28,TASK_PT_GR28(%r1) /* Seems to me that dp could be wrong here, if the syscall involved @@ -1796,13 +1757,14 @@ ENTRY(syscall_exit) syscall_check_resched: /* check for reschedule */ - - LDREG TI_FLAGS-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r19 /* long */ + mfctl %cr30,%r19 + LDREG TASK_TI_FLAGS(%r19),%r19 /* long */ bb,<,n %r19, 31-TIF_NEED_RESCHED, syscall_do_resched /* forward */ .import do_signal,code syscall_check_sig: - LDREG TI_FLAGS-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r19 + mfctl %cr30,%r19 + LDREG TASK_TI_FLAGS(%r19),%r19 ldi (_TIF_USER_WORK_MASK & ~_TIF_NEED_RESCHED), %r26 and,COND(<>) %r19, %r26, %r0 b,n syscall_restore /* skip past if we've nothing to do */ @@ -1813,7 +1775,7 @@ syscall_do_signal: * consistent with all the relevant state of the process * before the syscall. We need to verify this. */ - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1), %r26 /* struct pt_regs *regs */ reg_save %r26 @@ -1824,17 +1786,17 @@ syscall_do_signal: BL do_notify_resume,%r2 ldi 1, %r25 /* long in_syscall = 1 */ - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 + mfctl %cr30,%r1 ldo TASK_REGS(%r1), %r20 /* reload pt_regs */ reg_restore %r20 b,n syscall_check_sig syscall_restore: - LDREG TI_TASK-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 + mfctl %cr30,%r1 /* Are we being ptraced? */ - LDREG TI_FLAGS-THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r19 + LDREG TASK_TI_FLAGS(%r1),%r19 ldi _TIF_SINGLESTEP|_TIF_BLOCKSTEP,%r2 and,COND(=) %r19,%r2,%r0 b,n syscall_restore_rfi @@ -1873,7 +1835,7 @@ syscall_restore: mtsp %r1,%sr5 /* Restore sr5 */ mtsp %r1,%sr6 /* Restore sr6 */ - depi 3,31,2,%r31 /* ensure return to user mode. */ + depi PRIV_USER,31,2,%r31 /* ensure return to user mode. */ #ifdef CONFIG_64BIT /* decide whether to reset the wide mode bit @@ -1949,7 +1911,7 @@ syscall_restore_rfi: STREG %r0,TASK_PT_SR2(%r1) LDREG TASK_PT_GR31(%r1),%r2 - depi 3,31,2,%r2 /* ensure return to user mode. */ + depi PRIV_USER,31,2,%r2 /* ensure return to user mode. */ STREG %r2,TASK_PT_IAOQ0(%r1) ldo 4(%r2),%r2 STREG %r2,TASK_PT_IAOQ1(%r1) @@ -1958,10 +1920,10 @@ syscall_restore_rfi: pt_regs_ok: LDREG TASK_PT_IAOQ0(%r1),%r2 - depi 3,31,2,%r2 /* ensure return to user mode. */ + depi PRIV_USER,31,2,%r2 /* ensure return to user mode. */ STREG %r2,TASK_PT_IAOQ0(%r1) LDREG TASK_PT_IAOQ1(%r1),%r2 - depi 3,31,2,%r2 + depi PRIV_USER,31,2,%r2 STREG %r2,TASK_PT_IAOQ1(%r1) b intr_restore copy %r25,%r16 diff --git a/arch/parisc/kernel/firmware.c b/arch/parisc/kernel/firmware.c index 7034227dbd..3370e347dd 100644 --- a/arch/parisc/kernel/firmware.c +++ b/arch/parisc/kernel/firmware.c @@ -1061,6 +1061,38 @@ int pdc_mem_pdt_read_entries(struct pdc_mem_read_pdt *pret, return retval; } +/** + * pdc_pim_toc11 - Fetch TOC PIM 1.1 data from firmware. + * @ret: pointer to return buffer + */ +int pdc_pim_toc11(struct pdc_toc_pim_11 *ret) +{ + int retval; + unsigned long flags; + + spin_lock_irqsave(&pdc_lock, flags); + retval = mem_pdc_call(PDC_PIM, PDC_PIM_TOC, __pa(pdc_result), + __pa(ret), sizeof(*ret)); + spin_unlock_irqrestore(&pdc_lock, flags); + return retval; +} + +/** + * pdc_pim_toc20 - Fetch TOC PIM 2.0 data from firmware. + * @ret: pointer to return buffer + */ +int pdc_pim_toc20(struct pdc_toc_pim_20 *ret) +{ + int retval; + unsigned long flags; + + spin_lock_irqsave(&pdc_lock, flags); + retval = mem_pdc_call(PDC_PIM, PDC_PIM_TOC, __pa(pdc_result), + __pa(ret), sizeof(*ret)); + spin_unlock_irqrestore(&pdc_lock, flags); + return retval; +} + /** * pdc_tod_set - Set the Time-Of-Day clock. * @sec: The number of seconds since epoch. diff --git a/arch/parisc/kernel/ftrace.c b/arch/parisc/kernel/ftrace.c index 0a1e75af53..4d392e4ed3 100644 --- a/arch/parisc/kernel/ftrace.c +++ b/arch/parisc/kernel/ftrace.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -24,6 +25,8 @@ #define __hot __section(".text.hot") #ifdef CONFIG_FUNCTION_GRAPH_TRACER +static DEFINE_STATIC_KEY_FALSE(ftrace_graph_enable); + /* * Hook the return address and push it in the stack of return addrs * in current thread info. @@ -48,25 +51,19 @@ static void __hot prepare_ftrace_return(unsigned long *parent, } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ +static ftrace_func_t ftrace_func; + void notrace __hot ftrace_function_trampoline(unsigned long parent, unsigned long self_addr, unsigned long org_sp_gr3, struct ftrace_regs *fregs) { -#ifndef CONFIG_DYNAMIC_FTRACE - extern ftrace_func_t ftrace_trace_function; -#endif extern struct ftrace_ops *function_trace_op; - if (function_trace_op->flags & FTRACE_OPS_FL_ENABLED && - ftrace_trace_function != ftrace_stub) - ftrace_trace_function(self_addr, parent, - function_trace_op, fregs); + ftrace_func(self_addr, parent, function_trace_op, fregs); #ifdef CONFIG_FUNCTION_GRAPH_TRACER - if (dereference_function_descriptor(ftrace_graph_return) != - dereference_function_descriptor(ftrace_stub) || - ftrace_graph_entry != ftrace_graph_entry_stub) { + if (static_branch_unlikely(&ftrace_graph_enable)) { unsigned long *parent_rp; /* calculate pointer to %rp in stack */ @@ -84,23 +81,21 @@ void notrace __hot ftrace_function_trampoline(unsigned long parent, #ifdef CONFIG_FUNCTION_GRAPH_TRACER int ftrace_enable_ftrace_graph_caller(void) { + static_key_enable(&ftrace_graph_enable.key); return 0; } int ftrace_disable_ftrace_graph_caller(void) { + static_key_enable(&ftrace_graph_enable.key); return 0; } #endif #ifdef CONFIG_DYNAMIC_FTRACE - -int __init ftrace_dyn_arch_init(void) -{ - return 0; -} int ftrace_update_ftrace_func(ftrace_func_t func) { + ftrace_func = func; return 0; } @@ -216,7 +211,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, return; regs = ftrace_get_regs(fregs); - preempt_disable_notrace(); p = get_kprobe((kprobe_opcode_t *)ip); if (unlikely(!p) || kprobe_disabled(p)) goto out; @@ -245,7 +239,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, } __this_cpu_write(current_kprobe, NULL); out: - preempt_enable_notrace(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(kprobe_ftrace_handler); diff --git a/arch/parisc/kernel/head.S b/arch/parisc/kernel/head.S index aa93d775c3..b24f77748c 100644 --- a/arch/parisc/kernel/head.S +++ b/arch/parisc/kernel/head.S @@ -35,7 +35,8 @@ END(boot_args) __HEAD .align 4 - .import init_thread_union,data + .import init_task,data + .import init_stack,data .import fault_vector_20,code /* IVA parisc 2.0 32 bit */ #ifndef CONFIG_64BIT .import fault_vector_11,code /* IVA parisc 1.1 32 bit */ @@ -123,12 +124,12 @@ $pgt_fill_loop: load32 start_parisc,%r11 /* And the initial task pointer */ - load32 init_thread_union,%r6 + load32 init_task,%r6 mtctl %r6,%cr30 /* And the stack pointer too */ - ldo THREAD_SZ_ALGN(%r6),%sp - + load32 init_stack,%sp + tophys_r1 %sp #if defined(CONFIG_64BIT) && defined(CONFIG_FUNCTION_TRACER) .import _mcount,data /* initialize mcount FPTR */ @@ -186,12 +187,11 @@ common_stext: #endif /*CONFIG_SMP*/ #ifdef CONFIG_64BIT - tophys_r1 %sp + mfctl %cr30,%r6 /* PCX-W2 firmware bug */ + tophys_r1 %r6 /* Save the rfi target address */ - ldd TI_TASK-THREAD_SZ_ALGN(%sp), %r10 - tophys_r1 %r10 - std %r11, TASK_PT_GR11(%r10) + STREG %r11, TASK_PT_GR11(%r6) /* Switch to wide mode Superdome doesn't support narrow PDC ** calls. */ @@ -206,7 +206,6 @@ common_stext: ** Someday, palo might not do this for the Monarch either. */ 2: - mfctl %cr30,%r6 /* PCX-W2 firmware bug */ ldo PDC_PSW(%r0),%arg0 /* 21 */ ldo PDC_PSW_SET_DEFAULTS(%r0),%arg1 /* 2 */ @@ -216,13 +215,9 @@ common_stext: copy %r0,%arg3 stext_pdc_ret: + LDREG TASK_PT_GR11(%r6), %r11 + tovirt_r1 %r6 mtctl %r6,%cr30 /* restore task thread info */ - - /* restore rfi target address*/ - ldd TI_TASK-THREAD_SZ_ALGN(%sp), %r10 - tophys_r1 %r10 - ldd TASK_PT_GR11(%r10), %r11 - tovirt_r1 %sp #endif /* PARANOID: clear user scratch/user space SR's */ @@ -287,7 +282,9 @@ aligned_rfi: load32 KERNEL_PSW,%r10 mtctl %r10,%ipsw - + + tovirt_r1 %sp + /* Jump through hyperspace to Virt Mode */ rfi nop @@ -343,12 +340,13 @@ smp_slave_stext: #endif /* Initialize the SP - monarch sets up smp_init_current_idle_task */ - load32 PA(smp_init_current_idle_task),%sp - LDREG 0(%sp),%sp /* load task address */ + load32 PA(smp_init_current_idle_task),%r6 + LDREG 0(%r6),%r6 + mtctl %r6,%cr30 + tophys_r1 %r6 + LDREG TASK_STACK(%r6),%sp tophys_r1 %sp - LDREG TASK_THREAD_INFO(%sp),%sp - mtctl %sp,%cr30 /* store in cr30 */ - ldo THREAD_SZ_ALGN(%sp),%sp + ldo FRAME_SIZE(%sp),%sp /* point CPU to kernel page tables */ load32 PA(swapper_pg_dir),%r4 diff --git a/arch/parisc/kernel/hpmc.S b/arch/parisc/kernel/hpmc.S index c298140177..eb2e4bd670 100644 --- a/arch/parisc/kernel/hpmc.S +++ b/arch/parisc/kernel/hpmc.S @@ -43,10 +43,8 @@ * IODC requires 7K byte stack. That leaves 1K byte for os_hpmc. */ - __PAGE_ALIGNED_BSS - .align 4096 -hpmc_stack: - .block 16384 + .import toc_stack,data +#define hpmc_stack toc_stack /* re-use the TOC stack */ #define HPMC_IODC_BUF_SIZE 0x8000 diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c index 0d46b19dc4..eb18e16362 100644 --- a/arch/parisc/kernel/irq.c +++ b/arch/parisc/kernel/irq.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -399,8 +400,7 @@ static inline void stack_overflow_check(struct pt_regs *regs) #ifdef CONFIG_DEBUG_STACKOVERFLOW #define STACK_MARGIN (256*6) - /* Our stack starts directly behind the thread_info struct. */ - unsigned long stack_start = (unsigned long) current_thread_info(); + unsigned long stack_start = (unsigned long) task_stack_page(current); unsigned long sp = regs->gr[30]; unsigned long stack_usage; unsigned int *last_usage; @@ -476,7 +476,7 @@ static void execute_on_irq_stack(void *func, unsigned long param1) union_ptr = &per_cpu(irq_stack_union, smp_processor_id()); irq_stack = (unsigned long) &union_ptr->stack; irq_stack = ALIGN(irq_stack + sizeof(irq_stack_union.slock), - 64); /* align for stack frame usage */ + FRAME_ALIGN); /* align for stack frame usage */ /* We may be called recursive. If we are already using the irq stack, * just continue to use it. Use spinlocks to serialize diff --git a/arch/parisc/kernel/kgdb.c b/arch/parisc/kernel/kgdb.c index c4554ac13e..ab7620f695 100644 --- a/arch/parisc/kernel/kgdb.c +++ b/arch/parisc/kernel/kgdb.c @@ -3,6 +3,7 @@ * PA-RISC KGDB support * * Copyright (c) 2019 Sven Schnelle + * Copyright (c) 2022 Helge Deller * */ @@ -207,3 +208,23 @@ int kgdb_arch_handle_exception(int trap, int signo, } return -1; } + +/* KGDB console driver which uses PDC to read chars from keyboard */ + +static void kgdb_pdc_write_char(u8 chr) +{ + /* no need to print char. kgdb will do it. */ +} + +static struct kgdb_io kgdb_pdc_io_ops = { + .name = "kgdb_pdc", + .read_char = pdc_iodc_getc, + .write_char = kgdb_pdc_write_char, +}; + +static int __init kgdb_pdc_init(void) +{ + kgdb_register_io_module(&kgdb_pdc_io_ops); + return 0; +} +early_initcall(kgdb_pdc_init); diff --git a/arch/parisc/kernel/kprobes.c b/arch/parisc/kernel/kprobes.c index 6d21a515ee..e2bdb5a5f9 100644 --- a/arch/parisc/kernel/kprobes.c +++ b/arch/parisc/kernel/kprobes.c @@ -175,7 +175,7 @@ int __kprobes parisc_kprobe_ss_handler(struct pt_regs *regs) return 1; } -static inline void kretprobe_trampoline(void) +void __kretprobe_trampoline(void) { asm volatile("nop"); asm volatile("nop"); @@ -193,7 +193,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, { unsigned long orig_ret_address; - orig_ret_address = __kretprobe_trampoline_handler(regs, trampoline_p.addr, NULL); + orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); instruction_pointer_set(regs, orig_ret_address); return 1; @@ -217,6 +217,6 @@ int __kprobes arch_trampoline_kprobe(struct kprobe *p) int __init arch_init_kprobes(void) { trampoline_p.addr = (kprobe_opcode_t *) - dereference_function_descriptor(kretprobe_trampoline); + dereference_function_descriptor(__kretprobe_trampoline); return register_kprobe(&trampoline_p); } diff --git a/arch/parisc/kernel/pdt.c b/arch/parisc/kernel/pdt.c index fcc761b0e1..e391b175f5 100644 --- a/arch/parisc/kernel/pdt.c +++ b/arch/parisc/kernel/pdt.c @@ -352,12 +352,10 @@ static int __init pdt_initcall(void) if (pdt_type == PDT_NONE) return -ENODEV; - kpdtd_task = kthread_create(pdt_mainloop, NULL, "kpdtd"); + kpdtd_task = kthread_run(pdt_mainloop, NULL, "kpdtd"); if (IS_ERR(kpdtd_task)) return PTR_ERR(kpdtd_task); - wake_up_process(kpdtd_task); - return 0; } diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index 38ec4ae812..ea3d83b6fb 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -205,7 +205,7 @@ copy_thread(unsigned long clone_flags, unsigned long usp, /* Must exit via ret_from_kernel_thread in order * to call schedule_tail() */ - cregs->ksp = (unsigned long)stack + THREAD_SZ_ALGN + FRAME_SIZE; + cregs->ksp = (unsigned long) stack + FRAME_SIZE + PT_SZ_ALGN; cregs->kpc = (unsigned long) &ret_from_kernel_thread; /* * Copy function and argument to be called from @@ -228,7 +228,7 @@ copy_thread(unsigned long clone_flags, unsigned long usp, if (likely(usp)) cregs->gr[30] = usp; } - cregs->ksp = (unsigned long)stack + THREAD_SZ_ALGN + FRAME_SIZE; + cregs->ksp = (unsigned long) stack + FRAME_SIZE; cregs->kpc = (unsigned long) &child_return; /* Setup thread TLS area */ @@ -240,15 +240,12 @@ copy_thread(unsigned long clone_flags, unsigned long usp, } unsigned long -get_wchan(struct task_struct *p) +__get_wchan(struct task_struct *p) { struct unwind_frame_info info; unsigned long ip; int count = 0; - if (!p || p == current || task_is_running(p)) - return 0; - /* * These bracket the sleeping functions.. */ diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index cceb09855e..b91cb45ffd 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -48,6 +48,7 @@ struct proc_dir_entry * proc_mckinley_root __read_mostly = NULL; void __init setup_cmdline(char **cmdline_p) { extern unsigned int boot_args[]; + char *p; /* Collect stuff passed in from the boot loader */ @@ -56,9 +57,19 @@ void __init setup_cmdline(char **cmdline_p) /* called from hpux boot loader */ boot_command_line[0] = '\0'; } else { - strlcpy(boot_command_line, (char *)__va(boot_args[1]), + strscpy(boot_command_line, (char *)__va(boot_args[1]), COMMAND_LINE_SIZE); + /* autodetect console type (if not done by palo yet) */ + p = boot_command_line; + if (!str_has_prefix(p, "console=") && !strstr(p, " console=")) { + strlcat(p, " console=", COMMAND_LINE_SIZE); + if (PAGE0->mem_cons.cl_class == CL_DUPLEX) + strlcat(p, "ttyS0", COMMAND_LINE_SIZE); + else + strlcat(p, "tty0", COMMAND_LINE_SIZE); + } + #ifdef CONFIG_BLK_DEV_INITRD if (boot_args[2] != 0) /* did palo pass us a ramdisk? */ { @@ -68,7 +79,7 @@ void __init setup_cmdline(char **cmdline_p) #endif } - strcpy(command_line, boot_command_line); + strscpy(command_line, boot_command_line, COMMAND_LINE_SIZE); *cmdline_p = command_line; } diff --git a/arch/parisc/kernel/signal.c b/arch/parisc/kernel/signal.c index 0fb06d87b3..46b1050640 100644 --- a/arch/parisc/kernel/signal.c +++ b/arch/parisc/kernel/signal.c @@ -291,9 +291,9 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs, SIGRESTARTBLOCK_TRAMP+X. */ err |= __put_user(in_syscall ? INSN_LDI_R25_1 : INSN_LDI_R25_0, &frame->tramp[SIGRESTARTBLOCK_TRAMP+0]); - err |= __put_user(INSN_LDI_R20, + err |= __put_user(INSN_LDI_R20, &frame->tramp[SIGRESTARTBLOCK_TRAMP+1]); - err |= __put_user(INSN_BLE_SR2_R0, + err |= __put_user(INSN_BLE_SR2_R0, &frame->tramp[SIGRESTARTBLOCK_TRAMP+2]); err |= __put_user(INSN_NOP, &frame->tramp[SIGRESTARTBLOCK_TRAMP+3]); diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c index cf92ece20b..a32a882a2d 100644 --- a/arch/parisc/kernel/smp.c +++ b/arch/parisc/kernel/smp.c @@ -228,11 +228,13 @@ static inline void send_IPI_allbutself(enum ipi_message_type op) { int i; - + + preempt_disable(); for_each_online_cpu(i) { if (i != smp_processor_id()) send_IPI_single(i, op); } + preempt_enable(); } #ifdef CONFIG_KGDB @@ -337,8 +339,6 @@ int smp_boot_one_cpu(int cpuid, struct task_struct *idle) const struct cpuinfo_parisc *p = &per_cpu(cpu_data, cpuid); long timeout; - task_thread_info(idle)->cpu = cpuid; - /* Let _start know what logical CPU we're booting ** (offset into init_tasks[],cpu_data[]) */ diff --git a/arch/parisc/kernel/stacktrace.c b/arch/parisc/kernel/stacktrace.c index 34bf6d6bf6..023834ef58 100644 --- a/arch/parisc/kernel/stacktrace.c +++ b/arch/parisc/kernel/stacktrace.c @@ -2,45 +2,42 @@ /* * Stack trace management functions * - * Copyright (C) 2009 Helge Deller + * Copyright (C) 2009-2021 Helge Deller * based on arch/x86/kernel/stacktrace.c by Ingo Molnar * and parisc unwind functions by Randolph Chung * * TODO: Userspace stacktrace (CONFIG_USER_STACKTRACE_SUPPORT) */ -#include +#include #include #include -static void dump_trace(struct task_struct *task, struct stack_trace *trace) +static void notrace walk_stackframe(struct task_struct *task, + struct pt_regs *regs, bool (*fn)(void *, unsigned long), void *cookie) { struct unwind_frame_info info; unwind_frame_init_task(&info, task, NULL); - - /* unwind stack and save entries in stack_trace struct */ - trace->nr_entries = 0; - while (trace->nr_entries < trace->max_entries) { + while (1) { if (unwind_once(&info) < 0 || info.ip == 0) break; if (__kernel_text_address(info.ip)) - trace->entries[trace->nr_entries++] = info.ip; + if (!fn(cookie, info.ip)) + break; } } -/* - * Save stack-backtrace addresses into a stack_trace buffer. - */ -void save_stack_trace(struct stack_trace *trace) +void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, + struct task_struct *task, struct pt_regs *regs) { - dump_trace(current, trace); + walk_stackframe(task, regs, consume_entry, cookie); } -EXPORT_SYMBOL_GPL(save_stack_trace); -void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +int arch_stack_walk_reliable(stack_trace_consume_fn consume_entry, void *cookie, + struct task_struct *task) { - dump_trace(tsk, trace); + walk_stackframe(task, NULL, consume_entry, cookie); + return 1; } -EXPORT_SYMBOL_GPL(save_stack_trace_tsk); diff --git a/arch/parisc/kernel/sys_parisc.c b/arch/parisc/kernel/sys_parisc.c index 5f12537318..2b34294517 100644 --- a/arch/parisc/kernel/sys_parisc.c +++ b/arch/parisc/kernel/sys_parisc.c @@ -409,10 +409,12 @@ long parisc_personality(unsigned long personality) static int FIX_O_NONBLOCK(int flags) { - if (flags & O_NONBLOCK_MASK_OUT) { - struct task_struct *tsk = current; - pr_warn_once("%s(%d) uses a deprecated O_NONBLOCK value.\n", - tsk->comm, tsk->pid); + if ((flags & O_NONBLOCK_MASK_OUT) && + !test_thread_flag(TIF_NONBLOCK_WARNING)) { + set_thread_flag(TIF_NONBLOCK_WARNING); + pr_warn("%s(%d) uses a deprecated O_NONBLOCK value." + " Please recompile with newer glibc.\n", + current->comm, current->pid); } return flags & ~O_NONBLOCK_MASK_OUT; } diff --git a/arch/parisc/kernel/syscall.S b/arch/parisc/kernel/syscall.S index 9842dcb204..1373e51298 100644 --- a/arch/parisc/kernel/syscall.S +++ b/arch/parisc/kernel/syscall.S @@ -50,6 +50,22 @@ registers). .level PA_ASM_LEVEL + .macro lws_pagefault_disable reg1,reg2 + mfctl %cr30, \reg2 + ldo TASK_PAGEFAULT_DISABLED(\reg2), \reg2 + ldw 0(%sr2,\reg2), \reg1 + ldo 1(\reg1), \reg1 + stw \reg1, 0(%sr2,\reg2) + .endm + + .macro lws_pagefault_enable reg1,reg2 + mfctl %cr30, \reg2 + ldo TASK_PAGEFAULT_DISABLED(\reg2), \reg2 + ldw 0(%sr2,\reg2), \reg1 + ldo -1(\reg1), \reg1 + stw \reg1, 0(%sr2,\reg2) + .endm + .text .import syscall_exit,code @@ -74,11 +90,11 @@ ENTRY(linux_gateway_page) /* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */ /* Light-weight-syscall entry must always be located at 0xb0 */ /* WARNING: Keep this number updated with table size changes */ -#define __NR_lws_entries (3) +#define __NR_lws_entries (5) lws_entry: gate lws_start, %r0 /* increase privilege */ - depi 3, 31, 2, %r31 /* Ensure we return into user mode. */ + depi PRIV_USER, 31, 2, %r31 /* Ensure we return into user mode. */ /* Fill from 0xb8 to 0xe0 */ .rept 10 @@ -89,7 +105,7 @@ lws_entry: mechanism to work. DO NOT MOVE THIS CODE EVER! */ set_thread_pointer: gate .+8, %r0 /* increase privilege */ - depi 3, 31, 2, %r31 /* Ensure we return into user mode. */ + depi PRIV_USER, 31, 2, %r31 /* Ensure we return into user mode. */ be 0(%sr7,%r31) /* return to user space */ mtctl %r26, %cr27 /* move arg0 to the control register */ @@ -139,9 +155,9 @@ linux_gateway_entry: xor %r1,%r30,%r30 /* ye olde xor trick */ xor %r1,%r30,%r1 xor %r1,%r30,%r30 - - ldo THREAD_SZ_ALGN+FRAME_SIZE(%r30),%r30 /* set up kernel stack */ + LDREG TASK_STACK(%r30),%r30 /* set up kernel stack */ + ldo FRAME_SIZE(%r30),%r30 /* N.B.: It is critical that we don't set sr7 to 0 until r30 * contains a valid kernel stack pointer. It is also * critical that we don't start using the kernel stack @@ -152,7 +168,6 @@ linux_gateway_entry: ssm PSW_SM_I, %r0 /* enable interrupts */ STREGM %r1,FRAME_SIZE(%r30) /* save r1 (usp) here for now */ mfctl %cr30,%r1 /* get task ptr in %r1 */ - LDREG TI_TASK(%r1),%r1 /* Save some registers for sigcontext and potential task switch (see entry.S for the details of which ones are @@ -207,7 +222,7 @@ linux_gateway_entry: /* Are we being ptraced? */ mfctl %cr30, %r1 - LDREG TI_FLAGS(%r1),%r1 + LDREG TASK_TI_FLAGS(%r1),%r1 ldi _TIF_SYSCALL_TRACE_MASK, %r19 and,COND(=) %r1, %r19, %r0 b,n .Ltracesys @@ -272,8 +287,7 @@ tracesys: * C bit set, a non-straced syscall entry results in C and D clear * in the saved PSW. */ - ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ - LDREG TI_TASK(%r1), %r1 + mfctl %cr30,%r1 /* get task ptr */ ssm 0,%r2 STREG %r2,TASK_PT_PSW(%r1) /* Lower 8 bits only!! */ mfsp %sr0,%r2 @@ -327,8 +341,7 @@ tracesys_next: */ copy %ret0,%r20 - ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ - LDREG TI_TASK(%r1), %r1 + mfctl %cr30,%r1 /* get task ptr */ LDREG TASK_PT_GR28(%r1), %r28 /* Restore return value */ LDREG TASK_PT_GR26(%r1), %r26 /* Restore the users args */ LDREG TASK_PT_GR25(%r1), %r25 @@ -385,16 +398,14 @@ tracesys_next: makes a direct call to syscall_trace. */ tracesys_exit: - ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ - LDREG TI_TASK(%r1), %r1 + mfctl %cr30,%r1 /* get task ptr */ #ifdef CONFIG_64BIT ldo -16(%r30),%r29 /* Reference param save area */ #endif ldo TASK_REGS(%r1),%r26 BL do_syscall_trace_exit,%r2 STREG %r28,TASK_PT_GR28(%r1) /* save return value now */ - ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ - LDREG TI_TASK(%r1), %r1 + mfctl %cr30,%r1 /* get task ptr */ LDREG TASK_PT_GR28(%r1), %r28 /* Restore return val. */ ldil L%syscall_exit,%r1 @@ -407,8 +418,7 @@ tracesys_exit: ldo R%tracesys_sigexit(%r2),%r2 tracesys_sigexit: - ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ - LDREG TI_TASK(%r1), %r1 + mfctl %cr30,%r1 /* get task ptr */ #ifdef CONFIG_64BIT ldo -16(%r30),%r29 /* Reference param save area */ #endif @@ -496,8 +506,34 @@ lws_start: /* Jump to lws, lws table pointers already relocated */ be,n 0(%sr2,%r21) +lws_exit_noerror: + lws_pagefault_enable %r1,%r21 + stw,ma %r20, 0(%sr2,%r20) + ssm PSW_SM_I, %r0 + b lws_exit + copy %r0, %r21 + +lws_wouldblock: + ssm PSW_SM_I, %r0 + ldo 2(%r0), %r28 + b lws_exit + ldo -EAGAIN(%r0), %r21 + +lws_pagefault: + lws_pagefault_enable %r1,%r21 + stw,ma %r20, 0(%sr2,%r20) + ssm PSW_SM_I, %r0 + ldo 3(%r0),%r28 + b lws_exit + ldo -EAGAIN(%r0),%r21 + +lws_fault: + ldo 1(%r0),%r28 + b lws_exit + ldo -EFAULT(%r0),%r21 + lws_exit_nosys: - ldo -ENOSYS(%r0),%r21 /* set errno */ + ldo -ENOSYS(%r0),%r21 /* Fall through: Return to userspace */ lws_exit: @@ -524,27 +560,19 @@ lws_exit: %r28 - Return prev through this register. %r21 - Kernel error code - If debugging is DISabled: - - %r21 has the following meanings: - + %r21 returns the following error codes: EAGAIN - CAS is busy, ldcw failed, try again. EFAULT - Read or write failed. - If debugging is enabled: - - EDEADLOCK - CAS called recursively. - EAGAIN && r28 == 1 - CAS is busy. Lock contended. - EAGAIN && r28 == 2 - CAS is busy. ldcw failed. - EFAULT - Read or write failed. + If EAGAIN is returned, %r28 indicates the busy reason: + r28 == 1 - CAS is busy. lock contended. + r28 == 2 - CAS is busy. ldcw failed. + r28 == 3 - CAS is busy. page fault. Scratch: r20, r28, r1 ****************************************************/ - /* Do not enable LWS debugging */ -#define ENABLE_LWS_DEBUG 0 - /* ELF64 Process entry path */ lws_compare_and_swap64: #ifdef CONFIG_64BIT @@ -557,61 +585,45 @@ lws_compare_and_swap64: b,n lws_exit_nosys #endif - /* ELF32 Process entry path */ + /* ELF32/ELF64 Process entry path */ lws_compare_and_swap32: #ifdef CONFIG_64BIT - /* Clip all the input registers */ + /* Wide mode user process? */ + bb,<,n %sp, 31, lws_compare_and_swap + + /* Clip all the input registers for 32-bit processes */ depdi 0, 31, 32, %r26 depdi 0, 31, 32, %r25 depdi 0, 31, 32, %r24 #endif lws_compare_and_swap: + /* Trigger memory reference interruptions without writing to memory */ +1: ldw 0(%r26), %r28 +2: stbys,e %r0, 0(%r26) + + /* Calculate 8-bit hash index from virtual address */ + extru_safe %r26, 27, 8, %r20 + /* Load start of lock table */ - ldil L%lws_lock_start, %r20 - ldo R%lws_lock_start(%r20), %r28 + ldil L%lws_lock_start, %r28 + ldo R%lws_lock_start(%r28), %r28 - /* Extract eight bits from r26 and hash lock (Bits 3-11) */ - extru %r26, 28, 8, %r20 - - /* Find lock to use, the hash is either one of 0 to - 15, multiplied by 16 (keep it 16-byte aligned) + /* Find lock to use, the hash index is one of 0 to + 255, multiplied by 16 (keep it 16-byte aligned) and add to the lock table offset. */ shlw %r20, 4, %r20 add %r20, %r28, %r20 -# if ENABLE_LWS_DEBUG - /* - DEBUG, check for deadlock! - If the thread register values are the same - then we were the one that locked it last and - this is a recurisve call that will deadlock. - We *must* giveup this call and fail. - */ - ldw 4(%sr2,%r20), %r28 /* Load thread register */ - /* WARNING: If cr27 cycles to the same value we have problems */ - mfctl %cr27, %r21 /* Get current thread register */ - cmpb,<>,n %r21, %r28, cas_lock /* Called recursive? */ - b lws_exit /* Return error! */ - ldo -EDEADLOCK(%r0), %r21 -cas_lock: - cmpb,=,n %r0, %r28, cas_nocontend /* Is nobody using it? */ - ldo 1(%r0), %r28 /* 1st case */ - b lws_exit /* Contended... */ - ldo -EAGAIN(%r0), %r21 /* Spin in userspace */ -cas_nocontend: -# endif -/* ENABLE_LWS_DEBUG */ - rsm PSW_SM_I, %r0 /* Disable interrupts */ - /* COW breaks can cause contention on UP systems */ - LDCW 0(%sr2,%r20), %r28 /* Try to acquire the lock */ - cmpb,<>,n %r0, %r28, cas_action /* Did we get it? */ -cas_wouldblock: - ldo 2(%r0), %r28 /* 2nd case */ - ssm PSW_SM_I, %r0 - b lws_exit /* Contended... */ - ldo -EAGAIN(%r0), %r21 /* Spin in userspace */ + + /* Try to acquire the lock */ + LDCW 0(%sr2,%r20), %r28 + comclr,<> %r0, %r28, %r0 + b,n lws_wouldblock + + /* Disable page faults to prevent sleeping in critical region */ + lws_pagefault_disable %r21,%r28 /* prev = *addr; @@ -621,62 +633,35 @@ cas_wouldblock: */ /* NOTES: - This all works becuse intr_do_signal + This all works because intr_do_signal and schedule both check the return iasq and see that we are on the kernel page so this process is never scheduled off or is ever sent any signal of any sort, - thus it is wholly atomic from usrspaces + thus it is wholly atomic from usrspace's perspective */ -cas_action: -#if defined CONFIG_SMP && ENABLE_LWS_DEBUG - /* DEBUG */ - mfctl %cr27, %r1 - stw %r1, 4(%sr2,%r20) -#endif /* The load and store could fail */ -1: ldw 0(%r26), %r28 +3: ldw 0(%r26), %r28 sub,<> %r28, %r25, %r0 -2: stw %r24, 0(%r26) - /* Free lock */ - stw,ma %r20, 0(%sr2,%r20) -#if ENABLE_LWS_DEBUG - /* Clear thread register indicator */ - stw %r0, 4(%sr2,%r20) -#endif - /* Enable interrupts */ - ssm PSW_SM_I, %r0 - /* Return to userspace, set no error */ - b lws_exit - copy %r0, %r21 +4: stw %r24, 0(%r26) + b,n lws_exit_noerror -3: - /* Error occurred on load or store */ - /* Free lock */ - stw,ma %r20, 0(%sr2,%r20) -#if ENABLE_LWS_DEBUG - stw %r0, 4(%sr2,%r20) -#endif - ssm PSW_SM_I, %r0 - b lws_exit - ldo -EFAULT(%r0),%r21 /* set errno */ - nop - nop - nop - nop + /* A fault occurred on load or stbys,e store */ +5: b,n lws_fault + ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 5b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 5b-linux_gateway_page) - /* Two exception table entries, one for the load, - the other for the store. Either return -EFAULT. - Each of the entries must be relocated. */ - ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 3b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 3b-linux_gateway_page) + /* A page fault occurred in critical region */ +6: b,n lws_pagefault + ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 6b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 6b-linux_gateway_page) /*************************************************** New CAS implementation which uses pointers and variable size information. The value pointed by old and new MUST NOT change - while performing CAS. The lock only protect the value at %r26. + while performing CAS. The lock only protects the value at %r26. %r26 - Address to examine %r25 - Pointer to the value to check (old) @@ -685,25 +670,32 @@ cas_action: %r28 - Return non-zero on failure %r21 - Kernel error code - %r21 has the following meanings: - + %r21 returns the following error codes: EAGAIN - CAS is busy, ldcw failed, try again. EFAULT - Read or write failed. + If EAGAIN is returned, %r28 indicates the busy reason: + r28 == 1 - CAS is busy. lock contended. + r28 == 2 - CAS is busy. ldcw failed. + r28 == 3 - CAS is busy. page fault. + Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only) ****************************************************/ - /* ELF32 Process entry path */ lws_compare_and_swap_2: #ifdef CONFIG_64BIT - /* Clip the input registers. We don't need to clip %r23 as we - only use it for word operations */ + /* Wide mode user process? */ + bb,<,n %sp, 31, cas2_begin + + /* Clip the input registers for 32-bit processes. We don't + need to clip %r23 as we only use it for word operations */ depdi 0, 31, 32, %r26 depdi 0, 31, 32, %r25 depdi 0, 31, 32, %r24 #endif +cas2_begin: /* Check the validity of the size pointer */ subi,>>= 3, %r23, %r0 b,n lws_exit_nosys @@ -714,71 +706,77 @@ lws_compare_and_swap_2: blr %r29, %r0 nop - /* 8bit load */ -4: ldb 0(%r25), %r25 + /* 8-bit load */ +1: ldb 0(%r25), %r25 b cas2_lock_start -5: ldb 0(%r24), %r24 +2: ldb 0(%r24), %r24 nop nop nop nop nop - /* 16bit load */ -6: ldh 0(%r25), %r25 + /* 16-bit load */ +3: ldh 0(%r25), %r25 b cas2_lock_start -7: ldh 0(%r24), %r24 +4: ldh 0(%r24), %r24 nop nop nop nop nop - /* 32bit load */ -8: ldw 0(%r25), %r25 + /* 32-bit load */ +5: ldw 0(%r25), %r25 b cas2_lock_start -9: ldw 0(%r24), %r24 +6: ldw 0(%r24), %r24 nop nop nop nop nop - /* 64bit load */ + /* 64-bit load */ #ifdef CONFIG_64BIT -10: ldd 0(%r25), %r25 -11: ldd 0(%r24), %r24 +7: ldd 0(%r25), %r25 +8: ldd 0(%r24), %r24 #else /* Load old value into r22/r23 - high/low */ -10: ldw 0(%r25), %r22 -11: ldw 4(%r25), %r23 +7: ldw 0(%r25), %r22 +8: ldw 4(%r25), %r23 /* Load new value into fr4 for atomic store later */ -12: flddx 0(%r24), %fr4 +9: flddx 0(%r24), %fr4 #endif cas2_lock_start: + /* Trigger memory reference interruptions without writing to memory */ + copy %r26, %r28 + depi_safe 0, 31, 2, %r28 +10: ldw 0(%r28), %r1 +11: stbys,e %r0, 0(%r28) + + /* Calculate 8-bit hash index from virtual address */ + extru_safe %r26, 27, 8, %r20 + /* Load start of lock table */ - ldil L%lws_lock_start, %r20 - ldo R%lws_lock_start(%r20), %r28 + ldil L%lws_lock_start, %r28 + ldo R%lws_lock_start(%r28), %r28 - /* Extract eight bits from r26 and hash lock (Bits 3-11) */ - extru %r26, 28, 8, %r20 - - /* Find lock to use, the hash is either one of 0 to - 15, multiplied by 16 (keep it 16-byte aligned) + /* Find lock to use, the hash index is one of 0 to + 255, multiplied by 16 (keep it 16-byte aligned) and add to the lock table offset. */ shlw %r20, 4, %r20 add %r20, %r28, %r20 rsm PSW_SM_I, %r0 /* Disable interrupts */ - /* COW breaks can cause contention on UP systems */ - LDCW 0(%sr2,%r20), %r28 /* Try to acquire the lock */ - cmpb,<>,n %r0, %r28, cas2_action /* Did we get it? */ -cas2_wouldblock: - ldo 2(%r0), %r28 /* 2nd case */ - ssm PSW_SM_I, %r0 - b lws_exit /* Contended... */ - ldo -EAGAIN(%r0), %r21 /* Spin in userspace */ + + /* Try to acquire the lock */ + LDCW 0(%sr2,%r20), %r28 + comclr,<> %r0, %r28, %r0 + b,n lws_wouldblock + + /* Disable page faults to prevent sleeping in critical region */ + lws_pagefault_disable %r21,%r28 /* prev = *addr; @@ -788,115 +786,493 @@ cas2_wouldblock: */ /* NOTES: - This all works becuse intr_do_signal + This all works because intr_do_signal and schedule both check the return iasq and see that we are on the kernel page so this process is never scheduled off or is ever sent any signal of any sort, - thus it is wholly atomic from usrspaces + thus it is wholly atomic from usrspace's perspective */ -cas2_action: + /* Jump to the correct function */ blr %r29, %r0 /* Set %r28 as non-zero for now */ ldo 1(%r0),%r28 - /* 8bit CAS */ -13: ldb 0(%r26), %r29 + /* 8-bit CAS */ +12: ldb 0(%r26), %r29 sub,= %r29, %r25, %r0 - b,n cas2_end -14: stb %r24, 0(%r26) - b cas2_end + b,n lws_exit_noerror +13: stb %r24, 0(%r26) + b lws_exit_noerror copy %r0, %r28 nop nop - /* 16bit CAS */ -15: ldh 0(%r26), %r29 + /* 16-bit CAS */ +14: ldh 0(%r26), %r29 sub,= %r29, %r25, %r0 - b,n cas2_end -16: sth %r24, 0(%r26) - b cas2_end + b,n lws_exit_noerror +15: sth %r24, 0(%r26) + b lws_exit_noerror copy %r0, %r28 nop nop - /* 32bit CAS */ -17: ldw 0(%r26), %r29 + /* 32-bit CAS */ +16: ldw 0(%r26), %r29 sub,= %r29, %r25, %r0 - b,n cas2_end -18: stw %r24, 0(%r26) - b cas2_end + b,n lws_exit_noerror +17: stw %r24, 0(%r26) + b lws_exit_noerror copy %r0, %r28 nop nop - /* 64bit CAS */ + /* 64-bit CAS */ #ifdef CONFIG_64BIT -19: ldd 0(%r26), %r29 +18: ldd 0(%r26), %r29 sub,*= %r29, %r25, %r0 - b,n cas2_end -20: std %r24, 0(%r26) + b,n lws_exit_noerror +19: std %r24, 0(%r26) copy %r0, %r28 #else /* Compare first word */ -19: ldw 0(%r26), %r29 +18: ldw 0(%r26), %r29 sub,= %r29, %r22, %r0 - b,n cas2_end + b,n lws_exit_noerror /* Compare second word */ -20: ldw 4(%r26), %r29 +19: ldw 4(%r26), %r29 sub,= %r29, %r23, %r0 - b,n cas2_end + b,n lws_exit_noerror /* Perform the store */ -21: fstdx %fr4, 0(%r26) +20: fstdx %fr4, 0(%r26) copy %r0, %r28 #endif + b lws_exit_noerror + copy %r0, %r28 -cas2_end: - /* Free lock */ - stw,ma %r20, 0(%sr2,%r20) - /* Enable interrupts */ - ssm PSW_SM_I, %r0 - /* Return to userspace, set no error */ - b lws_exit - copy %r0, %r21 - -22: - /* Error occurred on load or store */ - /* Free lock */ - stw,ma %r20, 0(%sr2,%r20) - ssm PSW_SM_I, %r0 - ldo 1(%r0),%r28 - b lws_exit - ldo -EFAULT(%r0),%r21 /* set errno */ - nop - nop - nop - - /* Exception table entries, for the load and store, return EFAULT. - Each of the entries must be relocated. */ - ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 22b-linux_gateway_page) + /* A fault occurred on load or stbys,e store */ +30: b,n lws_fault + ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 30b-linux_gateway_page) #ifndef CONFIG_64BIT - ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 22b-linux_gateway_page) - ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 22b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 30b-linux_gateway_page) #endif + ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 30b-linux_gateway_page) + + /* A page fault occurred in critical region */ +31: b,n lws_pagefault + ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 31b-linux_gateway_page) +#ifndef CONFIG_64BIT + ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 31b-linux_gateway_page) +#endif + + + /*************************************************** + LWS atomic exchange. + + %r26 - Exchange address + %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) + %r24 - Address of new value + %r23 - Address of old value + %r28 - Return non-zero on failure + %r21 - Kernel error code + + %r21 returns the following error codes: + EAGAIN - CAS is busy, ldcw failed, try again. + EFAULT - Read or write failed. + + If EAGAIN is returned, %r28 indicates the busy reason: + r28 == 1 - CAS is busy. lock contended. + r28 == 2 - CAS is busy. ldcw failed. + r28 == 3 - CAS is busy. page fault. + + Scratch: r20, r1 + + ****************************************************/ + +lws_atomic_xchg: +#ifdef CONFIG_64BIT + /* Wide mode user process? */ + bb,<,n %sp, 31, atomic_xchg_begin + + /* Clip the input registers for 32-bit processes. We don't + need to clip %r23 as we only use it for word operations */ + depdi 0, 31, 32, %r26 + depdi 0, 31, 32, %r25 + depdi 0, 31, 32, %r24 + depdi 0, 31, 32, %r23 +#endif + +atomic_xchg_begin: + /* Check the validity of the size pointer */ + subi,>>= 3, %r25, %r0 + b,n lws_exit_nosys + + /* Jump to the functions which will load the old and new values into + registers depending on the their size */ + shlw %r25, 2, %r1 + blr %r1, %r0 + nop + + /* Perform exception checks */ + + /* 8-bit exchange */ +1: ldb 0(%r24), %r20 + copy %r23, %r20 + depi_safe 0, 31, 2, %r20 + b atomic_xchg_start +2: stbys,e %r0, 0(%r20) + nop + nop + nop + + /* 16-bit exchange */ +3: ldh 0(%r24), %r20 + copy %r23, %r20 + depi_safe 0, 31, 2, %r20 + b atomic_xchg_start +4: stbys,e %r0, 0(%r20) + nop + nop + nop + + /* 32-bit exchange */ +5: ldw 0(%r24), %r20 + b atomic_xchg_start +6: stbys,e %r0, 0(%r23) + nop + nop + nop + nop + nop + + /* 64-bit exchange */ +#ifdef CONFIG_64BIT +7: ldd 0(%r24), %r20 +8: stdby,e %r0, 0(%r23) +#else +7: ldw 0(%r24), %r20 +8: ldw 4(%r24), %r20 + copy %r23, %r20 + depi_safe 0, 31, 2, %r20 +9: stbys,e %r0, 0(%r20) +10: stbys,e %r0, 4(%r20) +#endif + +atomic_xchg_start: + /* Trigger memory reference interruptions without writing to memory */ + copy %r26, %r28 + depi_safe 0, 31, 2, %r28 +11: ldw 0(%r28), %r1 +12: stbys,e %r0, 0(%r28) + + /* Calculate 8-bit hash index from virtual address */ + extru_safe %r26, 27, 8, %r20 + + /* Load start of lock table */ + ldil L%lws_lock_start, %r28 + ldo R%lws_lock_start(%r28), %r28 + + /* Find lock to use, the hash index is one of 0 to + 255, multiplied by 16 (keep it 16-byte aligned) + and add to the lock table offset. */ + shlw %r20, 4, %r20 + add %r20, %r28, %r20 + + rsm PSW_SM_I, %r0 /* Disable interrupts */ + + /* Try to acquire the lock */ + LDCW 0(%sr2,%r20), %r28 + comclr,<> %r0, %r28, %r0 + b,n lws_wouldblock + + /* Disable page faults to prevent sleeping in critical region */ + lws_pagefault_disable %r21,%r28 + + /* NOTES: + This all works because intr_do_signal + and schedule both check the return iasq + and see that we are on the kernel page + so this process is never scheduled off + or is ever sent any signal of any sort, + thus it is wholly atomic from userspace's + perspective + */ + + /* Jump to the correct function */ + blr %r1, %r0 + /* Set %r28 as non-zero for now */ + ldo 1(%r0),%r28 + + /* 8-bit exchange */ +14: ldb 0(%r26), %r1 +15: stb %r1, 0(%r23) +15: ldb 0(%r24), %r1 +17: stb %r1, 0(%r26) + b lws_exit_noerror + copy %r0, %r28 + nop + nop + + /* 16-bit exchange */ +18: ldh 0(%r26), %r1 +19: sth %r1, 0(%r23) +20: ldh 0(%r24), %r1 +21: sth %r1, 0(%r26) + b lws_exit_noerror + copy %r0, %r28 + nop + nop + + /* 32-bit exchange */ +22: ldw 0(%r26), %r1 +23: stw %r1, 0(%r23) +24: ldw 0(%r24), %r1 +25: stw %r1, 0(%r26) + b lws_exit_noerror + copy %r0, %r28 + nop + nop + + /* 64-bit exchange */ +#ifdef CONFIG_64BIT +26: ldd 0(%r26), %r1 +27: std %r1, 0(%r23) +28: ldd 0(%r24), %r1 +29: std %r1, 0(%r26) +#else +26: flddx 0(%r26), %fr4 +27: fstdx %fr4, 0(%r23) +28: flddx 0(%r24), %fr4 +29: fstdx %fr4, 0(%r26) +#endif + b lws_exit_noerror + copy %r0, %r28 + + /* A fault occurred on load or stbys,e store */ +30: b,n lws_fault + ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 30b-linux_gateway_page) +#ifndef CONFIG_64BIT + ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 30b-linux_gateway_page) +#endif + + ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 30b-linux_gateway_page) + + /* A page fault occurred in critical region */ +31: b,n lws_pagefault + ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(22b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(23b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(24b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(25b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(26b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(27b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(28b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(29b-linux_gateway_page, 31b-linux_gateway_page) + + /*************************************************** + LWS atomic store. + + %r26 - Address to store + %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) + %r24 - Address of value to store + %r28 - Return non-zero on failure + %r21 - Kernel error code + + %r21 returns the following error codes: + EAGAIN - CAS is busy, ldcw failed, try again. + EFAULT - Read or write failed. + + If EAGAIN is returned, %r28 indicates the busy reason: + r28 == 1 - CAS is busy. lock contended. + r28 == 2 - CAS is busy. ldcw failed. + r28 == 3 - CAS is busy. page fault. + + Scratch: r20, r1 + + ****************************************************/ + +lws_atomic_store: +#ifdef CONFIG_64BIT + /* Wide mode user process? */ + bb,<,n %sp, 31, atomic_store_begin + + /* Clip the input registers for 32-bit processes. We don't + need to clip %r23 as we only use it for word operations */ + depdi 0, 31, 32, %r26 + depdi 0, 31, 32, %r25 + depdi 0, 31, 32, %r24 +#endif + +atomic_store_begin: + /* Check the validity of the size pointer */ + subi,>>= 3, %r25, %r0 + b,n lws_exit_nosys + + shlw %r25, 1, %r1 + blr %r1, %r0 + nop + + /* Perform exception checks */ + + /* 8-bit store */ +1: ldb 0(%r24), %r20 + b,n atomic_store_start + nop + nop + + /* 16-bit store */ +2: ldh 0(%r24), %r20 + b,n atomic_store_start + nop + nop + + /* 32-bit store */ +3: ldw 0(%r24), %r20 + b,n atomic_store_start + nop + nop + + /* 64-bit store */ +#ifdef CONFIG_64BIT +4: ldd 0(%r24), %r20 +#else +4: ldw 0(%r24), %r20 +5: ldw 4(%r24), %r20 +#endif + +atomic_store_start: + /* Trigger memory reference interruptions without writing to memory */ + copy %r26, %r28 + depi_safe 0, 31, 2, %r28 +6: ldw 0(%r28), %r1 +7: stbys,e %r0, 0(%r28) + + /* Calculate 8-bit hash index from virtual address */ + extru_safe %r26, 27, 8, %r20 + + /* Load start of lock table */ + ldil L%lws_lock_start, %r28 + ldo R%lws_lock_start(%r28), %r28 + + /* Find lock to use, the hash index is one of 0 to + 255, multiplied by 16 (keep it 16-byte aligned) + and add to the lock table offset. */ + shlw %r20, 4, %r20 + add %r20, %r28, %r20 + + rsm PSW_SM_I, %r0 /* Disable interrupts */ + + /* Try to acquire the lock */ + LDCW 0(%sr2,%r20), %r28 + comclr,<> %r0, %r28, %r0 + b,n lws_wouldblock + + /* Disable page faults to prevent sleeping in critical region */ + lws_pagefault_disable %r21,%r28 + + /* NOTES: + This all works because intr_do_signal + and schedule both check the return iasq + and see that we are on the kernel page + so this process is never scheduled off + or is ever sent any signal of any sort, + thus it is wholly atomic from userspace's + perspective + */ + + /* Jump to the correct function */ + blr %r1, %r0 + /* Set %r28 as non-zero for now */ + ldo 1(%r0),%r28 + + /* 8-bit store */ +9: ldb 0(%r24), %r1 +10: stb %r1, 0(%r26) + b lws_exit_noerror + copy %r0, %r28 + + /* 16-bit store */ +11: ldh 0(%r24), %r1 +12: sth %r1, 0(%r26) + b lws_exit_noerror + copy %r0, %r28 + + /* 32-bit store */ +13: ldw 0(%r24), %r1 +14: stw %r1, 0(%r26) + b lws_exit_noerror + copy %r0, %r28 + + /* 64-bit store */ +#ifdef CONFIG_64BIT +15: ldd 0(%r24), %r1 +16: std %r1, 0(%r26) +#else +15: flddx 0(%r24), %fr4 +16: fstdx %fr4, 0(%r26) +#endif + b lws_exit_noerror + copy %r0, %r28 + + /* A fault occurred on load or stbys,e store */ +30: b,n lws_fault + ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) +#ifndef CONFIG_64BIT + ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) +#endif + + ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) + + /* A page fault occurred in critical region */ +31: b,n lws_pagefault + ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) + ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) + /* Make sure nothing else is placed on this page */ .align PAGE_SIZE END(linux_gateway_page) @@ -915,7 +1291,9 @@ ENTRY(end_linux_gateway_page) ENTRY(lws_table) LWS_ENTRY(compare_and_swap32) /* 0 - ELF32 Atomic 32bit CAS */ LWS_ENTRY(compare_and_swap64) /* 1 - ELF64 Atomic 32bit CAS */ - LWS_ENTRY(compare_and_swap_2) /* 2 - ELF32 Atomic 64bit CAS */ + LWS_ENTRY(compare_and_swap_2) /* 2 - Atomic 64bit CAS */ + LWS_ENTRY(atomic_xchg) /* 3 - Atomic Exchange */ + LWS_ENTRY(atomic_store) /* 4 - Atomic Store */ END(lws_table) /* End of lws table */ diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index bf751e0732..68b46fe2f1 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -446,3 +446,5 @@ 446 common landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index afe8b902a8..b6fdebddc8 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -143,7 +144,7 @@ void show_regs(struct pt_regs *regs) printk("%s IIR: %08lx ISR: " RFMT " IOR: " RFMT "\n", level, regs->iir, regs->isr, regs->ior); printk("%s CPU: %8d CR30: " RFMT " CR31: " RFMT "\n", - level, current_thread_info()->cpu, cr30, cr31); + level, task_cpu(current), cr30, cr31); printk("%s ORIG_R28: " RFMT "\n", level, regs->orig_r28); if (user) { @@ -268,7 +269,7 @@ void die_if_kernel(char *str, struct pt_regs *regs, long err) panic("Fatal exception"); oops_exit(); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } /* gdb uses break 4,8 */ @@ -480,7 +481,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs) if (code == 1) pdc_console_restart(); /* switch back to pdc if HPMC */ - else + else if (!irqs_disabled_flags(regs->gr[0])) local_irq_enable(); /* Security check: @@ -789,6 +790,10 @@ void notrace handle_interruption(int code, struct pt_regs *regs) /* Clean up and return if in exception table. */ if (fixup_exception(regs)) return; + /* Clean up and return if handled by kfence. */ + if (kfence_handle_page_fault(fault_address, + parisc_acctyp(code, regs->iir) == VM_WRITE, regs)) + return; pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC); parisc_terminate("Kernel Fault", regs, code, fault_address); } diff --git a/arch/parisc/kernel/unwind.c b/arch/parisc/kernel/unwind.c index 86a57fb0e6..42acc3b520 100644 --- a/arch/parisc/kernel/unwind.c +++ b/arch/parisc/kernel/unwind.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -299,17 +300,15 @@ static void unwind_frame_regs(struct unwind_frame_info *info) info->prev_sp = sp - 64; info->prev_ip = 0; - /* The stack is at the end inside the thread_union - * struct. If we reach data, we have reached the - * beginning of the stack and should stop unwinding. */ - if (info->prev_sp >= (unsigned long) task_thread_info(info->t) && - info->prev_sp < ((unsigned long) task_thread_info(info->t) - + THREAD_SZ_ALGN)) { + /* Check if stack is inside kernel stack area */ + if ((info->prev_sp - (unsigned long) task_stack_page(info->t)) + >= THREAD_SIZE) { info->prev_sp = 0; break; } - if (get_user(tmp, (unsigned long *)(info->prev_sp - RP_OFFSET))) + if (copy_from_kernel_nofault(&tmp, + (void *)info->prev_sp - RP_OFFSET, sizeof(tmp))) break; info->prev_ip = tmp; sp = info->prev_sp; diff --git a/arch/parisc/lib/bitops.c b/arch/parisc/lib/bitops.c index 9ac683bf6a..36a3141990 100644 --- a/arch/parisc/lib/bitops.c +++ b/arch/parisc/lib/bitops.c @@ -18,7 +18,7 @@ arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned = { #endif #ifdef CONFIG_64BIT -unsigned long __xchg64(unsigned long x, volatile unsigned long *ptr) +unsigned long notrace __xchg64(unsigned long x, volatile unsigned long *ptr) { unsigned long temp, flags; @@ -30,7 +30,7 @@ unsigned long __xchg64(unsigned long x, volatile unsigned long *ptr) } #endif -unsigned long __xchg32(int x, volatile int *ptr) +unsigned long notrace __xchg32(int x, volatile int *ptr) { unsigned long flags; long temp; @@ -43,7 +43,7 @@ unsigned long __xchg32(int x, volatile int *ptr) } -unsigned long __xchg8(char x, volatile char *ptr) +unsigned long notrace __xchg8(char x, volatile char *ptr) { unsigned long flags; long temp; @@ -56,7 +56,7 @@ unsigned long __xchg8(char x, volatile char *ptr) } -u64 __cmpxchg_u64(volatile u64 *ptr, u64 old, u64 new) +u64 notrace __cmpxchg_u64(volatile u64 *ptr, u64 old, u64 new) { unsigned long flags; u64 prev; @@ -68,7 +68,7 @@ u64 __cmpxchg_u64(volatile u64 *ptr, u64 old, u64 new) return prev; } -unsigned long __cmpxchg_u32(volatile unsigned int *ptr, unsigned int old, unsigned int new) +unsigned long notrace __cmpxchg_u32(volatile unsigned int *ptr, unsigned int old, unsigned int new) { unsigned long flags; unsigned int prev; @@ -80,7 +80,7 @@ unsigned long __cmpxchg_u32(volatile unsigned int *ptr, unsigned int old, unsign return (unsigned long)prev; } -u8 __cmpxchg_u8(volatile u8 *ptr, u8 old, u8 new) +u8 notrace __cmpxchg_u8(volatile u8 *ptr, u8 old, u8 new) { unsigned long flags; u8 prev; diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index 716960f5d9..e9eabf8f14 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -48,7 +48,7 @@ int show_unhandled_signals = 1; * VM_WRITE if write operation * VM_EXEC if execute operation */ -static unsigned long +unsigned long parisc_acctyp(unsigned long code, unsigned int inst) { if (code == 6 || code == 16) @@ -148,11 +148,11 @@ int fixup_exception(struct pt_regs *regs) * Fix up get_user() and put_user(). * ASM_EXCEPTIONTABLE_ENTRY_EFAULT() sets the least-significant * bit in the relative address of the fixup routine to indicate - * that %r8 should be loaded with -EFAULT to report a userspace - * access error. + * that gr[ASM_EXCEPTIONTABLE_REG] should be loaded with + * -EFAULT to report a userspace access error. */ if (fix->fixup & 1) { - regs->gr[8] = -EFAULT; + regs->gr[ASM_EXCEPTIONTABLE_REG] = -EFAULT; /* zero target register for get_user() */ if (parisc_acctyp(0, regs->iir) == VM_READ) { @@ -266,14 +266,14 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, unsigned long acc_type; vm_fault_t fault = 0; unsigned int flags; - - if (faulthandler_disabled()) - goto no_context; + char *msg; tsk = current; mm = tsk->mm; - if (!mm) + if (!mm) { + msg = "Page fault: no context"; goto no_context; + } flags = FAULT_FLAG_DEFAULT; if (user_mode(regs)) @@ -324,16 +324,14 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, goto bad_area; BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - /* - * No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ - flags |= FAULT_FLAG_TRIED; - goto retry; - } + if (fault & VM_FAULT_RETRY) { + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + flags |= FAULT_FLAG_TRIED; + goto retry; } mmap_read_unlock(mm); return; @@ -409,6 +407,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, force_sig_fault(signo, si_code, (void __user *) address); return; } + msg = "Page fault: bad address"; no_context: @@ -416,11 +415,13 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, return; } - parisc_terminate("Bad Address (null pointer deref?)", regs, code, address); + parisc_terminate(msg, regs, code, address); - out_of_memory: +out_of_memory: mmap_read_unlock(mm); - if (!user_mode(regs)) + if (!user_mode(regs)) { + msg = "Page fault: out of memory"; goto no_context; + } pagefault_out_of_memory(); } diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c index e5c18313b5..1dc2e88e7b 100644 --- a/arch/parisc/mm/init.c +++ b/arch/parisc/mm/init.c @@ -127,16 +127,12 @@ static void __init setup_bootmem(void) int j; for (j = i; j > 0; j--) { - physmem_range_t tmp; - if (pmem_ranges[j-1].start_pfn < pmem_ranges[j].start_pfn) { break; } - tmp = pmem_ranges[j-1]; - pmem_ranges[j-1] = pmem_ranges[j]; - pmem_ranges[j] = tmp; + swap(pmem_ranges[j-1], pmem_ranges[j]); } } diff --git a/arch/powerpc/Kbuild b/arch/powerpc/Kbuild index 5e2f9eaa3e..22cd0d55a8 100644 --- a/arch/powerpc/Kbuild +++ b/arch/powerpc/Kbuild @@ -16,3 +16,6 @@ obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_PERF_EVENTS) += perf/ obj-$(CONFIG_KEXEC_CORE) += kexec/ obj-$(CONFIG_KEXEC_FILE) += purgatory/ + +# for cleaning +subdir- += boot diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 6b9f523882..b779603978 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -55,15 +55,6 @@ config ARCH_MMAP_RND_COMPAT_BITS_MIN default 9 if PPC_16K_PAGES # 9 = 23 (8MB) - 14 (16K) default 11 # 11 = 23 (8MB) - 12 (4K) -config HAVE_SETUP_PER_CPU_AREA - def_bool PPC64 - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y if PPC64 - -config NEED_PER_CPU_PAGE_FIRST_CHUNK - def_bool y if PPC64 - config NR_IRQS int "Number of virtual interrupt numbers" range 32 1048576 @@ -129,7 +120,7 @@ config PPC select ARCH_HAS_KCOV select ARCH_HAS_MEMBARRIER_CALLBACKS select ARCH_HAS_MEMBARRIER_SYNC_CORE - select ARCH_HAS_MEMREMAP_COMPAT_ALIGN + select ARCH_HAS_MEMREMAP_COMPAT_ALIGN if PPC_64S_HASH_MMU select ARCH_HAS_MMIOWB if PPC64 select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE select ARCH_HAS_PHYS_TO_DMA @@ -139,6 +130,7 @@ config PPC select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64 select ARCH_HAS_SET_MEMORY select ARCH_HAS_STRICT_KERNEL_RWX if (PPC_BOOK3S || PPC_8xx || 40x) && !HIBERNATION + select ARCH_HAS_STRICT_KERNEL_RWX if FSL_BOOKE && !HIBERNATION && !RANDOMIZE_BASE select ARCH_HAS_STRICT_MODULE_RWX if ARCH_HAS_STRICT_KERNEL_RWX && !PPC_BOOK3S_32 select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UACCESS_FLUSHCACHE @@ -148,6 +140,7 @@ config PPC select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX + select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx || 40x @@ -163,6 +156,7 @@ config PPC select BINFMT_ELF select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS + select CPUMASK_OFFSTACK if NR_CPUS >= 8192 select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN select DMA_OPS_BYPASS if PPC64 select DMA_OPS if PPC64 @@ -203,7 +197,7 @@ config PPC select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_STACKOVERFLOW select HAVE_DYNAMIC_FTRACE - select HAVE_DYNAMIC_FTRACE_WITH_REGS if MPROFILE_KERNEL + select HAVE_DYNAMIC_FTRACE_WITH_REGS if MPROFILE_KERNEL || PPC32 select HAVE_EBPF_JIT select HAVE_EFFICIENT_UNALIGNED_ACCESS if !(CPU_LITTLE_ENDIAN && POWER7_CPU) select HAVE_FAST_GUP @@ -227,7 +221,7 @@ config PPC select HAVE_KPROBES_ON_FTRACE select HAVE_KRETPROBES select HAVE_LD_DEAD_CODE_DATA_ELIMINATION - select HAVE_LIVEPATCH if HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_LIVEPATCH if HAVE_DYNAMIC_FTRACE_WITH_REGS && PPC64 select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI if PERF_EVENTS || (PPC64 && PPC_BOOK3S) select HAVE_OPTPROBES @@ -238,9 +232,11 @@ config PPC select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_RSEQ + select HAVE_SETUP_PER_CPU_AREA if PPC64 select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2) select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13) + select HAVE_STATIC_CALL if PPC32 select HAVE_SYSCALL_TRACEPOINTS select HAVE_VIRT_CPU_ACCOUNTING select HUGETLB_PAGE_SIZE_VARIABLE if PPC_BOOK3S_64 && HUGETLB_PAGE @@ -251,6 +247,8 @@ config PPC select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE if PPC64 || NOT_COHERENT_CACHE + select NEED_PER_CPU_EMBED_FIRST_CHUNK if PPC64 + select NEED_PER_CPU_PAGE_FIRST_CHUNK if PPC64 select NEED_SG_DMA_LENGTH select OF select OF_DMA_DEFAULT_COHERENT if !NOT_COHERENT_CACHE @@ -656,6 +654,7 @@ config NUMA bool "NUMA Memory Allocation and Scheduler Support" depends on PPC64 && SMP default y if PPC_PSERIES || PPC_POWERNV + select USE_PERCPU_NUMA_NODE_ID help Enable NUMA (Non-Uniform Memory Access) support. @@ -669,10 +668,6 @@ config NODES_SHIFT default "4" depends on NUMA -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - config HAVE_MEMORYLESS_NODES def_bool y depends on NUMA @@ -707,6 +702,7 @@ config ARCH_MEMORY_PROBE choice prompt "Page size" + default PPC_64K_PAGES if PPC_BOOK3S_64 default PPC_4K_PAGES help Select the kernel logical page size. Increasing the page size @@ -778,7 +774,8 @@ config DATA_SHIFT_BOOL bool "Set custom data alignment" depends on ADVANCED_OPTIONS depends on STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE - depends on PPC_BOOK3S_32 || (PPC_8xx && !PIN_TLB_DATA && !STRICT_KERNEL_RWX) + depends on PPC_BOOK3S_32 || (PPC_8xx && !PIN_TLB_DATA && !STRICT_KERNEL_RWX) || \ + FSL_BOOKE help This option allows you to set the kernel data alignment. When RAM is mapped by blocks, the alignment needs to fit the size and @@ -791,11 +788,13 @@ config DATA_SHIFT default 24 if STRICT_KERNEL_RWX && PPC64 range 17 28 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32 range 19 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_8xx + range 20 24 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_FSL_BOOKE default 22 if STRICT_KERNEL_RWX && PPC_BOOK3S_32 default 18 if (DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32 default 23 if STRICT_KERNEL_RWX && PPC_8xx default 23 if (DEBUG_PAGEALLOC || KFENCE) && PPC_8xx && PIN_TLB_DATA default 19 if (DEBUG_PAGEALLOC || KFENCE) && PPC_8xx + default 24 if STRICT_KERNEL_RWX && FSL_BOOKE default PPC_PAGE_SHIFT help On Book3S 32 (603+), DBATs are used to map kernel text and rodata RO. @@ -838,7 +837,7 @@ config FORCE_MAX_ZONEORDER config PPC_SUBPAGE_PROT bool "Support setting protections for 4k subpages (subpage_prot syscall)" default n - depends on PPC_BOOK3S_64 && PPC_64K_PAGES + depends on PPC_64S_HASH_MMU && PPC_64K_PAGES help This option adds support for system call to allow user programs to set access permissions (read/write, readonly, or no access) @@ -936,6 +935,7 @@ config PPC_MEM_KEYS prompt "PowerPC Memory Protection Keys" def_bool y depends on PPC_BOOK3S_64 + depends on PPC_64S_HASH_MMU select ARCH_USES_HIGH_VMA_FLAGS select ARCH_HAS_PKEYS help @@ -1123,7 +1123,10 @@ config LOWMEM_CAM_NUM_BOOL config LOWMEM_CAM_NUM depends on FSL_BOOKE int "Number of CAMs to use to map low memory" if LOWMEM_CAM_NUM_BOOL - default 3 + default 3 if !STRICT_KERNEL_RWX + default 9 if DATA_SHIFT >= 24 + default 12 if DATA_SHIFT >= 22 + default 15 config DYNAMIC_MEMSTART bool "Enable page aligned dynamic load address for kernel" diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index aa6808e706..5f16ac1583 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -1,7 +1,5 @@ # This file is included by the global makefile so that you can add your own -# architecture-specific flags and dependencies. Remember to do have actions -# for "archclean" and "archdep" for cleaning up and making dependencies for -# this architecture. +# architecture-specific flags and dependencies. # # This file is subject to the terms and conditions of the GNU General Public # License. See the file "COPYING" in the main directory of this archive @@ -247,7 +245,9 @@ cpu-as-$(CONFIG_E500) += -Wa,-me500 # When using '-many -mpower4' gas will first try and find a matching power4 # mnemonic and failing that it will allow any valid mnemonic that GAS knows # about. GCC will pass -many to GAS when assembling, clang does not. -cpu-as-$(CONFIG_PPC_BOOK3S_64) += -Wa,-mpower4 -Wa,-many +# LLVM IAS doesn't understand either flag: https://github.com/ClangBuiltLinux/linux/issues/675 +# but LLVM IAS only supports ISA >= 2.06 for Book3S 64 anyway... +cpu-as-$(CONFIG_PPC_BOOK3S_64) += $(call as-option,-Wa$(comma)-mpower4) $(call as-option,-Wa$(comma)-many) cpu-as-$(CONFIG_PPC_E500MC) += $(call as-option,-Wa$(comma)-me500mc) KBUILD_AFLAGS += $(cpu-as-y) @@ -411,9 +411,6 @@ install: sh -x $(srctree)/$(boot)/install.sh "$(KERNELRELEASE)" vmlinux \ System.map "$(INSTALL_PATH)" -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - ifeq ($(KBUILD_EXTMOD),) # We need to generate vdso-offsets.h before compiling certain files in kernel/. # In order to do that, we should use the archprepare target, but we can't since @@ -446,25 +443,15 @@ else endif endif -ifdef CONFIG_SMP -ifdef CONFIG_PPC32 -prepare: task_cpu_prepare - -PHONY += task_cpu_prepare -task_cpu_prepare: prepare0 - $(eval KBUILD_CFLAGS += -D_TASK_CPU=$(shell awk '{if ($$2 == "TASK_CPU") print $$3;}' include/generated/asm-offsets.h)) - -endif # CONFIG_PPC32 -endif # CONFIG_SMP - PHONY += checkbin # Check toolchain versions: # - gcc-4.6 is the minimum kernel-wide version so nothing required. checkbin: - @if test "x${CONFIG_CPU_LITTLE_ENDIAN}" = "xy" \ - && $(LD) --version | head -1 | grep ' 2\.24$$' >/dev/null ; then \ + @if test "x${CONFIG_LD_IS_LLD}" != "xy" -a \ + "x$(call ld-ifversion, -le, 22400, y)" = "xy" ; then \ echo -n '*** binutils 2.24 miscompiles weak symbols ' ; \ echo 'in some circumstances.' ; \ + echo '*** binutils 2.23 do not define the TOC symbol ' ; \ echo -n '*** Please use a different binutils version.' ; \ false ; \ fi diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile index 089ee3ea55..4b4827c475 100644 --- a/arch/powerpc/boot/Makefile +++ b/arch/powerpc/boot/Makefile @@ -123,7 +123,7 @@ src-wlib-y := string.S crt0.S stdio.c decompress.c main.c \ oflib.c ofconsole.c cuboot.c src-wlib-$(CONFIG_PPC_MPC52xx) += mpc52xx-psc.c -src-wlib-$(CONFIG_PPC64_BOOT_WRAPPER) += opal-calls.S opal.c +src-wlib-$(CONFIG_PPC_POWERNV) += opal-calls.S opal.c ifndef CONFIG_PPC64_BOOT_WRAPPER src-wlib-y += crtsavres.S endif @@ -365,7 +365,7 @@ image-$(CONFIG_PPC_PMAC) += zImage.coff zImage.miboot endif # Allow extra targets to be added to the defconfig -image-y += $(subst ",,$(CONFIG_EXTRA_TARGETS)) +image-y += $(CONFIG_EXTRA_TARGETS) initrd- := $(patsubst zImage%, zImage.initrd%, $(image-)) initrd-y := $(patsubst zImage%, zImage.initrd%, \ diff --git a/arch/powerpc/boot/crt0.S b/arch/powerpc/boot/crt0.S index 1d83966f5e..feadee18e2 100644 --- a/arch/powerpc/boot/crt0.S +++ b/arch/powerpc/boot/crt0.S @@ -28,7 +28,7 @@ p_etext: .8byte _etext p_bss_start: .8byte __bss_start p_end: .8byte _end -p_toc: .8byte __toc_start + 0x8000 - p_base +p_toc: .8byte .TOC. - p_base p_dyn: .8byte __dynamic_start - p_base p_rela: .8byte __rela_dyn_start - p_base p_prom: .8byte 0 @@ -226,16 +226,19 @@ p_base: mflr r10 /* r10 now points to runtime addr of p_base */ #ifdef __powerpc64__ #define PROM_FRAME_SIZE 512 -#define SAVE_GPR(n, base) std n,8*(n)(base) -#define REST_GPR(n, base) ld n,8*(n)(base) -#define SAVE_2GPRS(n, base) SAVE_GPR(n, base); SAVE_GPR(n+1, base) -#define SAVE_4GPRS(n, base) SAVE_2GPRS(n, base); SAVE_2GPRS(n+2, base) -#define SAVE_8GPRS(n, base) SAVE_4GPRS(n, base); SAVE_4GPRS(n+4, base) -#define SAVE_10GPRS(n, base) SAVE_8GPRS(n, base); SAVE_2GPRS(n+8, base) -#define REST_2GPRS(n, base) REST_GPR(n, base); REST_GPR(n+1, base) -#define REST_4GPRS(n, base) REST_2GPRS(n, base); REST_2GPRS(n+2, base) -#define REST_8GPRS(n, base) REST_4GPRS(n, base); REST_4GPRS(n+4, base) -#define REST_10GPRS(n, base) REST_8GPRS(n, base); REST_2GPRS(n+8, base) + +.macro OP_REGS op, width, start, end, base, offset + .Lreg=\start + .rept (\end - \start + 1) + \op .Lreg,\offset+\width*.Lreg(\base) + .Lreg=.Lreg+1 + .endr +.endm + +#define SAVE_GPRS(start, end, base) OP_REGS std, 8, start, end, base, 0 +#define REST_GPRS(start, end, base) OP_REGS ld, 8, start, end, base, 0 +#define SAVE_GPR(n, base) SAVE_GPRS(n, n, base) +#define REST_GPR(n, base) REST_GPRS(n, n, base) /* prom handles the jump into and return from firmware. The prom args pointer is loaded in r3. */ @@ -246,9 +249,7 @@ prom: stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */ SAVE_GPR(2, r1) - SAVE_GPR(13, r1) - SAVE_8GPRS(14, r1) - SAVE_10GPRS(22, r1) + SAVE_GPRS(13, 31, r1) mfcr r10 std r10,8*32(r1) mfmsr r10 @@ -283,9 +284,7 @@ prom: /* Restore other registers */ REST_GPR(2, r1) - REST_GPR(13, r1) - REST_8GPRS(14, r1) - REST_10GPRS(22, r1) + REST_GPRS(13, 31, r1) ld r10,8*32(r1) mtcr r10 diff --git a/arch/powerpc/boot/dts/a4m072.dts b/arch/powerpc/boot/dts/a4m072.dts index a9cef57264..d4270a2ec6 100644 --- a/arch/powerpc/boot/dts/a4m072.dts +++ b/arch/powerpc/boot/dts/a4m072.dts @@ -140,8 +140,8 @@ pci@f0000d00 { clock-frequency = <0>; /* From boot loader */ interrupts = <2 8 0 2 9 0 2 10 0>; bus-range = <0 0>; - ranges = <0x42000000 0 0x80000000 0x80000000 0 0x10000000 - 0x02000000 0 0x90000000 0x90000000 0 0x10000000 - 0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x10000000>, + <0x02000000 0 0x90000000 0x90000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; }; }; diff --git a/arch/powerpc/boot/dts/bluestone.dts b/arch/powerpc/boot/dts/bluestone.dts index aa1ae94cd7..6971595319 100644 --- a/arch/powerpc/boot/dts/bluestone.dts +++ b/arch/powerpc/boot/dts/bluestone.dts @@ -366,30 +366,5 @@ PCIE0: pcie@d00000000 { 0x0 0x0 0x0 0x3 &UIC3 0xe 0x4 /* swizzled int C */ 0x0 0x0 0x0 0x4 &UIC3 0xf 0x4 /* swizzled int D */>; }; - - MSI: ppc4xx-msi@C10000000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = < 0xC 0x10000000 0x100 - 0xC 0x10000000 0x100>; - sdr-base = <0x36C>; - msi-data = <0x00004440>; - msi-mask = <0x0000ffe0>; - interrupts =<0 1 2 3 4 5 6 7>; - interrupt-parent = <&MSI>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - msi-available-ranges = <0x0 0x100>; - interrupt-map = < - 0 &UIC3 0x18 1 - 1 &UIC3 0x19 1 - 2 &UIC3 0x1A 1 - 3 &UIC3 0x1B 1 - 4 &UIC3 0x1C 1 - 5 &UIC3 0x1D 1 - 6 &UIC3 0x1E 1 - 7 &UIC3 0x1F 1 - >; - }; }; }; diff --git a/arch/powerpc/boot/dts/canyonlands.dts b/arch/powerpc/boot/dts/canyonlands.dts index c5fbb08e0a..5db1bff6b2 100644 --- a/arch/powerpc/boot/dts/canyonlands.dts +++ b/arch/powerpc/boot/dts/canyonlands.dts @@ -544,23 +544,5 @@ PCIE1: pcie@d20000000 { 0x0 0x0 0x0 0x3 &UIC3 0x12 0x4 /* swizzled int C */ 0x0 0x0 0x0 0x4 &UIC3 0x13 0x4 /* swizzled int D */>; }; - - MSI: ppc4xx-msi@C10000000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = < 0xC 0x10000000 0x100>; - sdr-base = <0x36C>; - msi-data = <0x00000000>; - msi-mask = <0x44440000>; - interrupt-count = <3>; - interrupts = <0 1 2 3>; - interrupt-parent = <&UIC3>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - interrupt-map = <0 &UIC3 0x18 1 - 1 &UIC3 0x19 1 - 2 &UIC3 0x1A 1 - 3 &UIC3 0x1B 1>; - }; }; }; diff --git a/arch/powerpc/boot/dts/charon.dts b/arch/powerpc/boot/dts/charon.dts index cd589539f3..ea6e76ae25 100644 --- a/arch/powerpc/boot/dts/charon.dts +++ b/arch/powerpc/boot/dts/charon.dts @@ -225,8 +225,8 @@ pci@f0000d00 { clock-frequency = <0>; // From boot loader interrupts = <2 8 0 2 9 0 2 10 0>; bus-range = <0 0>; - ranges = <0x42000000 0 0x80000000 0x80000000 0 0x10000000 - 0x02000000 0 0x90000000 0x90000000 0 0x10000000 - 0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x10000000>, + <0x02000000 0 0x90000000 0x90000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; }; }; diff --git a/arch/powerpc/boot/dts/digsy_mtc.dts b/arch/powerpc/boot/dts/digsy_mtc.dts index 19a14e62e6..dfaf974c0c 100644 --- a/arch/powerpc/boot/dts/digsy_mtc.dts +++ b/arch/powerpc/boot/dts/digsy_mtc.dts @@ -25,14 +25,6 @@ rtc@800 { status = "disabled"; }; - spi@f00 { - msp430@0 { - compatible = "spidev"; - spi-max-frequency = <32000>; - reg = <0>; - }; - }; - psc@2000 { // PSC1 status = "disabled"; }; @@ -98,9 +90,9 @@ pci@f0000d00 { clock-frequency = <0>; // From boot loader interrupts = <2 8 0 2 9 0 2 10 0>; bus-range = <0 0>; - ranges = <0x42000000 0 0x80000000 0x80000000 0 0x10000000 - 0x02000000 0 0x90000000 0x90000000 0 0x10000000 - 0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x10000000>, + <0x02000000 0 0x90000000 0x90000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; }; localbus { diff --git a/arch/powerpc/boot/dts/katmai.dts b/arch/powerpc/boot/dts/katmai.dts index a8f353229f..4262b2bbd6 100644 --- a/arch/powerpc/boot/dts/katmai.dts +++ b/arch/powerpc/boot/dts/katmai.dts @@ -442,24 +442,6 @@ PCIE2: pcie@d40000000 { 0x0 0x0 0x0 0x4 &UIC3 0xb 0x4 /* swizzled int D */>; }; - MSI: ppc4xx-msi@400300000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = < 0x4 0x00300000 0x100>; - sdr-base = <0x3B0>; - msi-data = <0x00000000>; - msi-mask = <0x44440000>; - interrupt-count = <3>; - interrupts =<0 1 2 3>; - interrupt-parent = <&UIC0>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - interrupt-map = <0 &UIC0 0xC 1 - 1 &UIC0 0x0D 1 - 2 &UIC0 0x0E 1 - 3 &UIC0 0x0F 1>; - }; - I2O: i2o@400100000 { compatible = "ibm,i2o-440spe"; reg = <0x00000004 0x00100000 0x100>; diff --git a/arch/powerpc/boot/dts/kilauea.dts b/arch/powerpc/boot/dts/kilauea.dts index a709fb47a1..c07a7525a7 100644 --- a/arch/powerpc/boot/dts/kilauea.dts +++ b/arch/powerpc/boot/dts/kilauea.dts @@ -403,33 +403,5 @@ PCIE1: pcie@c0000000 { 0x0 0x0 0x0 0x3 &UIC2 0xd 0x4 /* swizzled int C */ 0x0 0x0 0x0 0x4 &UIC2 0xe 0x4 /* swizzled int D */>; }; - - MSI: ppc4xx-msi@C10000000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = <0xEF620000 0x100>; - sdr-base = <0x4B0>; - msi-data = <0x00000000>; - msi-mask = <0x44440000>; - interrupt-count = <12>; - interrupts = <0 1 2 3 4 5 6 7 8 9 0xA 0xB 0xC 0xD>; - interrupt-parent = <&UIC2>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - interrupt-map = <0 &UIC2 0x10 1 - 1 &UIC2 0x11 1 - 2 &UIC2 0x12 1 - 2 &UIC2 0x13 1 - 2 &UIC2 0x14 1 - 2 &UIC2 0x15 1 - 2 &UIC2 0x16 1 - 2 &UIC2 0x17 1 - 2 &UIC2 0x18 1 - 2 &UIC2 0x19 1 - 2 &UIC2 0x1A 1 - 2 &UIC2 0x1B 1 - 2 &UIC2 0x1C 1 - 3 &UIC2 0x1D 1>; - }; }; }; diff --git a/arch/powerpc/boot/dts/lite5200.dts b/arch/powerpc/boot/dts/lite5200.dts index e7b194775d..b9d8487813 100644 --- a/arch/powerpc/boot/dts/lite5200.dts +++ b/arch/powerpc/boot/dts/lite5200.dts @@ -283,9 +283,9 @@ pci@f0000d00 { clock-frequency = <0>; // From boot loader interrupts = <2 8 0 2 9 0 2 10 0>; bus-range = <0 0>; - ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000 - 0x02000000 0 0xa0000000 0xa0000000 0 0x10000000 - 0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000>, + <0x02000000 0 0xa0000000 0xa0000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; }; localbus { diff --git a/arch/powerpc/boot/dts/lite5200b.dts b/arch/powerpc/boot/dts/lite5200b.dts index 547cbe726f..7e2d91c7cb 100644 --- a/arch/powerpc/boot/dts/lite5200b.dts +++ b/arch/powerpc/boot/dts/lite5200b.dts @@ -116,9 +116,9 @@ pci@f0000d00 { clock-frequency = <0>; // From boot loader interrupts = <2 8 0 2 9 0 2 10 0>; bus-range = <0 0>; - ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000 - 0x02000000 0 0xa0000000 0xa0000000 0 0x10000000 - 0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000>, + <0x02000000 0 0xa0000000 0xa0000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; }; localbus { diff --git a/arch/powerpc/boot/dts/media5200.dts b/arch/powerpc/boot/dts/media5200.dts index f3188018fa..96524ede16 100644 --- a/arch/powerpc/boot/dts/media5200.dts +++ b/arch/powerpc/boot/dts/media5200.dts @@ -96,9 +96,9 @@ pci@f0000d00 { 0xe000 0 0 1 &media5200_fpga 0 5 // CoralIP >; - ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000 - 0x02000000 0 0xa0000000 0xa0000000 0 0x10000000 - 0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000>, + <0x02000000 0 0xa0000000 0xa0000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; interrupt-parent = <&mpc5200_pic>; }; diff --git a/arch/powerpc/boot/dts/mpc5200b.dtsi b/arch/powerpc/boot/dts/mpc5200b.dtsi index 8b796f3b11..ffa82c7e10 100644 --- a/arch/powerpc/boot/dts/mpc5200b.dtsi +++ b/arch/powerpc/boot/dts/mpc5200b.dtsi @@ -276,7 +276,9 @@ pci: pci@f0000d00 { clock-frequency = <0>; // From boot loader interrupts = <2 8 0 2 9 0 2 10 0>; bus-range = <0 0>; - // ranges = need to add + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x10000000>, + <0x02000000 0 0x90000000 0x90000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; }; localbus: localbus { diff --git a/arch/powerpc/boot/dts/mucmc52.dts b/arch/powerpc/boot/dts/mucmc52.dts index c6c6630630..e88a7bd403 100644 --- a/arch/powerpc/boot/dts/mucmc52.dts +++ b/arch/powerpc/boot/dts/mucmc52.dts @@ -106,9 +106,9 @@ pci@f0000d00 { 0x8000 0 0 3 &mpc5200_pic 0 2 3 0x8000 0 0 4 &mpc5200_pic 0 1 3 >; - ranges = <0x42000000 0 0x60000000 0x60000000 0 0x10000000 - 0x02000000 0 0x90000000 0x90000000 0 0x10000000 - 0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; + ranges = <0x42000000 0 0x60000000 0x60000000 0 0x10000000>, + <0x02000000 0 0x90000000 0x90000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; }; localbus { diff --git a/arch/powerpc/boot/dts/o2d.dtsi b/arch/powerpc/boot/dts/o2d.dtsi index b55a9e5bd8..7e52509fa5 100644 --- a/arch/powerpc/boot/dts/o2d.dtsi +++ b/arch/powerpc/boot/dts/o2d.dtsi @@ -34,12 +34,6 @@ psc@2000 { // PSC1 #address-cells = <1>; #size-cells = <0>; cell-index = <0>; - - spidev@0 { - compatible = "spidev"; - spi-max-frequency = <250000>; - reg = <0>; - }; }; psc@2200 { // PSC2 diff --git a/arch/powerpc/boot/dts/pcm030.dts b/arch/powerpc/boot/dts/pcm030.dts index b1bc731f7a..5cee474dcc 100644 --- a/arch/powerpc/boot/dts/pcm030.dts +++ b/arch/powerpc/boot/dts/pcm030.dts @@ -90,9 +90,9 @@ pci@f0000d00 { 0xc800 0 0 2 &mpc5200_pic 1 2 3 0xc800 0 0 3 &mpc5200_pic 1 3 3 0xc800 0 0 4 &mpc5200_pic 0 0 3>; - ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000 - 0x02000000 0 0xa0000000 0xa0000000 0 0x10000000 - 0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000>, + <0x02000000 0 0xa0000000 0xa0000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; }; localbus { diff --git a/arch/powerpc/boot/dts/pcm032.dts b/arch/powerpc/boot/dts/pcm032.dts index 1895bc9590..d00f13b625 100644 --- a/arch/powerpc/boot/dts/pcm032.dts +++ b/arch/powerpc/boot/dts/pcm032.dts @@ -87,9 +87,9 @@ pci@f0000d00 { 0xc800 0 0 2 &mpc5200_pic 1 2 3 0xc800 0 0 3 &mpc5200_pic 1 3 3 0xc800 0 0 4 &mpc5200_pic 0 0 3>; - ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000 - 0x02000000 0 0xa0000000 0xa0000000 0 0x10000000 - 0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x20000000>, + <0x02000000 0 0xa0000000 0xa0000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xb0000000 0 0x01000000>; }; localbus { diff --git a/arch/powerpc/boot/dts/redwood.dts b/arch/powerpc/boot/dts/redwood.dts index f38035a1f4..3c849e23e5 100644 --- a/arch/powerpc/boot/dts/redwood.dts +++ b/arch/powerpc/boot/dts/redwood.dts @@ -358,25 +358,6 @@ PCIE2: pcie@d40000000 { 0x0 0x0 0x0 0x4 &UIC3 0xb 0x4 /* swizzled int D */>; }; - MSI: ppc4xx-msi@400300000 { - compatible = "amcc,ppc4xx-msi", "ppc4xx-msi"; - reg = < 0x4 0x00300000 0x100 - 0x4 0x00300000 0x100>; - sdr-base = <0x3B0>; - msi-data = <0x00000000>; - msi-mask = <0x44440000>; - interrupt-count = <3>; - interrupts =<0 1 2 3>; - interrupt-parent = <&UIC0>; - #interrupt-cells = <1>; - #address-cells = <0>; - #size-cells = <0>; - interrupt-map = <0 &UIC0 0xC 1 - 1 &UIC0 0x0D 1 - 2 &UIC0 0x0E 1 - 3 &UIC0 0x0F 1>; - }; - }; diff --git a/arch/powerpc/boot/dts/tqm5200.dts b/arch/powerpc/boot/dts/tqm5200.dts index 5bb25a9e40..372177b19e 100644 --- a/arch/powerpc/boot/dts/tqm5200.dts +++ b/arch/powerpc/boot/dts/tqm5200.dts @@ -200,8 +200,8 @@ pci@f0000d00 { clock-frequency = <0>; // From boot loader interrupts = <2 8 0 2 9 0 2 10 0>; bus-range = <0 0>; - ranges = <0x42000000 0 0x80000000 0x80000000 0 0x10000000 - 0x02000000 0 0x90000000 0x90000000 0 0x10000000 - 0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; + ranges = <0x42000000 0 0x80000000 0x80000000 0 0x10000000>, + <0x02000000 0 0x90000000 0x90000000 0 0x10000000>, + <0x01000000 0 0x00000000 0xa0000000 0 0x01000000>; }; }; diff --git a/arch/powerpc/boot/dts/wii.dts b/arch/powerpc/boot/dts/wii.dts index e9c945b123..e46143c323 100644 --- a/arch/powerpc/boot/dts/wii.dts +++ b/arch/powerpc/boot/dts/wii.dts @@ -168,6 +168,11 @@ PIC1: pic1@d800030 { interrupts = <14>; }; + srnprot@d800060 { + compatible = "nintendo,hollywood-srnprot"; + reg = <0x0d800060 0x4>; + }; + GPIO: gpio@d8000c0 { #gpio-cells = <2>; compatible = "nintendo,hollywood-gpio"; diff --git a/arch/powerpc/boot/serial.c b/arch/powerpc/boot/serial.c index 9a19e59054..54d2522be4 100644 --- a/arch/powerpc/boot/serial.c +++ b/arch/powerpc/boot/serial.c @@ -132,7 +132,7 @@ int serial_console_init(void) else if (dt_is_compatible(devp, "fsl,mpc5200-psc-uart")) rc = mpc5200_psc_console_init(devp, &serial_cd); #endif -#ifdef CONFIG_PPC64_BOOT_WRAPPER +#ifdef CONFIG_PPC_POWERNV else if (dt_is_compatible(devp, "ibm,opal-console-raw")) rc = opal_console_init(devp, &serial_cd); #endif diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper index 1cd82564c9..9184eda780 100644 --- a/arch/powerpc/boot/wrapper +++ b/arch/powerpc/boot/wrapper @@ -26,6 +26,8 @@ # Stop execution if any command fails set -e +export LC_ALL=C + # Allow for verbose output if [ "$V" = 1 ]; then set -x diff --git a/arch/powerpc/boot/zImage.lds.S b/arch/powerpc/boot/zImage.lds.S index d6f0728656..d65cd55a6f 100644 --- a/arch/powerpc/boot/zImage.lds.S +++ b/arch/powerpc/boot/zImage.lds.S @@ -36,12 +36,9 @@ SECTIONS } #ifdef CONFIG_PPC64_BOOT_WRAPPER - . = ALIGN(256); - .got : + .got : ALIGN(256) { - __toc_start = .; - *(.got) - *(.toc) + *(.got .toc) } #endif diff --git a/arch/powerpc/configs/cell_defconfig b/arch/powerpc/configs/cell_defconfig index cc2c0d51f4..7fd9e596ea 100644 --- a/arch/powerpc/configs/cell_defconfig +++ b/arch/powerpc/configs/cell_defconfig @@ -36,7 +36,6 @@ CONFIG_GEN_RTC=y CONFIG_BINFMT_MISC=m CONFIG_IRQ_ALL_CPUS=y CONFIG_NUMA=y -CONFIG_PPC_64K_PAGES=y CONFIG_SCHED_SMT=y CONFIG_PCIEPORTBUS=y CONFIG_NET=y diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig index 63d611cc16..9d6212a8b1 100644 --- a/arch/powerpc/configs/g5_defconfig +++ b/arch/powerpc/configs/g5_defconfig @@ -26,6 +26,7 @@ CONFIG_CPU_FREQ_PMAC64=y CONFIG_GEN_RTC=y CONFIG_KEXEC=y CONFIG_IRQ_ALL_CPUS=y +CONFIG_PPC_4K_PAGES=y CONFIG_PCI_MSI=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/powerpc/configs/gamecube_defconfig b/arch/powerpc/configs/gamecube_defconfig index 24c0e0ea5a..91a1b99f4e 100644 --- a/arch/powerpc/configs/gamecube_defconfig +++ b/arch/powerpc/configs/gamecube_defconfig @@ -68,7 +68,7 @@ CONFIG_SND_SEQUENCER=y CONFIG_SND_SEQUENCER_OSS=y # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y -CONFIG_RTC_DRV_GENERIC=y +CONFIG_RTC_DRV_GAMECUBE=y CONFIG_EXT2_FS=y CONFIG_EXT4_FS=y CONFIG_ISO9660_FS=y diff --git a/arch/powerpc/configs/maple_defconfig b/arch/powerpc/configs/maple_defconfig index 9424c1e67e..c821a97f4a 100644 --- a/arch/powerpc/configs/maple_defconfig +++ b/arch/powerpc/configs/maple_defconfig @@ -25,6 +25,7 @@ CONFIG_UDBG_RTAS_CONSOLE=y CONFIG_GEN_RTC=y CONFIG_KEXEC=y CONFIG_IRQ_ALL_CPUS=y +CONFIG_PPC_4K_PAGES=y CONFIG_PCI_MSI=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/powerpc/configs/microwatt_defconfig b/arch/powerpc/configs/microwatt_defconfig index 9465209b8c..eff933ebbb 100644 --- a/arch/powerpc/configs/microwatt_defconfig +++ b/arch/powerpc/configs/microwatt_defconfig @@ -15,6 +15,8 @@ CONFIG_EMBEDDED=y # CONFIG_COMPAT_BRK is not set # CONFIG_SLAB_MERGE_DEFAULT is not set CONFIG_PPC64=y +CONFIG_POWER9_CPU=y +# CONFIG_PPC_64S_HASH_MMU is not set # CONFIG_PPC_KUEP is not set # CONFIG_PPC_KUAP is not set CONFIG_CPU_LITTLE_ENDIAN=y @@ -26,7 +28,7 @@ CONFIG_PPC_MICROWATT=y # CONFIG_PPC_OF_BOOT_TRAMPOLINE is not set CONFIG_CPU_FREQ=y CONFIG_HZ_100=y -# CONFIG_PPC_MEM_KEYS is not set +CONFIG_PPC_4K_PAGES=y # CONFIG_SECCOMP is not set # CONFIG_MQ_IOSCHED_KYBER is not set # CONFIG_COREDUMP is not set diff --git a/arch/powerpc/configs/pasemi_defconfig b/arch/powerpc/configs/pasemi_defconfig index 78606b7e42..e00a703581 100644 --- a/arch/powerpc/configs/pasemi_defconfig +++ b/arch/powerpc/configs/pasemi_defconfig @@ -22,7 +22,6 @@ CONFIG_CPU_FREQ_GOV_POWERSAVE=y CONFIG_CPU_FREQ_GOV_USERSPACE=y CONFIG_CPU_FREQ_GOV_ONDEMAND=y CONFIG_HZ_1000=y -CONFIG_PPC_64K_PAGES=y # CONFIG_SECCOMP is not set CONFIG_PCI_MSI=y CONFIG_PCCARD=y diff --git a/arch/powerpc/configs/pmac32_defconfig b/arch/powerpc/configs/pmac32_defconfig index 7aefac5afa..13885ec563 100644 --- a/arch/powerpc/configs/pmac32_defconfig +++ b/arch/powerpc/configs/pmac32_defconfig @@ -169,7 +169,6 @@ CONFIG_USB_USBNET=m CONFIG_B43=m CONFIG_B43LEGACY=m CONFIG_P54_COMMON=m -CONFIG_PRISM54=m CONFIG_INPUT_EVDEV=y # CONFIG_KEYBOARD_ATKBD is not set # CONFIG_MOUSE_PS2 is not set diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig index 8bfeea6c7d..49f49c2639 100644 --- a/arch/powerpc/configs/powernv_defconfig +++ b/arch/powerpc/configs/powernv_defconfig @@ -62,7 +62,6 @@ CONFIG_MEMORY_FAILURE=y CONFIG_HWPOISON_INJECT=m CONFIG_TRANSPARENT_HUGEPAGE=y CONFIG_DEFERRED_STRUCT_PAGE_INIT=y -CONFIG_PPC_64K_PAGES=y CONFIG_SCHED_SMT=y CONFIG_PM=y CONFIG_HOTPLUG_PCI=y diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index 0ad2291337..c8b0e80d61 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -26,7 +26,6 @@ CONFIG_PPC64=y CONFIG_NR_CPUS=2048 CONFIG_PPC_SPLPAR=y CONFIG_DTL=y -CONFIG_SCANLOG=m CONFIG_PPC_SMLPAR=y CONFIG_IBMEBUS=y CONFIG_PPC_SVM=y @@ -52,7 +51,6 @@ CONFIG_KEXEC_FILE=y CONFIG_CRASH_DUMP=y CONFIG_FA_DUMP=y CONFIG_IRQ_ALL_CPUS=y -CONFIG_PPC_64K_PAGES=y CONFIG_SCHED_SMT=y CONFIG_HOTPLUG_PCI=y CONFIG_HOTPLUG_PCI_RPA=m diff --git a/arch/powerpc/configs/ps3_defconfig b/arch/powerpc/configs/ps3_defconfig index f300dcb937..7c95fab4b9 100644 --- a/arch/powerpc/configs/ps3_defconfig +++ b/arch/powerpc/configs/ps3_defconfig @@ -30,6 +30,7 @@ CONFIG_PS3_LPM=m # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=y CONFIG_KEXEC=y +CONFIG_PPC_4K_PAGES=y # CONFIG_SPARSEMEM_VMEMMAP is not set # CONFIG_COMPACTION is not set CONFIG_SCHED_SMT=y diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig index d0494fbb49..b571d084c1 100644 --- a/arch/powerpc/configs/pseries_defconfig +++ b/arch/powerpc/configs/pseries_defconfig @@ -38,7 +38,6 @@ CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_PARTITION_ADVANCED=y CONFIG_PPC_SPLPAR=y CONFIG_DTL=y -CONFIG_SCANLOG=m CONFIG_PPC_SMLPAR=y CONFIG_IBMEBUS=y CONFIG_PAPR_SCM=m @@ -56,7 +55,6 @@ CONFIG_MEMORY_HOTPLUG=y CONFIG_MEMORY_HOTREMOVE=y CONFIG_KSM=y CONFIG_TRANSPARENT_HUGEPAGE=y -CONFIG_PPC_64K_PAGES=y CONFIG_SCHED_SMT=y CONFIG_HOTPLUG_PCI=y CONFIG_HOTPLUG_PCI_RPA=m diff --git a/arch/powerpc/configs/skiroot_defconfig b/arch/powerpc/configs/skiroot_defconfig index b806a5d3a6..f491875700 100644 --- a/arch/powerpc/configs/skiroot_defconfig +++ b/arch/powerpc/configs/skiroot_defconfig @@ -43,7 +43,6 @@ CONFIG_KEXEC_FILE=y CONFIG_PRESERVE_FA_DUMP=y CONFIG_IRQ_ALL_CPUS=y CONFIG_NUMA=y -CONFIG_PPC_64K_PAGES=y CONFIG_SCHED_SMT=y CONFIG_CMDLINE="console=tty0 console=hvc0 ipr.fast_reboot=1 quiet" # CONFIG_SECCOMP is not set @@ -275,7 +274,6 @@ CONFIG_NLS_UTF8=y CONFIG_ENCRYPTED_KEYS=y CONFIG_SECURITY=y CONFIG_HARDENED_USERCOPY=y -# CONFIG_HARDENED_USERCOPY_FALLBACK is not set CONFIG_HARDENED_USERCOPY_PAGESPAN=y CONFIG_FORTIFY_SOURCE=y CONFIG_SECURITY_LOCKDOWN_LSM=y diff --git a/arch/powerpc/configs/wii_defconfig b/arch/powerpc/configs/wii_defconfig index a0c45bf2bf..0ab78c5145 100644 --- a/arch/powerpc/configs/wii_defconfig +++ b/arch/powerpc/configs/wii_defconfig @@ -98,7 +98,7 @@ CONFIG_LEDS_TRIGGERS=y CONFIG_LEDS_TRIGGER_HEARTBEAT=y CONFIG_LEDS_TRIGGER_PANIC=y CONFIG_RTC_CLASS=y -CONFIG_RTC_DRV_GENERIC=y +CONFIG_RTC_DRV_GAMECUBE=y CONFIG_NVMEM_NINTENDO_OTP=y CONFIG_EXT2_FS=y CONFIG_EXT4_FS=y diff --git a/arch/powerpc/crypto/md5-asm.S b/arch/powerpc/crypto/md5-asm.S index 948d100a29..fa6bc440cf 100644 --- a/arch/powerpc/crypto/md5-asm.S +++ b/arch/powerpc/crypto/md5-asm.S @@ -38,15 +38,11 @@ #define INITIALIZE \ PPC_STLU r1,-INT_FRAME_SIZE(r1); \ - SAVE_8GPRS(14, r1); /* push registers onto stack */ \ - SAVE_4GPRS(22, r1); \ - SAVE_GPR(26, r1) + SAVE_GPRS(14, 26, r1) /* push registers onto stack */ #define FINALIZE \ - REST_8GPRS(14, r1); /* pop registers from stack */ \ - REST_4GPRS(22, r1); \ - REST_GPR(26, r1); \ - addi r1,r1,INT_FRAME_SIZE; + REST_GPRS(14, 26, r1); /* pop registers from stack */ \ + addi r1,r1,INT_FRAME_SIZE #ifdef __BIG_ENDIAN__ #define LOAD_DATA(reg, off) \ diff --git a/arch/powerpc/crypto/sha1-powerpc-asm.S b/arch/powerpc/crypto/sha1-powerpc-asm.S index 23e248beff..f0d5ed557a 100644 --- a/arch/powerpc/crypto/sha1-powerpc-asm.S +++ b/arch/powerpc/crypto/sha1-powerpc-asm.S @@ -125,8 +125,7 @@ _GLOBAL(powerpc_sha_transform) PPC_STLU r1,-INT_FRAME_SIZE(r1) - SAVE_8GPRS(14, r1) - SAVE_10GPRS(22, r1) + SAVE_GPRS(14, 31, r1) /* Load up A - E */ lwz RA(0),0(r3) /* A */ @@ -184,7 +183,6 @@ _GLOBAL(powerpc_sha_transform) stw RD(0),12(r3) stw RE(0),16(r3) - REST_8GPRS(14, r1) - REST_10GPRS(22, r1) + REST_GPRS(14, 31, r1) addi r1,r1,INT_FRAME_SIZE blr diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c index 88e8ea73bf..9170892a85 100644 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ b/arch/powerpc/crypto/sha1-spe-glue.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -55,20 +56,6 @@ static inline void ppc_sha1_clear_context(struct sha1_state *sctx) do { *ptr++ = 0; } while (--count); } -static int ppc_spe_sha1_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA1_H0; - sctx->state[1] = SHA1_H1; - sctx->state[2] = SHA1_H2; - sctx->state[3] = SHA1_H3; - sctx->state[4] = SHA1_H4; - sctx->count = 0; - - return 0; -} - static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -168,7 +155,7 @@ static int ppc_spe_sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = ppc_spe_sha1_init, + .init = sha1_base_init, .update = ppc_spe_sha1_update, .final = ppc_spe_sha1_final, .export = ppc_spe_sha1_export, diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c index 7a55d790cd..f283bbd3f1 100644 --- a/arch/powerpc/crypto/sha1.c +++ b/arch/powerpc/crypto/sha1.c @@ -18,21 +18,11 @@ #include #include #include +#include #include void powerpc_sha_transform(u32 *state, const u8 *src); -static int powerpc_sha1_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} - static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -114,7 +104,7 @@ static int powerpc_sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = powerpc_sha1_init, + .init = sha1_base_init, .update = powerpc_sha1_update, .final = powerpc_sha1_final, .export = powerpc_sha1_export, diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index ffedea7e4b..2997d13236 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -56,40 +57,6 @@ static inline void ppc_sha256_clear_context(struct sha256_state *sctx) do { *ptr++ = 0; } while (--count); } -static int ppc_spe_sha256_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; -} - -static int ppc_spe_sha224_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - - return 0; -} - static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -214,7 +181,7 @@ static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in) static struct shash_alg algs[2] = { { .digestsize = SHA256_DIGEST_SIZE, - .init = ppc_spe_sha256_init, + .init = sha256_base_init, .update = ppc_spe_sha256_update, .final = ppc_spe_sha256_final, .export = ppc_spe_sha256_export, @@ -230,7 +197,7 @@ static struct shash_alg algs[2] = { { } }, { .digestsize = SHA224_DIGEST_SIZE, - .init = ppc_spe_sha224_init, + .init = sha224_base_init, .update = ppc_spe_sha256_update, .final = ppc_spe_sha224_final, .export = ppc_spe_sha256_export, diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index b1b23b4d56..4d78293995 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -48,7 +48,7 @@ obj-y := cputable.o syscalls.o \ udbg.o misc.o io.o misc_$(BITS).o \ of_platform.o prom_parse.o firmware.o \ hw_breakpoint_constraints.o interrupt.o \ - kdebugfs.o + kdebugfs.o stacktrace.o obj-y += ptrace/ obj-$(CONFIG_PPC64) += setup_64.o \ paca.o nvram_64.o note.o @@ -107,7 +107,7 @@ extra-y += vmlinux.lds obj-$(CONFIG_RELOCATABLE) += reloc_$(BITS).o -obj-$(CONFIG_PPC32) += entry_32.o setup_32.o early_32.o +obj-$(CONFIG_PPC32) += entry_32.o setup_32.o early_32.o static_call.o obj-$(CONFIG_PPC64) += dma-iommu.o iommu.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_BOOTX_TEXT) += btext.o @@ -117,7 +117,6 @@ obj-$(CONFIG_OPTPROBES) += optprobes.o optprobes_head.o obj-$(CONFIG_KPROBES_ON_FTRACE) += kprobes-ftrace.o obj-$(CONFIG_UPROBES) += uprobes.o obj-$(CONFIG_PPC_UDBG_16550) += legacy_serial.o udbg_16550.o -obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_SWIOTLB) += dma-swiotlb.o obj-$(CONFIG_ARCH_HAS_DMA_SET_MASK) += dma-mask.o diff --git a/arch/powerpc/kernel/align.c b/arch/powerpc/kernel/align.c index bbb4181621..3e37ece067 100644 --- a/arch/powerpc/kernel/align.c +++ b/arch/powerpc/kernel/align.c @@ -105,7 +105,7 @@ static struct aligninfo spe_aligninfo[32] = { * so we don't need the address swizzling. */ static int emulate_spe(struct pt_regs *regs, unsigned int reg, - struct ppc_inst ppc_instr) + ppc_inst_t ppc_instr) { union { u64 ll; @@ -300,7 +300,7 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg, int fix_alignment(struct pt_regs *regs) { - struct ppc_inst instr; + ppc_inst_t instr; struct instruction_op op; int r, type; @@ -349,6 +349,7 @@ int fix_alignment(struct pt_regs *regs) if (op.type != CACHEOP + DCBZ) return -EINVAL; PPC_WARN_ALIGNMENT(dcbz, regs); + WARN_ON_ONCE(!user_mode(regs)); r = emulate_dcbz(op.ea, regs); } else { if (type == LARX || type == STCX) diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index e563d3222d..7582f3e3a3 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -9,8 +9,6 @@ * #defines from the assembly-language output. */ -#define GENERATING_ASM_OFFSETS /* asm/smp.h */ - #include #include #include @@ -56,7 +54,7 @@ #endif #ifdef CONFIG_PPC32 -#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) +#ifdef CONFIG_BOOKE_OR_40x #include "head_booke.h" #endif #endif @@ -93,7 +91,7 @@ int main(void) #endif /* CONFIG_PPC64 */ OFFSET(TASK_STACK, task_struct, stack); #ifdef CONFIG_SMP - OFFSET(TASK_CPU, task_struct, cpu); + OFFSET(TASK_CPU, task_struct, thread_info.cpu); #endif #ifdef CONFIG_LIVEPATCH @@ -141,6 +139,7 @@ int main(void) OFFSET(THR11, thread_struct, r11); OFFSET(THLR, thread_struct, lr); OFFSET(THCTR, thread_struct, ctr); + OFFSET(THSR0, thread_struct, sr0); #endif #ifdef CONFIG_SPE OFFSET(THREAD_EVR0, thread_struct, evr[0]); @@ -220,10 +219,12 @@ int main(void) OFFSET(PACA_EXGEN, paca_struct, exgen); OFFSET(PACA_EXMC, paca_struct, exmc); OFFSET(PACA_EXNMI, paca_struct, exnmi); +#ifdef CONFIG_PPC_64S_HASH_MMU OFFSET(PACA_SLBSHADOWPTR, paca_struct, slb_shadow_ptr); OFFSET(SLBSHADOW_STACKVSID, slb_shadow, save_area[SLB_NUM_BOLTED - 1].vsid); OFFSET(SLBSHADOW_STACKESID, slb_shadow, save_area[SLB_NUM_BOLTED - 1].esid); OFFSET(SLBSHADOW_SAVEAREA, slb_shadow, save_area); +#endif OFFSET(LPPACA_PMCINUSE, lppaca, pmcregs_in_use); #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE OFFSET(PACA_PMCINUSE, paca_struct, pmcregs_in_use); diff --git a/arch/powerpc/kernel/audit.c b/arch/powerpc/kernel/audit.c index a2dddd7f3d..1bcfca5fdf 100644 --- a/arch/powerpc/kernel/audit.c +++ b/arch/powerpc/kernel/audit.c @@ -47,15 +47,17 @@ int audit_classify_syscall(int abi, unsigned syscall) #endif switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/powerpc/kernel/btext.c b/arch/powerpc/kernel/btext.c index 1cffb5e7c3..9d9d56b574 100644 --- a/arch/powerpc/kernel/btext.c +++ b/arch/powerpc/kernel/btext.c @@ -161,7 +161,7 @@ void btext_map(void) boot_text_mapped = 1; } -static int btext_initialize(struct device_node *np) +static int __init btext_initialize(struct device_node *np) { unsigned int width, height, depth, pitch; unsigned long address = 0; @@ -292,7 +292,7 @@ void btext_update_display(unsigned long phys, int width, int height, } EXPORT_SYMBOL(btext_update_display); -void btext_clearscreen(void) +void __init btext_clearscreen(void) { unsigned int *base = (unsigned int *)calc_base(0, 0); unsigned long width = ((dispDeviceRect[2] - dispDeviceRect[0]) * @@ -310,7 +310,7 @@ void btext_clearscreen(void) rmci_maybe_off(); } -void btext_flushscreen(void) +void __init btext_flushscreen(void) { unsigned int *base = (unsigned int *)calc_base(0, 0); unsigned long width = ((dispDeviceRect[2] - dispDeviceRect[0]) * @@ -329,7 +329,7 @@ void btext_flushscreen(void) __asm__ __volatile__ ("sync" ::: "memory"); } -void btext_flushline(void) +void __init btext_flushline(void) { unsigned int *base = (unsigned int *)calc_base(0, g_loc_Y << 4); unsigned long width = ((dispDeviceRect[2] - dispDeviceRect[0]) * @@ -544,7 +544,7 @@ void btext_drawstring(const char *c) btext_drawchar(*c++); } -void btext_drawtext(const char *c, unsigned int len) +void __init btext_drawtext(const char *c, unsigned int len) { if (!boot_text_mapped) return; @@ -552,7 +552,7 @@ void btext_drawtext(const char *c, unsigned int len) btext_drawchar(*c++); } -void btext_drawhex(unsigned long v) +void __init btext_drawhex(unsigned long v) { if (!boot_text_mapped) return; diff --git a/arch/powerpc/kernel/cacheinfo.c b/arch/powerpc/kernel/cacheinfo.c index cf1be75b78..00b0992be3 100644 --- a/arch/powerpc/kernel/cacheinfo.c +++ b/arch/powerpc/kernel/cacheinfo.c @@ -710,7 +710,7 @@ static struct kobj_attribute cache_shared_cpu_list_attr = __ATTR(shared_cpu_list, 0444, shared_cpu_list_show, NULL); /* Attributes which should always be created -- the kobject/sysfs core - * does this automatically via kobj_type->default_attrs. This is the + * does this automatically via kobj_type->default_groups. This is the * minimum data required to uniquely identify a cache. */ static struct attribute *cache_index_default_attrs[] = { @@ -720,6 +720,7 @@ static struct attribute *cache_index_default_attrs[] = { &cache_shared_cpu_list_attr.attr, NULL, }; +ATTRIBUTE_GROUPS(cache_index_default); /* Attributes which should be created if the cache device node has the * right properties -- see cacheinfo_create_index_opt_attrs @@ -738,7 +739,7 @@ static const struct sysfs_ops cache_index_ops = { static struct kobj_type cache_index_type = { .release = cache_index_release, .sysfs_ops = &cache_index_ops, - .default_attrs = cache_index_default_attrs, + .default_groups = cache_index_default_groups, }; static void cacheinfo_create_index_opt_attrs(struct cache_index_dir *dir) diff --git a/arch/powerpc/kernel/compat_audit.c b/arch/powerpc/kernel/compat_audit.c index 55c6ccda0a..d92ffe4e5d 100644 --- a/arch/powerpc/kernel/compat_audit.c +++ b/arch/powerpc/kernel/compat_audit.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #undef __powerpc64__ +#include #include unsigned ppc32_dir_class[] = { @@ -31,14 +32,16 @@ int ppc32_classify_syscall(unsigned syscall) { switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/powerpc/kernel/cpu_setup_power.c b/arch/powerpc/kernel/cpu_setup_power.c index 3cca88ee96..3dc61e203f 100644 --- a/arch/powerpc/kernel/cpu_setup_power.c +++ b/arch/powerpc/kernel/cpu_setup_power.c @@ -109,7 +109,7 @@ static void init_PMU_HV_ISA207(void) static void init_PMU(void) { mtspr(SPRN_MMCRA, 0); - mtspr(SPRN_MMCR0, 0); + mtspr(SPRN_MMCR0, MMCR0_FC); mtspr(SPRN_MMCR1, 0); mtspr(SPRN_MMCR2, 0); } @@ -123,7 +123,7 @@ static void init_PMU_ISA31(void) { mtspr(SPRN_MMCR3, 0); mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); - mtspr(SPRN_MMCR0, MMCR0_PMCCEXT); + mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMCCEXT); } /* @@ -137,6 +137,7 @@ void __setup_cpu_power7(unsigned long offset, struct cpu_spec *t) return; mtspr(SPRN_LPID, 0); + mtspr(SPRN_AMOR, ~0); mtspr(SPRN_PCR, PCR_MASK); init_LPCR_ISA206(mfspr(SPRN_LPCR), LPCR_LPES1 >> LPCR_LPES_SH); } @@ -150,6 +151,7 @@ void __restore_cpu_power7(void) return; mtspr(SPRN_LPID, 0); + mtspr(SPRN_AMOR, ~0); mtspr(SPRN_PCR, PCR_MASK); init_LPCR_ISA206(mfspr(SPRN_LPCR), LPCR_LPES1 >> LPCR_LPES_SH); } @@ -164,6 +166,7 @@ void __setup_cpu_power8(unsigned long offset, struct cpu_spec *t) return; mtspr(SPRN_LPID, 0); + mtspr(SPRN_AMOR, ~0); mtspr(SPRN_PCR, PCR_MASK); init_LPCR_ISA206(mfspr(SPRN_LPCR) | LPCR_PECEDH, 0); /* LPES = 0 */ init_HFSCR(); @@ -184,6 +187,7 @@ void __restore_cpu_power8(void) return; mtspr(SPRN_LPID, 0); + mtspr(SPRN_AMOR, ~0); mtspr(SPRN_PCR, PCR_MASK); init_LPCR_ISA206(mfspr(SPRN_LPCR) | LPCR_PECEDH, 0); /* LPES = 0 */ init_HFSCR(); @@ -202,6 +206,7 @@ void __setup_cpu_power9(unsigned long offset, struct cpu_spec *t) mtspr(SPRN_PSSCR, 0); mtspr(SPRN_LPID, 0); mtspr(SPRN_PID, 0); + mtspr(SPRN_AMOR, ~0); mtspr(SPRN_PCR, PCR_MASK); init_LPCR_ISA300((mfspr(SPRN_LPCR) | LPCR_PECEDH | LPCR_PECE_HVEE |\ LPCR_HVICE | LPCR_HEIC) & ~(LPCR_UPRT | LPCR_HR), 0); @@ -223,6 +228,7 @@ void __restore_cpu_power9(void) mtspr(SPRN_PSSCR, 0); mtspr(SPRN_LPID, 0); mtspr(SPRN_PID, 0); + mtspr(SPRN_AMOR, ~0); mtspr(SPRN_PCR, PCR_MASK); init_LPCR_ISA300((mfspr(SPRN_LPCR) | LPCR_PECEDH | LPCR_PECE_HVEE |\ LPCR_HVICE | LPCR_HEIC) & ~(LPCR_UPRT | LPCR_HR), 0); @@ -242,6 +248,7 @@ void __setup_cpu_power10(unsigned long offset, struct cpu_spec *t) mtspr(SPRN_PSSCR, 0); mtspr(SPRN_LPID, 0); mtspr(SPRN_PID, 0); + mtspr(SPRN_AMOR, ~0); mtspr(SPRN_PCR, PCR_MASK); init_LPCR_ISA300((mfspr(SPRN_LPCR) | LPCR_PECEDH | LPCR_PECE_HVEE |\ LPCR_HVICE | LPCR_HEIC) & ~(LPCR_UPRT | LPCR_HR), 0); @@ -264,6 +271,7 @@ void __restore_cpu_power10(void) mtspr(SPRN_PSSCR, 0); mtspr(SPRN_LPID, 0); mtspr(SPRN_PID, 0); + mtspr(SPRN_AMOR, ~0); mtspr(SPRN_PCR, PCR_MASK); init_LPCR_ISA300((mfspr(SPRN_LPCR) | LPCR_PECEDH | LPCR_PECE_HVEE |\ LPCR_HVICE | LPCR_HEIC) & ~(LPCR_UPRT | LPCR_HR), 0); diff --git a/arch/powerpc/kernel/dbell.c b/arch/powerpc/kernel/dbell.c index 5545c9cd17..f55c6fb34a 100644 --- a/arch/powerpc/kernel/dbell.c +++ b/arch/powerpc/kernel/dbell.c @@ -27,7 +27,8 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(doorbell_exception) ppc_msgsync(); - may_hard_irq_enable(); + if (should_hard_irq_enable()) + do_hard_irq_enable(); kvmppc_clear_host_ipi(smp_processor_id()); __this_cpu_inc(irq_stat.doorbell_irqs); diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index 358aee7c2d..7d1b2c4a48 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -80,6 +80,7 @@ static void __restore_cpu_cpufeatures(void) mtspr(SPRN_LPCR, system_registers.lpcr); if (hv_mode) { mtspr(SPRN_LPID, 0); + mtspr(SPRN_AMOR, ~0); mtspr(SPRN_HFSCR, system_registers.hfscr); mtspr(SPRN_PCR, system_registers.pcr); } @@ -216,6 +217,7 @@ static int __init feat_enable_hv(struct dt_cpu_feature *f) } mtspr(SPRN_LPID, 0); + mtspr(SPRN_AMOR, ~0); lpcr = mfspr(SPRN_LPCR); lpcr &= ~LPCR_LPES0; /* HV external interrupts */ @@ -271,6 +273,9 @@ static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) { u64 lpcr; + if (!IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) + return 0; + lpcr = mfspr(SPRN_LPCR); lpcr &= ~LPCR_ISL; @@ -290,6 +295,9 @@ static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) { u64 lpcr; + if (!IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) + return 0; + lpcr = mfspr(SPRN_LPCR); lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); mtspr(SPRN_LPCR, lpcr); @@ -303,15 +311,15 @@ static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) { -#ifdef CONFIG_PPC_RADIX_MMU + if (!IS_ENABLED(CONFIG_PPC_RADIX_MMU)) + return 0; + + cur_cpu_spec->mmu_features |= MMU_FTR_KERNEL_RO; cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; - cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; cur_cpu_spec->mmu_features |= MMU_FTR_GTSE; cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; return 1; -#endif - return 0; } static int __init feat_enable_dscr(struct dt_cpu_feature *f) @@ -336,7 +344,7 @@ static int __init feat_enable_dscr(struct dt_cpu_feature *f) return 1; } -static void hfscr_pmu_enable(void) +static void __init hfscr_pmu_enable(void) { u64 hfscr = mfspr(SPRN_HFSCR); hfscr |= PPC_BIT(60); @@ -351,7 +359,7 @@ static void init_pmu_power8(void) } mtspr(SPRN_MMCRA, 0); - mtspr(SPRN_MMCR0, 0); + mtspr(SPRN_MMCR0, MMCR0_FC); mtspr(SPRN_MMCR1, 0); mtspr(SPRN_MMCR2, 0); mtspr(SPRN_MMCRS, 0); @@ -390,7 +398,7 @@ static void init_pmu_power9(void) mtspr(SPRN_MMCRC, 0); mtspr(SPRN_MMCRA, 0); - mtspr(SPRN_MMCR0, 0); + mtspr(SPRN_MMCR0, MMCR0_FC); mtspr(SPRN_MMCR1, 0); mtspr(SPRN_MMCR2, 0); } @@ -426,7 +434,7 @@ static void init_pmu_power10(void) mtspr(SPRN_MMCR3, 0); mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); - mtspr(SPRN_MMCR0, MMCR0_PMCCEXT); + mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMCCEXT); } static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f) @@ -1095,8 +1103,8 @@ static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char cpufeatures_setup_finished(); - memblock_free(__pa(dt_cpu_features), - sizeof(struct dt_cpu_feature)*nr_dt_cpu_features); + memblock_free(dt_cpu_features, + sizeof(struct dt_cpu_feature) * nr_dt_cpu_features); return 0; } diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index e9b597ed42..28bb1e7263 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -399,6 +399,14 @@ static int eeh_phb_check_failure(struct eeh_pe *pe) return ret; } +static inline const char *eeh_driver_name(struct pci_dev *pdev) +{ + if (pdev) + return dev_driver_string(&pdev->dev); + + return ""; +} + /** * eeh_dev_check_failure - Check if all 1's data is due to EEH slot freeze * @edev: eeh device @@ -589,6 +597,7 @@ EXPORT_SYMBOL(eeh_check_failure); /** * eeh_pci_enable - Enable MMIO or DMA transfers for this slot * @pe: EEH PE + * @function: EEH option * * This routine should be called to reenable frozen MMIO or DMA * so that it would work correctly again. It's useful while doing @@ -761,8 +770,8 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat } /** - * eeh_set_pe_freset - Check the required reset for the indicated device - * @data: EEH device + * eeh_set_dev_freset - Check the required reset for the indicated device + * @edev: EEH device * @flag: return value * * Each device might have its preferred reset type: fundamental or @@ -801,6 +810,7 @@ static void eeh_pe_refreeze_passed(struct eeh_pe *root) /** * eeh_pe_reset_full - Complete a full reset process on the indicated PE * @pe: EEH PE + * @include_passed: include passed-through devices? * * This function executes a full reset procedure on a PE, including setting * the appropriate flags, performing a fundamental or hot reset, and then @@ -937,6 +947,7 @@ static struct notifier_block eeh_device_nb = { /** * eeh_init - System wide EEH initialization + * @ops: struct to trace EEH operation callback functions * * It's the platform's job to call this from an arch_initcall(). */ @@ -1442,6 +1453,7 @@ static int eeh_pe_reenable_devices(struct eeh_pe *pe, bool include_passed) * eeh_pe_reset - Issue PE reset according to specified type * @pe: EEH PE * @option: reset type + * @include_passed: include passed-through devices? * * The routine is called to reset the specified PE with the * indicated type, either fundamental reset or hot reset. @@ -1513,12 +1525,12 @@ EXPORT_SYMBOL_GPL(eeh_pe_configure); * eeh_pe_inject_err - Injecting the specified PCI error to the indicated PE * @pe: the indicated PE * @type: error type - * @function: error function + * @func: error function * @addr: address * @mask: address mask * * The routine is called to inject the specified PCI error, which - * is determined by @type and @function, to the indicated PE for + * is determined by @type and @func, to the indicated PE for * testing purpose. */ int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func, diff --git a/arch/powerpc/kernel/eeh_cache.c b/arch/powerpc/kernel/eeh_cache.c index 9bdaaf7fdd..2f9dbf8ad2 100644 --- a/arch/powerpc/kernel/eeh_cache.c +++ b/arch/powerpc/kernel/eeh_cache.c @@ -280,7 +280,7 @@ static int eeh_addr_cache_show(struct seq_file *s, void *v) } DEFINE_SHOW_ATTRIBUTE(eeh_addr_cache); -void eeh_cache_debugfs_init(void) +void __init eeh_cache_debugfs_init(void) { debugfs_create_file_unsafe("eeh_address_cache", 0400, arch_debugfs_dir, NULL, diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index 3eff6a4888..422f80b5b2 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -104,13 +104,13 @@ static bool eeh_edev_actionable(struct eeh_dev *edev) */ static inline struct pci_driver *eeh_pcid_get(struct pci_dev *pdev) { - if (!pdev || !pdev->driver) + if (!pdev || !pdev->dev.driver) return NULL; - if (!try_module_get(pdev->driver->driver.owner)) + if (!try_module_get(pdev->dev.driver->owner)) return NULL; - return pdev->driver; + return to_pci_driver(pdev->dev.driver); } /** @@ -122,10 +122,10 @@ static inline struct pci_driver *eeh_pcid_get(struct pci_dev *pdev) */ static inline void eeh_pcid_put(struct pci_dev *pdev) { - if (!pdev || !pdev->driver) + if (!pdev || !pdev->dev.driver) return; - module_put(pdev->driver->driver.owner); + module_put(pdev->dev.driver->owner); } /** @@ -905,18 +905,19 @@ void eeh_handle_normal_event(struct eeh_pe *pe) } #endif /* CONFIG_STACKTRACE */ + eeh_for_each_pe(pe, tmp_pe) + eeh_pe_for_each_dev(tmp_pe, edev, tmp) + edev->mode &= ~EEH_DEV_NO_HANDLER; + eeh_pe_update_time_stamp(pe); pe->freeze_count++; if (pe->freeze_count > eeh_max_freezes) { pr_err("EEH: PHB#%x-PE#%x has failed %d times in the last hour and has been permanently disabled.\n", pe->phb->global_number, pe->addr, pe->freeze_count); - result = PCI_ERS_RESULT_DISCONNECT; - } - eeh_for_each_pe(pe, tmp_pe) - eeh_pe_for_each_dev(tmp_pe, edev, tmp) - edev->mode &= ~EEH_DEV_NO_HANDLER; + goto recover_failed; + } /* Walk the various device drivers attached to this slot through * a reset sequence, giving each an opportunity to do what it needs @@ -928,39 +929,38 @@ void eeh_handle_normal_event(struct eeh_pe *pe) * the error. Override the result if necessary to have partially * hotplug for this case. */ - if (result != PCI_ERS_RESULT_DISCONNECT) { - pr_warn("EEH: This PCI device has failed %d times in the last hour and will be permanently disabled after %d failures.\n", - pe->freeze_count, eeh_max_freezes); - pr_info("EEH: Notify device drivers to shutdown\n"); - eeh_set_channel_state(pe, pci_channel_io_frozen); - eeh_set_irq_state(pe, false); - eeh_pe_report("error_detected(IO frozen)", pe, - eeh_report_error, &result); - if ((pe->type & EEH_PE_PHB) && - result != PCI_ERS_RESULT_NONE && - result != PCI_ERS_RESULT_NEED_RESET) - result = PCI_ERS_RESULT_NEED_RESET; - } + pr_warn("EEH: This PCI device has failed %d times in the last hour and will be permanently disabled after %d failures.\n", + pe->freeze_count, eeh_max_freezes); + pr_info("EEH: Notify device drivers to shutdown\n"); + eeh_set_channel_state(pe, pci_channel_io_frozen); + eeh_set_irq_state(pe, false); + eeh_pe_report("error_detected(IO frozen)", pe, + eeh_report_error, &result); + if (result == PCI_ERS_RESULT_DISCONNECT) + goto recover_failed; + + /* + * Error logged on a PHB are always fences which need a full + * PHB reset to clear so force that to happen. + */ + if ((pe->type & EEH_PE_PHB) && result != PCI_ERS_RESULT_NONE) + result = PCI_ERS_RESULT_NEED_RESET; /* Get the current PCI slot state. This can take a long time, * sometimes over 300 seconds for certain systems. */ - if (result != PCI_ERS_RESULT_DISCONNECT) { - rc = eeh_wait_state(pe, MAX_WAIT_FOR_RECOVERY*1000); - if (rc < 0 || rc == EEH_STATE_NOT_SUPPORT) { - pr_warn("EEH: Permanent failure\n"); - result = PCI_ERS_RESULT_DISCONNECT; - } + rc = eeh_wait_state(pe, MAX_WAIT_FOR_RECOVERY * 1000); + if (rc < 0 || rc == EEH_STATE_NOT_SUPPORT) { + pr_warn("EEH: Permanent failure\n"); + goto recover_failed; } /* Since rtas may enable MMIO when posting the error log, * don't post the error log until after all dev drivers * have been informed. */ - if (result != PCI_ERS_RESULT_DISCONNECT) { - pr_info("EEH: Collect temporary log\n"); - eeh_slot_error_detail(pe, EEH_LOG_TEMP); - } + pr_info("EEH: Collect temporary log\n"); + eeh_slot_error_detail(pe, EEH_LOG_TEMP); /* If all device drivers were EEH-unaware, then shut * down all of the device drivers, and hope they @@ -970,9 +970,8 @@ void eeh_handle_normal_event(struct eeh_pe *pe) pr_info("EEH: Reset with hotplug activity\n"); rc = eeh_reset_device(pe, bus, NULL, false); if (rc) { - pr_warn("%s: Unable to reset, err=%d\n", - __func__, rc); - result = PCI_ERS_RESULT_DISCONNECT; + pr_warn("%s: Unable to reset, err=%d\n", __func__, rc); + goto recover_failed; } } @@ -980,10 +979,10 @@ void eeh_handle_normal_event(struct eeh_pe *pe) if (result == PCI_ERS_RESULT_CAN_RECOVER) { pr_info("EEH: Enable I/O for affected devices\n"); rc = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); + if (rc < 0) + goto recover_failed; - if (rc < 0) { - result = PCI_ERS_RESULT_DISCONNECT; - } else if (rc) { + if (rc) { result = PCI_ERS_RESULT_NEED_RESET; } else { pr_info("EEH: Notify device drivers to resume I/O\n"); @@ -991,15 +990,13 @@ void eeh_handle_normal_event(struct eeh_pe *pe) eeh_report_mmio_enabled, &result); } } - - /* If all devices reported they can proceed, then re-enable DMA */ if (result == PCI_ERS_RESULT_CAN_RECOVER) { pr_info("EEH: Enabled DMA for affected devices\n"); rc = eeh_pci_enable(pe, EEH_OPT_THAW_DMA); + if (rc < 0) + goto recover_failed; - if (rc < 0) { - result = PCI_ERS_RESULT_DISCONNECT; - } else if (rc) { + if (rc) { result = PCI_ERS_RESULT_NEED_RESET; } else { /* @@ -1017,16 +1014,15 @@ void eeh_handle_normal_event(struct eeh_pe *pe) pr_info("EEH: Reset without hotplug activity\n"); rc = eeh_reset_device(pe, bus, &rmv_data, true); if (rc) { - pr_warn("%s: Cannot reset, err=%d\n", - __func__, rc); - result = PCI_ERS_RESULT_DISCONNECT; - } else { - result = PCI_ERS_RESULT_NONE; - eeh_set_channel_state(pe, pci_channel_io_normal); - eeh_set_irq_state(pe, true); - eeh_pe_report("slot_reset", pe, eeh_report_reset, - &result); + pr_warn("%s: Cannot reset, err=%d\n", __func__, rc); + goto recover_failed; } + + result = PCI_ERS_RESULT_NONE; + eeh_set_channel_state(pe, pci_channel_io_normal); + eeh_set_irq_state(pe, true); + eeh_pe_report("slot_reset", pe, eeh_report_reset, + &result); } if ((result == PCI_ERS_RESULT_RECOVERED) || @@ -1054,45 +1050,47 @@ void eeh_handle_normal_event(struct eeh_pe *pe) } pr_info("EEH: Recovery successful.\n"); - } else { - /* - * About 90% of all real-life EEH failures in the field - * are due to poorly seated PCI cards. Only 10% or so are - * due to actual, failed cards. - */ - pr_err("EEH: Unable to recover from failure from PHB#%x-PE#%x.\n" - "Please try reseating or replacing it\n", - pe->phb->global_number, pe->addr); + goto out; + } - eeh_slot_error_detail(pe, EEH_LOG_PERM); +recover_failed: + /* + * About 90% of all real-life EEH failures in the field + * are due to poorly seated PCI cards. Only 10% or so are + * due to actual, failed cards. + */ + pr_err("EEH: Unable to recover from failure from PHB#%x-PE#%x.\n" + "Please try reseating or replacing it\n", + pe->phb->global_number, pe->addr); - /* Notify all devices that they're about to go down. */ - eeh_set_channel_state(pe, pci_channel_io_perm_failure); - eeh_set_irq_state(pe, false); - eeh_pe_report("error_detected(permanent failure)", pe, - eeh_report_failure, NULL); + eeh_slot_error_detail(pe, EEH_LOG_PERM); - /* Mark the PE to be removed permanently */ - eeh_pe_state_mark(pe, EEH_PE_REMOVED); + /* Notify all devices that they're about to go down. */ + eeh_set_channel_state(pe, pci_channel_io_perm_failure); + eeh_set_irq_state(pe, false); + eeh_pe_report("error_detected(permanent failure)", pe, + eeh_report_failure, NULL); - /* - * Shut down the device drivers for good. We mark - * all removed devices correctly to avoid access - * the their PCI config any more. - */ - if (pe->type & EEH_PE_VF) { - eeh_pe_dev_traverse(pe, eeh_rmv_device, NULL); - eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); - } else { - eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true); - eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); + /* Mark the PE to be removed permanently */ + eeh_pe_state_mark(pe, EEH_PE_REMOVED); - pci_lock_rescan_remove(); - pci_hp_remove_devices(bus); - pci_unlock_rescan_remove(); - /* The passed PE should no longer be used */ - return; - } + /* + * Shut down the device drivers for good. We mark + * all removed devices correctly to avoid access + * the their PCI config any more. + */ + if (pe->type & EEH_PE_VF) { + eeh_pe_dev_traverse(pe, eeh_rmv_device, NULL); + eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); + } else { + eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true); + eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); + + pci_lock_rescan_remove(); + pci_hp_remove_devices(bus); + pci_unlock_rescan_remove(); + /* The passed PE should no longer be used */ + return; } out: diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index 61fdd53cdd..7748c278d1 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -73,13 +73,39 @@ prepare_transfer_to_handler: _ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler) #endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */ +#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32) + .globl __kuep_lock +__kuep_lock: + lwz r9, THREAD+THSR0(r2) + update_user_segments_by_4 r9, r10, r11, r12 + blr + +__kuep_unlock: + lwz r9, THREAD+THSR0(r2) + rlwinm r9,r9,0,~SR_NX + update_user_segments_by_4 r9, r10, r11, r12 + blr + +.macro kuep_lock + bl __kuep_lock +.endm +.macro kuep_unlock + bl __kuep_unlock +.endm +#else +.macro kuep_lock +.endm +.macro kuep_unlock +.endm +#endif + .globl transfer_to_syscall transfer_to_syscall: stw r11, GPR1(r1) stw r11, 0(r1) mflr r12 stw r12, _LINK(r1) -#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) +#ifdef CONFIG_BOOKE_OR_40x rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ #endif lis r12,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ @@ -90,10 +116,10 @@ transfer_to_syscall: stw r12,8(r1) stw r2,_TRAP(r1) SAVE_GPR(0, r1) - SAVE_4GPRS(3, r1) - SAVE_2GPRS(7, r1) + SAVE_GPRS(3, 8, r1) addi r2,r10,-THREAD SAVE_NVGPRS(r1) + kuep_lock /* Calling convention has r9 = orig r0, r10 = regs */ addi r10,r1,STACK_FRAME_OVERHEAD @@ -110,6 +136,7 @@ ret_from_syscall: cmplwi cr0,r5,0 bne- 2f #endif /* CONFIG_PPC_47x */ + kuep_unlock lwz r4,_LINK(r1) lwz r5,_CCR(r1) mtlr r4 @@ -139,7 +166,7 @@ syscall_exit_finish: mtxer r5 lwz r0,GPR0(r1) lwz r3,GPR3(r1) - REST_8GPRS(4,r1) + REST_GPRS(4, 11, r1) lwz r12,GPR12(r1) b 1b @@ -232,9 +259,9 @@ fast_exception_return: beq 3f /* if not, we've got problems */ #endif -2: REST_4GPRS(3, r11) +2: REST_GPRS(3, 6, r11) lwz r10,_CCR(r11) - REST_2GPRS(1, r11) + REST_GPRS(1, 2, r11) mtcr r10 lwz r10,_LINK(r11) mtlr r10 @@ -273,6 +300,7 @@ interrupt_return: beq .Lkernel_interrupt_return bl interrupt_exit_user_prepare cmpwi r3,0 + kuep_unlock bne- .Lrestore_nvgprs .Lfast_user_interrupt_return: @@ -298,16 +326,14 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) * the reliable stack unwinder later on. Clear it. */ stw r0,8(r1) - REST_4GPRS(7, r1) - REST_2GPRS(11, r1) + REST_GPRS(7, 12, r1) mtcr r3 mtlr r4 mtctr r5 mtspr SPRN_XER,r6 - REST_4GPRS(2, r1) - REST_GPR(6, r1) + REST_GPRS(2, 6, r1) REST_GPR(0, r1) REST_GPR(1, r1) rfi @@ -341,8 +367,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) lwz r6,_CCR(r1) li r0,0 - REST_4GPRS(7, r1) - REST_2GPRS(11, r1) + REST_GPRS(7, 12, r1) mtlr r3 mtctr r4 @@ -354,7 +379,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) */ stw r0,8(r1) - REST_4GPRS(2, r1) + REST_GPRS(2, 5, r1) bne- cr1,1f /* emulate stack store */ mtcr r6 @@ -430,8 +455,7 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return) bne interrupt_return; \ lwz r0,GPR0(r1); \ lwz r2,GPR2(r1); \ - REST_4GPRS(3, r1); \ - REST_2GPRS(7, r1); \ + REST_GPRS(3, 8, r1); \ lwz r10,_XER(r1); \ lwz r11,_CTR(r1); \ mtspr SPRN_XER,r10; \ diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 70cff7b49e..9581906b5e 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -180,7 +180,7 @@ _GLOBAL(_switch) #endif ld r8,KSP(r4) /* new stack pointer */ -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU BEGIN_MMU_FTR_SECTION b 2f END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) @@ -232,7 +232,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) slbmte r7,r0 isync 2: -#endif /* CONFIG_PPC_BOOK3S_64 */ +#endif /* CONFIG_PPC_64S_HASH_MMU */ clrrdi r7, r8, THREAD_SHIFT /* base of new stack */ /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE diff --git a/arch/powerpc/kernel/epapr_paravirt.c b/arch/powerpc/kernel/epapr_paravirt.c index 93b0f3ec8f..d4b8aff208 100644 --- a/arch/powerpc/kernel/epapr_paravirt.c +++ b/arch/powerpc/kernel/epapr_paravirt.c @@ -37,7 +37,7 @@ static int __init early_init_dt_scan_epapr(unsigned long node, return -1; for (i = 0; i < (len / 4); i++) { - struct ppc_inst inst = ppc_inst(be32_to_cpu(insts[i])); + ppc_inst_t inst = ppc_inst(be32_to_cpu(insts[i])); patch_instruction(epapr_hypercall_start + i, inst); #if !defined(CONFIG_64BIT) || defined(CONFIG_PPC_BOOK3E_64) patch_instruction(epapr_ev_idle_start + i, inst); diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 711c66b76d..67dc4e3179 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -198,8 +198,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) stdcx. r0,0,r1 /* to clear the reservation */ - REST_4GPRS(2, r1) - REST_4GPRS(6, r1) + REST_GPRS(2, 9, r1) ld r10,_CTR(r1) ld r11,_XER(r1) @@ -375,9 +374,7 @@ ret_from_mc_except: exc_##n##_common: \ std r0,GPR0(r1); /* save r0 in stackframe */ \ std r2,GPR2(r1); /* save r2 in stackframe */ \ - SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \ - SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \ - std r9,GPR9(r1); /* save r9 in stackframe */ \ + SAVE_GPRS(3, 9, r1); /* save r3 - r9 in stackframe */ \ std r10,_NIP(r1); /* save SRR0 to stackframe */ \ std r11,_MSR(r1); /* save SRR1 to stackframe */ \ beq 2f; /* if from kernel mode */ \ @@ -1061,9 +1058,7 @@ bad_stack_book3e: std r11,_ESR(r1) std r0,GPR0(r1); /* save r0 in stackframe */ \ std r2,GPR2(r1); /* save r2 in stackframe */ \ - SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \ - SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \ - std r9,GPR9(r1); /* save r9 in stackframe */ \ + SAVE_GPRS(3, 9, r1); /* save r3 - r9 in stackframe */ \ ld r3,PACA_EXGEN+EX_R10(r13);/* get back r10 */ \ ld r4,PACA_EXGEN+EX_R11(r13);/* get back r11 */ \ mfspr r5,SPRN_SPRG_GEN_SCRATCH;/* get back r13 XXX can be wrong */ \ @@ -1077,8 +1072,7 @@ bad_stack_book3e: std r10,_LINK(r1) std r11,_CTR(r1) std r12,_XER(r1) - SAVE_10GPRS(14,r1) - SAVE_8GPRS(24,r1) + SAVE_GPRS(14, 31, r1) lhz r12,PACA_TRAP_SAVE(r13) std r12,_TRAP(r1) addi r11,r1,INT_FRAME_SIZE diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index eaf1f72131..55caeee37c 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -48,7 +48,7 @@ .balign IFETCH_ALIGN_BYTES; \ .global name; \ _ASM_NOKPROBE_SYMBOL(name); \ - DEFINE_FIXED_SYMBOL(name); \ + DEFINE_FIXED_SYMBOL(name, text); \ name: #define TRAMP_REAL_BEGIN(name) \ @@ -76,31 +76,18 @@ name: ld reg,PACAKBASE(r13); /* get high part of &label */ \ ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) -#define __LOAD_HANDLER(reg, label) \ +#define __LOAD_HANDLER(reg, label, section) \ ld reg,PACAKBASE(r13); \ - ori reg,reg,(ABS_ADDR(label))@l + ori reg,reg,(ABS_ADDR(label, section))@l /* * Branches from unrelocated code (e.g., interrupts) to labels outside * head-y require >64K offsets. */ -#define __LOAD_FAR_HANDLER(reg, label) \ +#define __LOAD_FAR_HANDLER(reg, label, section) \ ld reg,PACAKBASE(r13); \ - ori reg,reg,(ABS_ADDR(label))@l; \ - addis reg,reg,(ABS_ADDR(label))@h - -/* - * Branch to label using its 0xC000 address. This results in instruction - * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned - * on using mtmsr rather than rfid. - * - * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than - * load KBASE for a slight optimisation. - */ -#define BRANCH_TO_C000(reg, label) \ - __LOAD_FAR_HANDLER(reg, label); \ - mtctr reg; \ - bctr + ori reg,reg,(ABS_ADDR(label, section))@l; \ + addis reg,reg,(ABS_ADDR(label, section))@h /* * Interrupt code generation macros @@ -111,9 +98,10 @@ name: #define IAREA .L_IAREA_\name\() /* PACA save area */ #define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */ #define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */ +#define ICFAR .L_ICFAR_\name\() /* Uses CFAR */ +#define ICFAR_IF_HVMODE .L_ICFAR_IF_HVMODE_\name\() /* Uses CFAR if HV */ #define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */ #define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */ -#define ISET_RI .L_ISET_RI_\name\() /* Run common code w/ MSR[RI]=1 */ #define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ #define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ #define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ @@ -151,15 +139,18 @@ do_define_int n .ifndef IISIDE IISIDE=0 .endif + .ifndef ICFAR + ICFAR=1 + .endif + .ifndef ICFAR_IF_HVMODE + ICFAR_IF_HVMODE=0 + .endif .ifndef IDAR IDAR=0 .endif .ifndef IDSISR IDSISR=0 .endif - .ifndef ISET_RI - ISET_RI=1 - .endif .ifndef IBRANCH_TO_COMMON IBRANCH_TO_COMMON=1 .endif @@ -291,9 +282,21 @@ BEGIN_FTR_SECTION END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) HMT_MEDIUM std r10,IAREA+EX_R10(r13) /* save r10 - r12 */ + .if ICFAR BEGIN_FTR_SECTION mfspr r10,SPRN_CFAR END_FTR_SECTION_IFSET(CPU_FTR_CFAR) + .elseif ICFAR_IF_HVMODE +BEGIN_FTR_SECTION + BEGIN_FTR_SECTION_NESTED(69) + mfspr r10,SPRN_CFAR + END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) +FTR_SECTION_ELSE + BEGIN_FTR_SECTION_NESTED(69) + li r10,0 + END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 69) +ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) + .endif .if \ool .if !\virt b tramp_real_\name @@ -309,9 +312,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) BEGIN_FTR_SECTION std r9,IAREA+EX_PPR(r13) END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) + .if ICFAR || ICFAR_IF_HVMODE BEGIN_FTR_SECTION std r10,IAREA+EX_CFAR(r13) END_FTR_SECTION_IFSET(CPU_FTR_CFAR) + .endif INTERRUPT_TO_KERNEL mfctr r10 std r10,IAREA+EX_CTR(r13) @@ -376,7 +381,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) * This switches to virtual mode and sets MSR[RI]. */ .macro __GEN_COMMON_ENTRY name -DEFINE_FIXED_SYMBOL(\name\()_common_real) +DEFINE_FIXED_SYMBOL(\name\()_common_real, text) \name\()_common_real: .if IKVM_REAL KVMTEST \name kvm_interrupt @@ -399,7 +404,7 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real) .endif .balign IFETCH_ALIGN_BYTES -DEFINE_FIXED_SYMBOL(\name\()_common_virt) +DEFINE_FIXED_SYMBOL(\name\()_common_virt, text) \name\()_common_virt: .if IKVM_VIRT KVMTEST \name kvm_interrupt @@ -413,7 +418,7 @@ DEFINE_FIXED_SYMBOL(\name\()_common_virt) * want to run in real mode. */ .macro __GEN_REALMODE_COMMON_ENTRY name -DEFINE_FIXED_SYMBOL(\name\()_common_real) +DEFINE_FIXED_SYMBOL(\name\()_common_real, text) \name\()_common_real: .if IKVM_REAL KVMTEST \name kvm_interrupt @@ -512,11 +517,6 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real) stb r10,PACASRR_VALID(r13) .endif - .if ISET_RI - li r10,MSR_RI - mtmsrd r10,1 /* Set MSR_RI */ - .endif - .if ISTACK .if IKUAP kuap_save_amr_and_lock r9, r10, cr1, cr0 @@ -568,14 +568,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) .endif BEGIN_FTR_SECTION + .if ICFAR || ICFAR_IF_HVMODE ld r10,IAREA+EX_CFAR(r13) + .else + li r10,0 + .endif std r10,ORIG_GPR3(r1) END_FTR_SECTION_IFSET(CPU_FTR_CFAR) ld r10,IAREA+EX_CTR(r13) std r10,_CTR(r1) std r2,GPR2(r1) /* save r2 in stackframe */ - SAVE_4GPRS(3, r1) /* save r3 - r6 in stackframe */ - SAVE_2GPRS(7, r1) /* save r7, r8 in stackframe */ + SAVE_GPRS(3, 8, r1) /* save r3 - r8 in stackframe */ mflr r9 /* Get LR, later save to stack */ ld r2,PACATOC(r13) /* get kernel TOC into r2 */ std r9,_LINK(r1) @@ -693,8 +696,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) mtlr r9 ld r9,_CCR(r1) mtcr r9 - REST_8GPRS(2, r1) - REST_4GPRS(10, r1) + REST_GPRS(2, 13, r1) REST_GPR(0, r1) /* restore original r1. */ ld r1,GPR1(r1) @@ -850,12 +852,12 @@ SOFT_MASK_TABLE(0xc000000000003000, 0xc000000000004000) #ifdef CONFIG_RELOCATABLE TRAMP_VIRT_BEGIN(system_call_vectored_tramp) - __LOAD_HANDLER(r10, system_call_vectored_common) + __LOAD_HANDLER(r10, system_call_vectored_common, virt_trampolines) mtctr r10 bctr TRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp) - __LOAD_HANDLER(r10, system_call_vectored_sigill) + __LOAD_HANDLER(r10, system_call_vectored_sigill, virt_trampolines) mtctr r10 bctr #endif @@ -902,11 +904,6 @@ INT_DEFINE_BEGIN(system_reset) IVEC=0x100 IAREA=PACA_EXNMI IVIRT=0 /* no virt entry point */ - /* - * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is - * being used, so a nested NMI exception would corrupt it. - */ - ISET_RI=0 ISTACK=0 IKVM_REAL=1 INT_DEFINE_END(system_reset) @@ -964,7 +961,9 @@ TRAMP_REAL_BEGIN(system_reset_idle_wake) /* We are waking up from idle, so may clobber any volatile register */ cmpwi cr1,r5,2 bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ - BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss)) + __LOAD_FAR_HANDLER(r12, DOTSYM(idle_return_gpr_loss), real_trampolines) + mtctr r12 + bctr #endif #ifdef CONFIG_PPC_PSERIES @@ -979,16 +978,14 @@ TRAMP_REAL_BEGIN(system_reset_fwnmi) EXC_COMMON_BEGIN(system_reset_common) __GEN_COMMON_ENTRY system_reset /* - * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able - * to recover, but nested NMI will notice in_nmi and not recover - * because of the use of the NMI stack. in_nmi reentrancy is tested in - * system_reset_exception. + * Increment paca->in_nmi. When the interrupt entry wrapper later + * enable MSR_RI, then SLB or MCE will be able to recover, but a nested + * NMI will notice in_nmi and not recover because of the use of the NMI + * stack. in_nmi reentrancy is tested in system_reset_exception. */ lhz r10,PACA_IN_NMI(r13) addi r10,r10,1 sth r10,PACA_IN_NMI(r13) - li r10,MSR_RI - mtmsrd r10,1 mr r10,r1 ld r1,PACA_NMI_EMERG_SP(r13) @@ -1062,12 +1059,6 @@ INT_DEFINE_BEGIN(machine_check_early) IAREA=PACA_EXMC IVIRT=0 /* no virt entry point */ IREALMODE_COMMON=1 - /* - * MSR_RI is not enabled, because PACA_EXMC is being used, so a - * nested machine check corrupts it. machine_check_common enables - * MSR_RI. - */ - ISET_RI=0 ISTACK=0 IDAR=1 IDSISR=1 @@ -1078,7 +1069,6 @@ INT_DEFINE_BEGIN(machine_check) IVEC=0x200 IAREA=PACA_EXMC IVIRT=0 /* no virt entry point */ - ISET_RI=0 IDAR=1 IDSISR=1 IKVM_REAL=1 @@ -1148,9 +1138,6 @@ EXC_COMMON_BEGIN(machine_check_early_common) BEGIN_FTR_SECTION bl enable_machine_check END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) - li r10,MSR_RI - mtmsrd r10,1 - addi r3,r1,STACK_FRAME_OVERHEAD bl machine_check_early std r3,RESULT(r1) /* Save result */ @@ -1238,10 +1225,6 @@ EXC_COMMON_BEGIN(machine_check_common) * save area: PACA_EXMC instead of PACA_EXGEN. */ GEN_COMMON machine_check - - /* Enable MSR_RI when finished with PACA_EXMC */ - li r10,MSR_RI - mtmsrd r10,1 addi r3,r1,STACK_FRAME_OVERHEAD bl machine_check_exception_async b interrupt_return_srr @@ -1369,11 +1352,15 @@ EXC_COMMON_BEGIN(data_access_common) addi r3,r1,STACK_FRAME_OVERHEAD andis. r0,r4,DSISR_DABRMATCH@h bne- 1f +#ifdef CONFIG_PPC_64S_HASH_MMU BEGIN_MMU_FTR_SECTION bl do_hash_fault MMU_FTR_SECTION_ELSE bl do_page_fault ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) +#else + bl do_page_fault +#endif b interrupt_return_srr 1: bl do_break @@ -1416,6 +1403,7 @@ EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) EXC_VIRT_END(data_access_slb, 0x4380, 0x80) EXC_COMMON_BEGIN(data_access_slb_common) GEN_COMMON data_access_slb +#ifdef CONFIG_PPC_64S_HASH_MMU BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ addi r3,r1,STACK_FRAME_OVERHEAD @@ -1428,9 +1416,12 @@ MMU_FTR_SECTION_ELSE /* Radix case, access is outside page table range */ li r3,-EFAULT ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) +#else + li r3,-EFAULT +#endif std r3,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD - bl do_bad_slb_fault + bl do_bad_segment_interrupt b interrupt_return_srr @@ -1462,11 +1453,15 @@ EXC_VIRT_END(instruction_access, 0x4400, 0x80) EXC_COMMON_BEGIN(instruction_access_common) GEN_COMMON instruction_access addi r3,r1,STACK_FRAME_OVERHEAD +#ifdef CONFIG_PPC_64S_HASH_MMU BEGIN_MMU_FTR_SECTION bl do_hash_fault MMU_FTR_SECTION_ELSE bl do_page_fault ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) +#else + bl do_page_fault +#endif b interrupt_return_srr @@ -1496,6 +1491,7 @@ EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) EXC_COMMON_BEGIN(instruction_access_slb_common) GEN_COMMON instruction_access_slb +#ifdef CONFIG_PPC_64S_HASH_MMU BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ addi r3,r1,STACK_FRAME_OVERHEAD @@ -1508,9 +1504,12 @@ MMU_FTR_SECTION_ELSE /* Radix case, access is outside page table range */ li r3,-EFAULT ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) +#else + li r3,-EFAULT +#endif std r3,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD - bl do_bad_slb_fault + bl do_bad_segment_interrupt b interrupt_return_srr @@ -1536,6 +1535,12 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) * * If soft masked, the masked handler will note the pending interrupt for * replay, and clear MSR[EE] in the interrupted context. + * + * CFAR is not required because this is an asynchronous interrupt that in + * general won't have much bearing on the state of the CPU, with the possible + * exception of crash/debug IPIs, but those are generally moving to use SRESET + * IPIs. Unless this is an HV interrupt and KVM HV is possible, in which case + * it may be exiting the guest and need CFAR to be saved. */ INT_DEFINE_BEGIN(hardware_interrupt) IVEC=0x500 @@ -1543,6 +1548,10 @@ INT_DEFINE_BEGIN(hardware_interrupt) IMASK=IRQS_DISABLED IKVM_REAL=1 IKVM_VIRT=1 + ICFAR=0 +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + ICFAR_IF_HVMODE=1 +#endif INT_DEFINE_END(hardware_interrupt) EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) @@ -1764,6 +1773,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) * If PPC_WATCHDOG is configured, the soft masked handler will actually set * things back up to run soft_nmi_interrupt as a regular interrupt handler * on the emergency stack. + * + * CFAR is not required because this is asynchronous (see hardware_interrupt). + * A watchdog interrupt may like to have CFAR, but usually the interesting + * branch is long gone by that point (e.g., infinite loop). */ INT_DEFINE_BEGIN(decrementer) IVEC=0x900 @@ -1771,6 +1784,7 @@ INT_DEFINE_BEGIN(decrementer) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif + ICFAR=0 INT_DEFINE_END(decrementer) EXC_REAL_BEGIN(decrementer, 0x900, 0x80) @@ -1846,6 +1860,8 @@ EXC_COMMON_BEGIN(hdecrementer_common) * If soft masked, the masked handler will note the pending interrupt for * replay, leaving MSR[EE] enabled in the interrupted context because the * doorbells are edge triggered. + * + * CFAR is not required, similarly to hardware_interrupt. */ INT_DEFINE_BEGIN(doorbell_super) IVEC=0xa00 @@ -1853,6 +1869,7 @@ INT_DEFINE_BEGIN(doorbell_super) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif + ICFAR=0 INT_DEFINE_END(doorbell_super) EXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100) @@ -1904,6 +1921,7 @@ INT_DEFINE_BEGIN(system_call) IVEC=0xc00 IKVM_REAL=1 IKVM_VIRT=1 + ICFAR=0 INT_DEFINE_END(system_call) .macro SYSTEM_CALL virt @@ -1942,12 +1960,12 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) HMT_MEDIUM .if ! \virt - __LOAD_HANDLER(r10, system_call_common_real) + __LOAD_HANDLER(r10, system_call_common_real, real_vectors) mtctr r10 bctr .else #ifdef CONFIG_RELOCATABLE - __LOAD_HANDLER(r10, system_call_common) + __LOAD_HANDLER(r10, system_call_common, virt_vectors) mtctr r10 bctr #else @@ -2001,7 +2019,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) * Requires __LOAD_FAR_HANDLER beause kvmppc_hcall lives * outside the head section. */ - __LOAD_FAR_HANDLER(r10, kvmppc_hcall) + __LOAD_FAR_HANDLER(r10, kvmppc_hcall, real_trampolines) mtctr r10 bctr #else @@ -2202,6 +2220,11 @@ EXC_COMMON_BEGIN(hmi_exception_common) * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. * This is an asynchronous interrupt in response to a msgsnd doorbell. * Similar to the 0xa00 doorbell but for host rather than guest. + * + * CFAR is not required (similar to doorbell_interrupt), unless KVM HV + * is enabled, in which case it may be a guest exit. Most PowerNV kernels + * include KVM support so it would be nice if this could be dynamically + * patched out if KVM was not currently running any guests. */ INT_DEFINE_BEGIN(h_doorbell) IVEC=0xe80 @@ -2209,6 +2232,9 @@ INT_DEFINE_BEGIN(h_doorbell) IMASK=IRQS_DISABLED IKVM_REAL=1 IKVM_VIRT=1 +#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE + ICFAR=0 +#endif INT_DEFINE_END(h_doorbell) EXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20) @@ -2232,6 +2258,9 @@ EXC_COMMON_BEGIN(h_doorbell_common) * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. * This is an asynchronous interrupt in response to an "external exception". * Similar to 0x500 but for host only. + * + * Like h_doorbell, CFAR is only required for KVM HV because this can be + * a guest exit. */ INT_DEFINE_BEGIN(h_virt_irq) IVEC=0xea0 @@ -2239,6 +2268,9 @@ INT_DEFINE_BEGIN(h_virt_irq) IMASK=IRQS_DISABLED IKVM_REAL=1 IKVM_VIRT=1 +#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE + ICFAR=0 +#endif INT_DEFINE_END(h_virt_irq) EXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20) @@ -2275,6 +2307,8 @@ EXC_VIRT_NONE(0x4ee0, 0x20) * * If soft masked, the masked handler will note the pending interrupt for * replay, and clear MSR[EE] in the interrupted context. + * + * CFAR is not used by perf interrupts so not required. */ INT_DEFINE_BEGIN(performance_monitor) IVEC=0xf00 @@ -2282,6 +2316,7 @@ INT_DEFINE_BEGIN(performance_monitor) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif + ICFAR=0 INT_DEFINE_END(performance_monitor) EXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20) @@ -2706,6 +2741,7 @@ EXC_VIRT_NONE(0x5800, 0x100) INT_DEFINE_BEGIN(soft_nmi) IVEC=0x900 ISTACK=0 + ICFAR=0 INT_DEFINE_END(soft_nmi) /* @@ -3025,7 +3061,7 @@ USE_FIXED_SECTION(virt_trampolines) .align 7 .globl __end_interrupts __end_interrupts: -DEFINE_FIXED_SYMBOL(__end_interrupts) +DEFINE_FIXED_SYMBOL(__end_interrupts, virt_trampolines) CLOSE_FIXED_SECTION(real_vectors); CLOSE_FIXED_SECTION(real_trampolines); diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c index 60f5fc14aa..d03e488cfe 100644 --- a/arch/powerpc/kernel/fadump.c +++ b/arch/powerpc/kernel/fadump.c @@ -251,7 +251,7 @@ bool is_fadump_reserved_mem_contiguous(void) } /* Print firmware assisted dump configurations for debugging purpose. */ -static void fadump_show_config(void) +static void __init fadump_show_config(void) { int i; @@ -353,7 +353,7 @@ static __init u64 fadump_calculate_reserve_size(void) * Calculate the total memory size required to be reserved for * firmware-assisted dump registration. */ -static unsigned long get_fadump_area_size(void) +static unsigned long __init get_fadump_area_size(void) { unsigned long size = 0; @@ -462,7 +462,7 @@ static int __init fadump_get_boot_mem_regions(void) * with the given memory range. * False, otherwise. */ -static bool overlaps_reserved_ranges(u64 base, u64 end, int *idx) +static bool __init overlaps_reserved_ranges(u64 base, u64 end, int *idx) { bool ret = false; int i; @@ -737,7 +737,7 @@ void crash_fadump(struct pt_regs *regs, const char *str) fw_dump.ops->fadump_trigger(fdh, str); } -u32 *fadump_regs_to_elf_notes(u32 *buf, struct pt_regs *regs) +u32 *__init fadump_regs_to_elf_notes(u32 *buf, struct pt_regs *regs) { struct elf_prstatus prstatus; @@ -752,7 +752,7 @@ u32 *fadump_regs_to_elf_notes(u32 *buf, struct pt_regs *regs) return buf; } -void fadump_update_elfcore_header(char *bufp) +void __init fadump_update_elfcore_header(char *bufp) { struct elf_phdr *phdr; @@ -770,7 +770,7 @@ void fadump_update_elfcore_header(char *bufp) return; } -static void *fadump_alloc_buffer(unsigned long size) +static void *__init fadump_alloc_buffer(unsigned long size) { unsigned long count, i; struct page *page; @@ -792,7 +792,7 @@ static void fadump_free_buffer(unsigned long vaddr, unsigned long size) free_reserved_area((void *)vaddr, (void *)(vaddr + size), -1, NULL); } -s32 fadump_setup_cpu_notes_buf(u32 num_cpus) +s32 __init fadump_setup_cpu_notes_buf(u32 num_cpus) { /* Allocate buffer to hold cpu crash notes. */ fw_dump.cpu_notes_buf_size = num_cpus * sizeof(note_buf_t); @@ -1447,7 +1447,7 @@ static ssize_t release_mem_store(struct kobject *kobj, } /* Release the reserved memory and disable the FADump */ -static void unregister_fadump(void) +static void __init unregister_fadump(void) { fadump_cleanup(); fadump_release_memory(fw_dump.reserve_dump_area_start, @@ -1547,7 +1547,7 @@ ATTRIBUTE_GROUPS(fadump); DEFINE_SHOW_ATTRIBUTE(fadump_region); -static void fadump_init_files(void) +static void __init fadump_init_files(void) { int rc = 0; diff --git a/arch/powerpc/kernel/fpu.S b/arch/powerpc/kernel/fpu.S index ba4afe3b5a..f71f2bbd4d 100644 --- a/arch/powerpc/kernel/fpu.S +++ b/arch/powerpc/kernel/fpu.S @@ -81,7 +81,12 @@ EXPORT_SYMBOL(store_fp_state) */ _GLOBAL(load_up_fpu) mfmsr r5 +#ifdef CONFIG_PPC_BOOK3S_64 + /* interrupt doesn't set MSR[RI] and HPT can fault on current access */ + ori r5,r5,MSR_FP|MSR_RI +#else ori r5,r5,MSR_FP +#endif #ifdef CONFIG_VSX BEGIN_FTR_SECTION oris r5,r5,MSR_VSX@h diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h index 349c4a8202..c3286260a7 100644 --- a/arch/powerpc/kernel/head_32.h +++ b/arch/powerpc/kernel/head_32.h @@ -115,8 +115,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt) stw r10,8(r1) li r10, \trapno stw r10,_TRAP(r1) - SAVE_4GPRS(3, r1) - SAVE_2GPRS(7, r1) + SAVE_GPRS(3, 8, r1) SAVE_NVGPRS(r1) stw r2,GPR2(r1) stw r12,_NIP(r1) @@ -136,6 +135,12 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt) andi. r12,r9,MSR_PR bne 777f bl prepare_transfer_to_handler +#ifdef CONFIG_PPC_KUEP + b 778f +777: + bl __kuep_lock +778: +#endif 777: #endif .endm diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S index e783860bea..b6c6d1de5f 100644 --- a/arch/powerpc/kernel/head_40x.S +++ b/arch/powerpc/kernel/head_40x.S @@ -298,6 +298,10 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt) 3: mfspr r11,SPRN_SPRG_THREAD lwz r11,PGDIR(r11) +#ifdef CONFIG_PPC_KUAP + rlwinm. r9, r9, 0, 0xff + beq 5f /* Kuap fault */ +#endif 4: tophys(r11, r11) rlwimi r11, r10, 12, 20, 29 /* Create L1 (pgdir/pmd) address */ @@ -378,6 +382,10 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt) 3: mfspr r11,SPRN_SPRG_THREAD lwz r11,PGDIR(r11) +#ifdef CONFIG_PPC_KUAP + rlwinm. r9, r9, 0, 0xff + beq 5f /* Kuap fault */ +#endif 4: tophys(r11, r11) rlwimi r11, r10, 12, 20, 29 /* Create L1 (pgdir/pmd) address */ diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S index 02d2928d1e..b73a564669 100644 --- a/arch/powerpc/kernel/head_44x.S +++ b/arch/powerpc/kernel/head_44x.S @@ -334,6 +334,10 @@ interrupt_base: mfspr r12,SPRN_MMUCR mfspr r13,SPRN_PID /* Get PID */ rlwimi r12,r13,0,24,31 /* Set TID */ +#ifdef CONFIG_PPC_KUAP + cmpwi r13,0 + beq 2f /* KUAP Fault */ +#endif 4: mtspr SPRN_MMUCR,r12 @@ -444,6 +448,10 @@ interrupt_base: mfspr r12,SPRN_MMUCR mfspr r13,SPRN_PID /* Get PID */ rlwimi r12,r13,0,24,31 /* Set TID */ +#ifdef CONFIG_PPC_KUAP + cmpwi r13,0 + beq 2f /* KUAP Fault */ +#endif 4: mtspr SPRN_MMUCR,r12 @@ -532,10 +540,7 @@ finish_tlb_load_44x: andi. r10,r12,_PAGE_USER /* User page ? */ beq 1f /* nope, leave U bits empty */ rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */ -#ifdef CONFIG_PPC_KUEP -0: rlwinm r11,r11,0,~PPC44x_TLB_SX /* Clear SX if User page */ - patch_site 0b, patch__tlb_44x_kuep -#endif + rlwinm r11,r11,0,~PPC44x_TLB_SX /* Clear SX if User page */ 1: tlbwe r11,r13,PPC44x_TLB_ATTRIB /* Write ATTRIB */ /* Done...restore registers and get out of here. @@ -575,6 +580,10 @@ finish_tlb_load_44x: 3: mfspr r11,SPRN_SPRG3 lwz r11,PGDIR(r11) mfspr r12,SPRN_PID /* Get PID */ +#ifdef CONFIG_PPC_KUAP + cmpwi r12,0 + beq 2f /* KUAP Fault */ +#endif 4: mtspr SPRN_MMUCR,r12 /* Set MMUCR */ /* Mask of required permission bits. Note that while we @@ -672,6 +681,10 @@ finish_tlb_load_44x: 3: mfspr r11,SPRN_SPRG_THREAD lwz r11,PGDIR(r11) mfspr r12,SPRN_PID /* Get PID */ +#ifdef CONFIG_PPC_KUAP + cmpwi r12,0 + beq 2f /* KUAP Fault */ +#endif 4: mtspr SPRN_MMUCR,r12 /* Set MMUCR */ /* Make up the required permissions */ @@ -747,10 +760,7 @@ finish_tlb_load_47x: andi. r10,r12,_PAGE_USER /* User page ? */ beq 1f /* nope, leave U bits empty */ rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */ -#ifdef CONFIG_PPC_KUEP -0: rlwinm r11,r11,0,~PPC47x_TLB2_SX /* Clear SX if User page */ - patch_site 0b, patch__tlb_47x_kuep -#endif + rlwinm r11,r11,0,~PPC47x_TLB2_SX /* Clear SX if User page */ 1: tlbwe r11,r13,2 /* Done...restore registers and get out of here. diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index f17ae20837..5c5181e8d5 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -126,7 +126,7 @@ __secondary_hold_acknowledge: . = 0x5c .globl __run_at_load __run_at_load: -DEFINE_FIXED_SYMBOL(__run_at_load) +DEFINE_FIXED_SYMBOL(__run_at_load, first_256B) .long RUN_AT_LOAD_DEFAULT #endif @@ -156,7 +156,7 @@ __secondary_hold: /* Tell the master cpu we're here */ /* Relocation is off & we are located at an address less */ /* than 0x100, so only need to grab low order offset. */ - std r24,(ABS_ADDR(__secondary_hold_acknowledge))(0) + std r24,(ABS_ADDR(__secondary_hold_acknowledge, first_256B))(0) sync li r26,0 @@ -164,7 +164,7 @@ __secondary_hold: tovirt(r26,r26) #endif /* All secondary cpus wait here until told to start. */ -100: ld r12,(ABS_ADDR(__secondary_hold_spinloop))(r26) +100: ld r12,(ABS_ADDR(__secondary_hold_spinloop, first_256B))(r26) cmpdi 0,r12,0 beq 100b @@ -649,15 +649,15 @@ __after_prom_start: 3: #endif /* # bytes of memory to copy */ - lis r5,(ABS_ADDR(copy_to_here))@ha - addi r5,r5,(ABS_ADDR(copy_to_here))@l + lis r5,(ABS_ADDR(copy_to_here, text))@ha + addi r5,r5,(ABS_ADDR(copy_to_here, text))@l bl copy_and_flush /* copy the first n bytes */ /* this includes the code being */ /* executed here. */ /* Jump to the copy of this code that we just made */ - addis r8,r3,(ABS_ADDR(4f))@ha - addi r12,r8,(ABS_ADDR(4f))@l + addis r8,r3,(ABS_ADDR(4f, text))@ha + addi r12,r8,(ABS_ADDR(4f, text))@l mtctr r12 bctr @@ -669,8 +669,8 @@ p_end: .8byte _end - copy_to_here * Now copy the rest of the kernel up to _end, add * _end - copy_to_here to the copy limit and run again. */ - addis r8,r26,(ABS_ADDR(p_end))@ha - ld r8,(ABS_ADDR(p_end))@l(r8) + addis r8,r26,(ABS_ADDR(p_end, text))@ha + ld r8,(ABS_ADDR(p_end, text))@l(r8) add r5,r5,r8 5: bl copy_and_flush /* copy the rest */ @@ -904,7 +904,7 @@ _GLOBAL(relative_toc) blr .balign 8 -p_toc: .8byte __toc_start + 0x8000 - 0b +p_toc: .8byte .TOC. - 0b /* * This is where the main kernel code starts. diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S index 2e2a8211b1..b876ef8c70 100644 --- a/arch/powerpc/kernel/head_book3s_32.S +++ b/arch/powerpc/kernel/head_book3s_32.S @@ -931,7 +931,11 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS) _GLOBAL(load_segment_registers) li r0, NUM_USER_SEGMENTS /* load up user segment register values */ mtctr r0 /* for context 0 */ +#ifdef CONFIG_PPC_KUEP + lis r3, SR_NX@h /* Kp = 0, Ks = 0, VSID = 0 */ +#else li r3, 0 /* Kp = 0, Ks = 0, VSID = 0 */ +#endif li r4, 0 3: mtsrin r3, r4 addi r3, r3, 0x111 /* increment VSID */ diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h index ef8d1b1c23..bb6d5d0fc4 100644 --- a/arch/powerpc/kernel/head_booke.h +++ b/arch/powerpc/kernel/head_booke.h @@ -87,8 +87,7 @@ END_BTB_FLUSH_SECTION stw r10, 8(r1) li r10, \trapno stw r10,_TRAP(r1) - SAVE_4GPRS(3, r1) - SAVE_2GPRS(7, r1) + SAVE_GPRS(3, 8, r1) SAVE_NVGPRS(r1) stw r2,GPR2(r1) stw r12,_NIP(r1) diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S index 0a9a0f3014..ac2b4dcf5f 100644 --- a/arch/powerpc/kernel/head_fsl_booke.S +++ b/arch/powerpc/kernel/head_fsl_booke.S @@ -462,6 +462,12 @@ END_BTB_FLUSH_SECTION mfspr r11,SPRN_SPRG_THREAD lwz r11,PGDIR(r11) +#ifdef CONFIG_PPC_KUAP + mfspr r12, SPRN_MAS1 + rlwinm. r12,r12,0,0x3fff0000 + beq 2f /* KUAP fault */ +#endif + 4: /* Mask of required permission bits. Note that while we * do copy ESR:ST to _PAGE_RW position as trying to write @@ -571,6 +577,12 @@ END_BTB_FLUSH_SECTION mfspr r11,SPRN_SPRG_THREAD lwz r11,PGDIR(r11) +#ifdef CONFIG_PPC_KUAP + mfspr r12, SPRN_MAS1 + rlwinm. r12,r12,0,0x3fff0000 + beq 2f /* KUAP fault */ +#endif + /* Make up the required permissions for user code */ #ifdef CONFIG_PTE_64BIT li r13,_PAGE_PRESENT | _PAGE_BAP_UX @@ -777,6 +789,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS) andi. r10, r11, _PAGE_USER /* Test for _PAGE_USER */ slwi r10, r12, 1 or r10, r10, r12 + rlwinm r10, r10, 0, ~_PAGE_EXEC /* Clear SX on user pages */ iseleq r12, r12, r10 rlwimi r13, r12, 0, 20, 31 /* Get RPN from PTE, merge w/ perms */ mtspr SPRN_MAS3, r13 diff --git a/arch/powerpc/kernel/hw_breakpoint.c b/arch/powerpc/kernel/hw_breakpoint.c index 91a3be1480..2669f80b3a 100644 --- a/arch/powerpc/kernel/hw_breakpoint.c +++ b/arch/powerpc/kernel/hw_breakpoint.c @@ -523,7 +523,7 @@ static void larx_stcx_err(struct perf_event *bp, struct arch_hw_breakpoint *info static bool stepping_handler(struct pt_regs *regs, struct perf_event **bp, struct arch_hw_breakpoint **info, int *hit, - struct ppc_inst instr) + ppc_inst_t instr) { int i; int stepped; @@ -616,7 +616,7 @@ int hw_breakpoint_handler(struct die_args *args) int hit[HBP_NUM_MAX] = {0}; int nr_hit = 0; bool ptrace_bp = false; - struct ppc_inst instr = ppc_inst(0); + ppc_inst_t instr = ppc_inst(0); int type = 0; int size = 0; unsigned long ea; diff --git a/arch/powerpc/kernel/hw_breakpoint_constraints.c b/arch/powerpc/kernel/hw_breakpoint_constraints.c index 675d1f66ab..a74623025f 100644 --- a/arch/powerpc/kernel/hw_breakpoint_constraints.c +++ b/arch/powerpc/kernel/hw_breakpoint_constraints.c @@ -80,7 +80,7 @@ static bool check_dawrx_constraints(struct pt_regs *regs, int type, * Return true if the event is valid wrt dawr configuration, * including extraneous exception. Otherwise return false. */ -bool wp_check_constraints(struct pt_regs *regs, struct ppc_inst instr, +bool wp_check_constraints(struct pt_regs *regs, ppc_inst_t instr, unsigned long ea, int type, int size, struct arch_hw_breakpoint *info) { @@ -127,16 +127,7 @@ bool wp_check_constraints(struct pt_regs *regs, struct ppc_inst instr, return false; } -static int cache_op_size(void) -{ -#ifdef __powerpc64__ - return ppc64_caches.l1d.block_size; -#else - return L1_CACHE_BYTES; -#endif -} - -void wp_get_instr_detail(struct pt_regs *regs, struct ppc_inst *instr, +void wp_get_instr_detail(struct pt_regs *regs, ppc_inst_t *instr, int *type, int *size, unsigned long *ea) { struct instruction_op op; @@ -147,14 +138,14 @@ void wp_get_instr_detail(struct pt_regs *regs, struct ppc_inst *instr, analyse_instr(&op, regs, *instr); *type = GETTYPE(op.type); *ea = op.ea; -#ifdef __powerpc64__ + if (!(regs->msr & MSR_64BIT)) *ea &= 0xffffffffUL; -#endif + *size = GETSIZE(op.type); if (*type == CACHEOP) { - *size = cache_op_size(); + *size = l1_dcache_bytes(); *ea &= ~(*size - 1); } else if (*type == LOAD_VMX || *type == STORE_VMX) { *ea &= ~(*size - 1); diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c index 1f835539fd..4ad79eb638 100644 --- a/arch/powerpc/kernel/idle.c +++ b/arch/powerpc/kernel/idle.c @@ -82,7 +82,7 @@ void power4_idle(void) return; if (cpu_has_feature(CPU_FTR_ALTIVEC)) - asm volatile("DSSALL ; sync" ::: "memory"); + asm volatile(PPC_DSSALL " ; sync" ::: "memory"); power4_idle_nap(); diff --git a/arch/powerpc/kernel/idle_6xx.S b/arch/powerpc/kernel/idle_6xx.S index 13cad9297d..3c09735636 100644 --- a/arch/powerpc/kernel/idle_6xx.S +++ b/arch/powerpc/kernel/idle_6xx.S @@ -129,7 +129,7 @@ BEGIN_FTR_SECTION END_FTR_SECTION_IFCLR(CPU_FTR_NO_DPM) mtspr SPRN_HID0,r4 BEGIN_FTR_SECTION - DSSALL + PPC_DSSALL sync END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) lwz r8,TI_LOCAL_FLAGS(r2) /* set napping bit */ diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index df048e331c..7cd6ce3ec4 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -81,7 +81,7 @@ notrace long system_call_exception(long r3, long r4, long r5, { syscall_fn f; - kuep_lock(); + kuap_lock(); regs->orig_gpr3 = r3; @@ -181,7 +181,7 @@ notrace long system_call_exception(long r3, long r4, long r5, local_irq_enable(); - if (unlikely(current_thread_info()->flags & _TIF_SYSCALL_DOTRACE)) { + if (unlikely(read_thread_flags() & _TIF_SYSCALL_DOTRACE)) { if (unlikely(trap_is_unsupported_scv(regs))) { /* Unsupported scv vector */ _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); @@ -343,7 +343,7 @@ interrupt_exit_user_prepare_main(unsigned long ret, struct pt_regs *regs) unsigned long ti_flags; again: - ti_flags = READ_ONCE(current_thread_info()->flags); + ti_flags = read_thread_flags(); while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); if (ti_flags & _TIF_NEED_RESCHED) { @@ -359,7 +359,7 @@ interrupt_exit_user_prepare_main(unsigned long ret, struct pt_regs *regs) do_notify_resume(regs, ti_flags); } local_irq_disable(); - ti_flags = READ_ONCE(current_thread_info()->flags); + ti_flags = read_thread_flags(); } if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) { @@ -406,7 +406,6 @@ interrupt_exit_user_prepare_main(unsigned long ret, struct pt_regs *regs) /* Restore user access locks last */ kuap_user_restore(regs); - kuep_unlock(); return ret; } @@ -437,7 +436,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, /* Check whether the syscall is issued inside a restartable sequence */ rseq_syscall(regs); - ti_flags = current_thread_info()->flags; + ti_flags = read_thread_flags(); if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) { if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) { @@ -532,8 +531,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) unsigned long flags; unsigned long ret = 0; unsigned long kuap; - bool stack_store = current_thread_info()->flags & - _TIF_EMULATE_STACK_STORE; + bool stack_store = read_thread_flags() & _TIF_EMULATE_STACK_STORE; if (regs_is_unrecoverable(regs)) unrecoverable_exception(regs); @@ -554,7 +552,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) again: if (IS_ENABLED(CONFIG_PREEMPT)) { /* Return to preemptible kernel context */ - if (unlikely(current_thread_info()->flags & _TIF_NEED_RESCHED)) { + if (unlikely(read_thread_flags() & _TIF_NEED_RESCHED)) { if (preempt_count() == 0) preempt_schedule_irq(); } diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S index 4c6d1a8dce..7bab2d7de3 100644 --- a/arch/powerpc/kernel/interrupt_64.S +++ b/arch/powerpc/kernel/interrupt_64.S @@ -166,10 +166,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) * The value of AMR only matters while we're in the kernel. */ mtcr r2 - ld r2,GPR2(r1) - ld r3,GPR3(r1) - ld r13,GPR13(r1) - ld r1,GPR1(r1) + REST_GPRS(2, 3, r1) + REST_GPR(13, r1) + REST_GPR(1, r1) RFSCV_TO_USER b . /* prevent speculative execution */ @@ -187,9 +186,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) mtctr r3 mtlr r4 mtspr SPRN_XER,r5 - REST_10GPRS(2, r1) - REST_2GPRS(12, r1) - ld r1,GPR1(r1) + REST_GPRS(2, 13, r1) + REST_GPR(1, r1) RFI_TO_USER .Lsyscall_vectored_\name\()_rst_end: @@ -378,10 +376,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) * The value of AMR only matters while we're in the kernel. */ mtcr r2 - ld r2,GPR2(r1) - ld r3,GPR3(r1) - ld r13,GPR13(r1) - ld r1,GPR1(r1) + REST_GPRS(2, 3, r1) + REST_GPR(13, r1) + REST_GPR(1, r1) RFI_TO_USER b . /* prevent speculative execution */ @@ -392,8 +389,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) mtctr r3 mtspr SPRN_XER,r4 ld r0,GPR0(r1) - REST_8GPRS(4, r1) - ld r12,GPR12(r1) + REST_GPRS(4, 12, r1) b .Lsyscall_restore_regs_cont .Lsyscall_rst_end: @@ -522,17 +518,14 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) ld r6,_XER(r1) li r0,0 - REST_4GPRS(7, r1) - REST_2GPRS(11, r1) - REST_GPR(13, r1) + REST_GPRS(7, 13, r1) mtcr r3 mtlr r4 mtctr r5 mtspr SPRN_XER,r6 - REST_4GPRS(2, r1) - REST_GPR(6, r1) + REST_GPRS(2, 6, r1) REST_GPR(0, r1) REST_GPR(1, r1) .ifc \srr,srr @@ -629,8 +622,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) ld r6,_CCR(r1) li r0,0 - REST_4GPRS(7, r1) - REST_2GPRS(11, r1) + REST_GPRS(7, 12, r1) mtlr r3 mtctr r4 @@ -642,7 +634,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) */ std r0,STACK_FRAME_OVERHEAD-16(r1) - REST_4GPRS(2, r1) + REST_GPRS(2, 5, r1) bne- cr1,1f /* emulate stack store */ mtcr r6 @@ -707,7 +699,7 @@ interrupt_return_macro hsrr .globl __end_soft_masked __end_soft_masked: -DEFINE_FIXED_SYMBOL(__end_soft_masked) +DEFINE_FIXED_SYMBOL(__end_soft_masked, text) #endif /* CONFIG_PPC_BOOK3S */ #ifdef CONFIG_PPC_BOOK3S diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index c4f1d6b7d9..2cf31a9712 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -745,7 +745,8 @@ void __do_irq(struct pt_regs *regs) irq = ppc_md.get_irq(); /* We can hard enable interrupts now to allow perf interrupts */ - may_hard_irq_enable(); + if (should_hard_irq_enable()) + do_hard_irq_enable(); /* And finally process it */ if (unlikely(!irq)) @@ -811,7 +812,7 @@ void __init init_IRQ(void) ppc_md.init_IRQ(); } -#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) +#ifdef CONFIG_BOOKE_OR_40x void *critirq_ctx[NR_CPUS] __read_mostly; void *dbgirq_ctx[NR_CPUS] __read_mostly; void *mcheckirq_ctx[NR_CPUS] __read_mostly; diff --git a/arch/powerpc/kernel/kgdb.c b/arch/powerpc/kernel/kgdb.c index bdee7262c0..9f8d0fa7b7 100644 --- a/arch/powerpc/kernel/kgdb.c +++ b/arch/powerpc/kernel/kgdb.c @@ -48,7 +48,7 @@ static struct hard_trap_info { 0x0800, 0x08 /* SIGFPE */ }, /* fp unavailable */ { 0x0900, 0x0e /* SIGALRM */ }, /* decrementer */ { 0x0c00, 0x14 /* SIGCHLD */ }, /* system call */ -#if defined(CONFIG_40x) || defined(CONFIG_BOOKE) +#ifdef CONFIG_BOOKE_OR_40x { 0x2002, 0x05 /* SIGTRAP */ }, /* debug */ #if defined(CONFIG_FSL_BOOKE) { 0x2010, 0x08 /* SIGFPE */ }, /* spe unavailable */ @@ -67,7 +67,7 @@ static struct hard_trap_info { 0x2010, 0x08 /* SIGFPE */ }, /* fp unavailable */ { 0x2020, 0x08 /* SIGFPE */ }, /* ap unavailable */ #endif -#else /* ! (defined(CONFIG_40x) || defined(CONFIG_BOOKE)) */ +#else /* !CONFIG_BOOKE_OR_40x */ { 0x0d00, 0x05 /* SIGTRAP */ }, /* single-step */ #if defined(CONFIG_PPC_8xx) { 0x1000, 0x04 /* SIGILL */ }, /* software emulation */ diff --git a/arch/powerpc/kernel/kprobes-ftrace.c b/arch/powerpc/kernel/kprobes-ftrace.c index 7154d58338..072ebe7f29 100644 --- a/arch/powerpc/kernel/kprobes-ftrace.c +++ b/arch/powerpc/kernel/kprobes-ftrace.c @@ -26,7 +26,6 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, return; regs = ftrace_get_regs(fregs); - preempt_disable_notrace(); p = get_kprobe((kprobe_opcode_t *)nip); if (unlikely(!p) || kprobe_disabled(p)) goto out; @@ -61,7 +60,6 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, __this_cpu_write(current_kprobe, NULL); } out: - preempt_enable_notrace(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(kprobe_ftrace_handler); diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c index 7a7cd6bda5..9a492fdec1 100644 --- a/arch/powerpc/kernel/kprobes.c +++ b/arch/powerpc/kernel/kprobes.c @@ -124,7 +124,7 @@ int arch_prepare_kprobe(struct kprobe *p) { int ret = 0; struct kprobe *prev; - struct ppc_inst insn = ppc_inst_read(p->addr); + ppc_inst_t insn = ppc_inst_read(p->addr); if ((unsigned long)p->addr & 0x03) { printk("Attempt to register kprobe at an unaligned address\n"); @@ -237,14 +237,14 @@ void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) ri->fp = NULL; /* Replace the return addr with trampoline addr */ - regs->link = (unsigned long)kretprobe_trampoline; + regs->link = (unsigned long)__kretprobe_trampoline; } NOKPROBE_SYMBOL(arch_prepare_kretprobe); static int try_to_emulate(struct kprobe *p, struct pt_regs *regs) { int ret; - struct ppc_inst insn = ppc_inst_read(p->ainsn.insn); + ppc_inst_t insn = ppc_inst_read(p->ainsn.insn); /* regs->nip is also adjusted if emulate_step returns 1 */ ret = emulate_step(regs, insn); @@ -403,12 +403,12 @@ NOKPROBE_SYMBOL(kprobe_handler); * - When the probed function returns, this probe * causes the handlers to fire */ -asm(".global kretprobe_trampoline\n" - ".type kretprobe_trampoline, @function\n" - "kretprobe_trampoline:\n" +asm(".global __kretprobe_trampoline\n" + ".type __kretprobe_trampoline, @function\n" + "__kretprobe_trampoline:\n" "nop\n" "blr\n" - ".size kretprobe_trampoline, .-kretprobe_trampoline\n"); + ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n"); /* * Called when the probe at kretprobe trampoline is hit @@ -417,7 +417,7 @@ static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) { unsigned long orig_ret_address; - orig_ret_address = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL); + orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); /* * We get here through one of two paths: * 1. by taking a trap -> kprobe_handler() -> here @@ -427,7 +427,7 @@ static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) * as it is used to determine the return address from the trap. * For (2), since nip is not honoured with optprobes, we instead setup * the link register properly so that the subsequent 'blr' in - * kretprobe_trampoline jumps back to the right instruction. + * __kretprobe_trampoline jumps back to the right instruction. * * For nip, we should set the address to the previous instruction since * we end up emulating it in kprobe_handler(), which increments the nip @@ -542,19 +542,8 @@ int kprobe_fault_handler(struct pt_regs *regs, int trapnr) } NOKPROBE_SYMBOL(kprobe_fault_handler); -unsigned long arch_deref_entry_point(void *entry) -{ -#ifdef PPC64_ELF_ABI_v1 - if (!kernel_text_address((unsigned long)entry)) - return ppc_global_function_entry(entry); - else -#endif - return (unsigned long)entry; -} -NOKPROBE_SYMBOL(arch_deref_entry_point); - static struct kprobe trampoline_p = { - .addr = (kprobe_opcode_t *) &kretprobe_trampoline, + .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, .pre_handler = trampoline_probe_handler }; @@ -565,7 +554,7 @@ int __init arch_init_kprobes(void) int arch_trampoline_kprobe(struct kprobe *p) { - if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline) + if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) return 1; return 0; diff --git a/arch/powerpc/kernel/kvm.c b/arch/powerpc/kernel/kvm.c index 617eba8253..6568823cf3 100644 --- a/arch/powerpc/kernel/kvm.c +++ b/arch/powerpc/kernel/kvm.c @@ -669,7 +669,8 @@ static void __init kvm_use_magic_page(void) on_each_cpu(kvm_map_magic_page, &features, 1); /* Quick self-test to see if the mapping works */ - if (!fault_in_pages_readable((const char *)KVM_MAGIC_PAGE, sizeof(u32))) { + if (fault_in_readable((const char __user *)KVM_MAGIC_PAGE, + sizeof(u32))) { kvm_patching_worked = false; return; } diff --git a/arch/powerpc/kernel/l2cr_6xx.S b/arch/powerpc/kernel/l2cr_6xx.S index 225511d73b..f2e03ed423 100644 --- a/arch/powerpc/kernel/l2cr_6xx.S +++ b/arch/powerpc/kernel/l2cr_6xx.S @@ -96,7 +96,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_L2CR) /* Stop DST streams */ BEGIN_FTR_SECTION - DSSALL + PPC_DSSALL sync END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) @@ -292,7 +292,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_L3CR) isync /* Stop DST streams */ - DSSALL + PPC_DSSALL sync /* Get the current enable bit of the L3CR into r4 */ @@ -401,7 +401,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_L3CR) _GLOBAL(__flush_disable_L1) /* Stop pending alitvec streams and memory accesses */ BEGIN_FTR_SECTION - DSSALL + PPC_DSSALL END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) sync diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c index fd829f7f25..2503dd4713 100644 --- a/arch/powerpc/kernel/mce.c +++ b/arch/powerpc/kernel/mce.c @@ -586,7 +586,7 @@ void machine_check_print_event_info(struct machine_check_event *evt, mc_error_class[evt->error_class] : "Unknown"; printk("%sMCE: CPU%d: %s\n", level, evt->cpu, subtype); -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU /* Display faulty slb contents for SLB errors. */ if (evt->error_type == MCE_ERROR_TYPE_SLB && !in_guest) slb_dump_contents(local_paca->mce_faulty_slbs); diff --git a/arch/powerpc/kernel/mce_power.c b/arch/powerpc/kernel/mce_power.c index c2f55fe709..71e8f2a92e 100644 --- a/arch/powerpc/kernel/mce_power.c +++ b/arch/powerpc/kernel/mce_power.c @@ -77,15 +77,15 @@ static bool mce_in_guest(void) } /* flush SLBs and reload */ -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU void flush_and_reload_slb(void) { - /* Invalidate all SLBs */ - slb_flush_all_realmode(); - if (early_radix_enabled()) return; + /* Invalidate all SLBs */ + slb_flush_all_realmode(); + /* * This probably shouldn't happen, but it may be possible it's * called in early boot before SLB shadows are allocated. @@ -99,7 +99,7 @@ void flush_and_reload_slb(void) void flush_erat(void) { -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU if (!early_cpu_has_feature(CPU_FTR_ARCH_300)) { flush_and_reload_slb(); return; @@ -114,7 +114,7 @@ void flush_erat(void) static int mce_flush(int what) { -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU if (what == MCE_FLUSH_SLB) { flush_and_reload_slb(); return 1; @@ -455,7 +455,7 @@ static int mce_find_instr_ea_and_phys(struct pt_regs *regs, uint64_t *addr, * in real-mode is tricky and can lead to recursive * faults */ - struct ppc_inst instr; + ppc_inst_t instr; unsigned long pfn, instr_addr; struct instruction_op op; struct pt_regs tmp = *regs; @@ -499,8 +499,10 @@ static int mce_handle_ierror(struct pt_regs *regs, unsigned long srr1, /* attempt to correct the error */ switch (table[i].error_type) { case MCE_ERROR_TYPE_SLB: +#ifdef CONFIG_PPC_64S_HASH_MMU if (local_paca->in_mce == 1) slb_save_contents(local_paca->mce_faulty_slbs); +#endif handled = mce_flush(MCE_FLUSH_SLB); break; case MCE_ERROR_TYPE_ERAT: @@ -588,8 +590,10 @@ static int mce_handle_derror(struct pt_regs *regs, /* attempt to correct the error */ switch (table[i].error_type) { case MCE_ERROR_TYPE_SLB: +#ifdef CONFIG_PPC_64S_HASH_MMU if (local_paca->in_mce == 1) slb_save_contents(local_paca->mce_faulty_slbs); +#endif if (mce_flush(MCE_FLUSH_SLB)) handled = 1; break; diff --git a/arch/powerpc/kernel/module_32.c b/arch/powerpc/kernel/module_32.c index f417afc08d..a491ad481d 100644 --- a/arch/powerpc/kernel/module_32.c +++ b/arch/powerpc/kernel/module_32.c @@ -273,6 +273,31 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, } #ifdef CONFIG_DYNAMIC_FTRACE +int module_trampoline_target(struct module *mod, unsigned long addr, + unsigned long *target) +{ + unsigned int jmp[4]; + + /* Find where the trampoline jumps to */ + if (copy_from_kernel_nofault(jmp, (void *)addr, sizeof(jmp))) + return -EFAULT; + + /* verify that this is what we expect it to be */ + if ((jmp[0] & 0xffff0000) != PPC_RAW_LIS(_R12, 0) || + (jmp[1] & 0xffff0000) != PPC_RAW_ADDI(_R12, _R12, 0) || + jmp[2] != PPC_RAW_MTCTR(_R12) || + jmp[3] != PPC_RAW_BCTR()) + return -EINVAL; + + addr = (jmp[1] & 0xffff) | ((jmp[0] & 0xffff) << 16); + if (addr & 0x8000) + addr -= 0x10000; + + *target = addr; + + return 0; +} + int module_finalize_ftrace(struct module *module, const Elf_Shdr *sechdrs) { module->arch.tramp = do_plt_call(module->core_layout.base, @@ -281,6 +306,14 @@ int module_finalize_ftrace(struct module *module, const Elf_Shdr *sechdrs) if (!module->arch.tramp) return -ENOENT; +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + module->arch.tramp_regs = do_plt_call(module->core_layout.base, + (unsigned long)ftrace_regs_caller, + sechdrs, module); + if (!module->arch.tramp_regs) + return -ENOENT; +#endif + return 0; } #endif diff --git a/arch/powerpc/kernel/nvram_64.c b/arch/powerpc/kernel/nvram_64.c index 3c8d9bbb51..0d9f9cd41e 100644 --- a/arch/powerpc/kernel/nvram_64.c +++ b/arch/powerpc/kernel/nvram_64.c @@ -540,7 +540,7 @@ static struct pstore_info nvram_pstore_info = { .write = nvram_pstore_write, }; -static int nvram_pstore_init(void) +static int __init nvram_pstore_init(void) { int rc = 0; @@ -562,7 +562,7 @@ static int nvram_pstore_init(void) return rc; } #else -static int nvram_pstore_init(void) +static int __init nvram_pstore_init(void) { return -1; } @@ -755,7 +755,7 @@ static unsigned char __init nvram_checksum(struct nvram_header *p) * Per the criteria passed via nvram_remove_partition(), should this * partition be removed? 1=remove, 0=keep */ -static int nvram_can_remove_partition(struct nvram_partition *part, +static int __init nvram_can_remove_partition(struct nvram_partition *part, const char *name, int sig, const char *exceptions[]) { if (part->header.signature != sig) diff --git a/arch/powerpc/kernel/optprobes.c b/arch/powerpc/kernel/optprobes.c index c79899abce..3b1c2236cb 100644 --- a/arch/powerpc/kernel/optprobes.c +++ b/arch/powerpc/kernel/optprobes.c @@ -56,7 +56,7 @@ static unsigned long can_optimize(struct kprobe *p) * has a 'nop' instruction, which can be emulated. * So further checks can be skipped. */ - if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline) + if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) return addr + sizeof(kprobe_opcode_t); /* @@ -153,7 +153,7 @@ static void patch_imm_load_insns(unsigned long val, int reg, kprobe_opcode_t *ad int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p) { - struct ppc_inst branch_op_callback, branch_emulate_step, temp; + ppc_inst_t branch_op_callback, branch_emulate_step, temp; unsigned long op_callback_addr, emulate_step_addr; kprobe_opcode_t *buff; long b_offset; @@ -228,12 +228,8 @@ int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p) /* * 3. load instruction to be emulated into relevant register, and */ - if (IS_ENABLED(CONFIG_PPC64)) { - temp = ppc_inst_read(p->ainsn.insn); - patch_imm_load_insns(ppc_inst_as_ulong(temp), 4, buff + TMPL_INSN_IDX); - } else { - patch_imm_load_insns((unsigned long)p->ainsn.insn, 4, buff + TMPL_INSN_IDX); - } + temp = ppc_inst_read(p->ainsn.insn); + patch_imm_load_insns(ppc_inst_as_ulong(temp), 4, buff + TMPL_INSN_IDX); /* * 4. branch back from trampoline @@ -269,7 +265,7 @@ int arch_check_optimized_kprobe(struct optimized_kprobe *op) void arch_optimize_kprobes(struct list_head *oplist) { - struct ppc_inst instr; + ppc_inst_t instr; struct optimized_kprobe *op; struct optimized_kprobe *tmp; @@ -301,8 +297,8 @@ void arch_unoptimize_kprobes(struct list_head *oplist, struct list_head *done_li } } -int arch_within_optimized_kprobe(struct optimized_kprobe *op, unsigned long addr) +int arch_within_optimized_kprobe(struct optimized_kprobe *op, kprobe_opcode_t *addr) { - return ((unsigned long)op->kp.addr <= addr && - (unsigned long)op->kp.addr + RELATIVEJUMP_SIZE > addr); + return (op->kp.addr <= addr && + op->kp.addr + (RELATIVEJUMP_SIZE / sizeof(kprobe_opcode_t)) > addr); } diff --git a/arch/powerpc/kernel/optprobes_head.S b/arch/powerpc/kernel/optprobes_head.S index 19ea331240..5c7f0b4b78 100644 --- a/arch/powerpc/kernel/optprobes_head.S +++ b/arch/powerpc/kernel/optprobes_head.S @@ -10,8 +10,8 @@ #include #ifdef CONFIG_PPC64 -#define SAVE_30GPRS(base) SAVE_10GPRS(2,base); SAVE_10GPRS(12,base); SAVE_10GPRS(22,base) -#define REST_30GPRS(base) REST_10GPRS(2,base); REST_10GPRS(12,base); REST_10GPRS(22,base) +#define SAVE_30GPRS(base) SAVE_GPRS(2, 31, base) +#define REST_30GPRS(base) REST_GPRS(2, 31, base) #define TEMPLATE_FOR_IMM_LOAD_INSNS nop; nop; nop; nop; nop #else #define SAVE_30GPRS(base) stmw r2, GPR2(base) diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c index 9bd30cac85..39da688a94 100644 --- a/arch/powerpc/kernel/paca.c +++ b/arch/powerpc/kernel/paca.c @@ -139,8 +139,7 @@ static struct lppaca * __init new_lppaca(int cpu, unsigned long limit) } #endif /* CONFIG_PPC_PSERIES */ -#ifdef CONFIG_PPC_BOOK3S_64 - +#ifdef CONFIG_PPC_64S_HASH_MMU /* * 3 persistent SLBs are allocated here. The buffer will be zero * initially, hence will all be invaild until we actually write them. @@ -169,8 +168,7 @@ static struct slb_shadow * __init new_slb_shadow(int cpu, unsigned long limit) return s; } - -#endif /* CONFIG_PPC_BOOK3S_64 */ +#endif /* CONFIG_PPC_64S_HASH_MMU */ #ifdef CONFIG_PPC_PSERIES /** @@ -226,7 +224,7 @@ void __init initialise_paca(struct paca_struct *new_paca, int cpu) new_paca->kexec_state = KEXEC_STATE_NONE; new_paca->__current = &init_task; new_paca->data_offset = 0xfeeeeeeeeeeeeeeeULL; -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU new_paca->slb_shadow_ptr = NULL; #endif @@ -307,7 +305,7 @@ void __init allocate_paca(int cpu) #ifdef CONFIG_PPC_PSERIES paca->lppaca_ptr = new_lppaca(cpu, limit); #endif -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU paca->slb_shadow_ptr = new_slb_shadow(cpu, limit); #endif #ifdef CONFIG_PPC_PSERIES @@ -322,17 +320,17 @@ void __init free_unused_pacas(void) new_ptrs_size = sizeof(struct paca_struct *) * nr_cpu_ids; if (new_ptrs_size < paca_ptrs_size) - memblock_free(__pa(paca_ptrs) + new_ptrs_size, - paca_ptrs_size - new_ptrs_size); + memblock_phys_free(__pa(paca_ptrs) + new_ptrs_size, + paca_ptrs_size - new_ptrs_size); paca_nr_cpu_ids = nr_cpu_ids; paca_ptrs_size = new_ptrs_size; -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU if (early_radix_enabled()) { /* Ugly fixup, see new_slb_shadow() */ - memblock_free(__pa(paca_ptrs[boot_cpuid]->slb_shadow_ptr), - sizeof(struct slb_shadow)); + memblock_phys_free(__pa(paca_ptrs[boot_cpuid]->slb_shadow_ptr), + sizeof(struct slb_shadow)); paca_ptrs[boot_cpuid]->slb_shadow_ptr = NULL; } #endif @@ -341,9 +339,9 @@ void __init free_unused_pacas(void) paca_ptrs_size + paca_struct_size, nr_cpu_ids); } +#ifdef CONFIG_PPC_64S_HASH_MMU void copy_mm_to_paca(struct mm_struct *mm) { -#ifdef CONFIG_PPC_BOOK3S mm_context_t *context = &mm->context; #ifdef CONFIG_PPC_MM_SLICES @@ -356,7 +354,5 @@ void copy_mm_to_paca(struct mm_struct *mm) get_paca()->mm_ctx_user_psize = context->user_psize; get_paca()->mm_ctx_sllp = context->sllp; #endif -#else /* !CONFIG_PPC_BOOK3S */ - return; -#endif } +#endif /* CONFIG_PPC_64S_HASH_MMU */ diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index c357343091..8bc9cf62cd 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -62,7 +62,7 @@ EXPORT_SYMBOL(isa_mem_base); static const struct dma_map_ops *pci_dma_ops; -void set_pci_dma_ops(const struct dma_map_ops *dma_ops) +void __init set_pci_dma_ops(const struct dma_map_ops *dma_ops) { pci_dma_ops = dma_ops; } @@ -1059,7 +1059,7 @@ void pcibios_bus_add_device(struct pci_dev *dev) ppc_md.pcibios_bus_add_device(dev); } -int pcibios_add_device(struct pci_dev *dev) +int pcibios_device_add(struct pci_dev *dev) { struct irq_domain *d; diff --git a/arch/powerpc/kernel/pci_32.c b/arch/powerpc/kernel/pci_32.c index b49e1060a3..48537964fb 100644 --- a/arch/powerpc/kernel/pci_32.c +++ b/arch/powerpc/kernel/pci_32.c @@ -37,7 +37,7 @@ int pcibios_assign_bus_offset = 1; EXPORT_SYMBOL(isa_io_base); EXPORT_SYMBOL(pci_dram_offset); -void pcibios_make_OF_bus_map(void); +void __init pcibios_make_OF_bus_map(void); static void fixup_cpc710_pci64(struct pci_dev* dev); static u8* pci_to_OF_bus_map; @@ -109,7 +109,7 @@ make_one_node_map(struct device_node* node, u8 pci_bus) } } -void +void __init pcibios_make_OF_bus_map(void) { int i; diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 50436b52c2..984813a4d5 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -628,7 +628,7 @@ static void do_break_handler(struct pt_regs *regs) { struct arch_hw_breakpoint null_brk = {0}; struct arch_hw_breakpoint *info; - struct ppc_inst instr = ppc_inst(0); + ppc_inst_t instr = ppc_inst(0); int type = 0; int size = 0; unsigned long ea; @@ -1156,6 +1156,40 @@ static inline void save_sprs(struct thread_struct *t) #endif } +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +void kvmppc_save_user_regs(void) +{ + unsigned long usermsr; + + if (!current->thread.regs) + return; + + usermsr = current->thread.regs->msr; + + if (usermsr & MSR_FP) + save_fpu(current); + + if (usermsr & MSR_VEC) + save_altivec(current); + +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM + if (usermsr & MSR_TM) { + current->thread.tm_tfhar = mfspr(SPRN_TFHAR); + current->thread.tm_tfiar = mfspr(SPRN_TFIAR); + current->thread.tm_texasr = mfspr(SPRN_TEXASR); + current->thread.regs->msr &= ~MSR_TM; + } +#endif +} +EXPORT_SYMBOL_GPL(kvmppc_save_user_regs); + +void kvmppc_save_current_sprs(void) +{ + save_sprs(¤t->thread); +} +EXPORT_SYMBOL_GPL(kvmppc_save_current_sprs); +#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ + static inline void restore_sprs(struct thread_struct *old_thread, struct thread_struct *new_thread) { @@ -1206,7 +1240,7 @@ struct task_struct *__switch_to(struct task_struct *prev, { struct thread_struct *new_thread, *old_thread; struct task_struct *last; -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU struct ppc64_tlb_batch *batch; #endif @@ -1215,7 +1249,7 @@ struct task_struct *__switch_to(struct task_struct *prev, WARN_ON(!irqs_disabled()); -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU batch = this_cpu_ptr(&ppc64_tlb_batch); if (batch->active) { current_thread_info()->local_flags |= _TLF_LAZY_MMU; @@ -1281,9 +1315,9 @@ struct task_struct *__switch_to(struct task_struct *prev, set_return_regs_changed(); /* _switch changes stack (and regs) */ -#ifdef CONFIG_PPC32 - kuap_assert_locked(); -#endif + if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + kuap_assert_locked(); + last = _switch(old_thread, new_thread); /* @@ -1294,6 +1328,7 @@ struct task_struct *__switch_to(struct task_struct *prev, */ #ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU /* * This applies to a process that was context switched while inside * arch_enter_lazy_mmu_mode(), to re-activate the batch that was @@ -1305,6 +1340,7 @@ struct task_struct *__switch_to(struct task_struct *prev, batch = this_cpu_ptr(&ppc64_tlb_batch); batch->active = 1; } +#endif /* * Math facilities are masked out of the child MSR in copy_thread. @@ -1655,7 +1691,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) static void setup_ksp_vsid(struct task_struct *p, unsigned long sp) { -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU unsigned long sp_vsid; unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp; @@ -1767,6 +1803,9 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, #if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_PPC_KUAP) p->thread.kuap = KUAP_NONE; #endif +#if defined(CONFIG_BOOKE_OR_40x) && defined(CONFIG_PPC_KUAP) + p->thread.pid = MMU_NO_CONTEXT; +#endif setup_ksp_vsid(p, sp); @@ -2111,14 +2150,11 @@ int validate_sp(unsigned long sp, struct task_struct *p, EXPORT_SYMBOL(validate_sp); -static unsigned long __get_wchan(struct task_struct *p) +static unsigned long ___get_wchan(struct task_struct *p) { unsigned long ip, sp; int count = 0; - if (!p || p == current || task_is_running(p)) - return 0; - sp = p->thread.ksp; if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD)) return 0; @@ -2137,14 +2173,14 @@ static unsigned long __get_wchan(struct task_struct *p) return 0; } -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { unsigned long ret; if (!try_get_task_stack(p)) return 0; - ret = __get_wchan(p); + ret = ___get_wchan(p); put_task_stack(p); @@ -2302,10 +2338,9 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) * the heap, we can put it above 1TB so it is backed by a 1TB * segment. Otherwise the heap will be in the bottom 1TB * which always uses 256MB segments and this may result in a - * performance penalty. We don't need to worry about radix. For - * radix, mmu_highuser_ssize remains unchanged from 256MB. + * performance penalty. */ - if (!is_32bit_task() && (mmu_highuser_ssize == MMU_SEGSIZE_1T)) + if (!radix_enabled() && !is_32bit_task() && (mmu_highuser_ssize == MMU_SEGSIZE_1T)) base = max_t(unsigned long, mm->brk, 1UL << SID_SHIFT_1T); #endif diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 2e67588f6f..3d30d40a0e 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -231,7 +231,7 @@ static void __init check_cpu_pa_features(unsigned long node) ibm_pa_features, ARRAY_SIZE(ibm_pa_features)); } -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU static void __init init_mmu_slb_size(unsigned long node) { const __be32 *slb_size_ptr; @@ -402,7 +402,7 @@ static int __init early_init_dt_scan_chosen_ppc(unsigned long node, const unsigned long *lprop; /* All these set by kernel, so no need to convert endian */ /* Use common scan routine to determine if this is the chosen node */ - if (early_init_dt_scan_chosen(node, uname, depth, data) == 0) + if (early_init_dt_scan_chosen(data) < 0) return 0; #ifdef CONFIG_PPC64 @@ -447,7 +447,7 @@ static int __init early_init_dt_scan_chosen_ppc(unsigned long node, */ #ifdef CONFIG_SPARSEMEM -static bool validate_mem_limit(u64 base, u64 *size) +static bool __init validate_mem_limit(u64 base, u64 *size) { u64 max_mem = 1UL << (MAX_PHYSMEM_BITS); @@ -458,7 +458,7 @@ static bool validate_mem_limit(u64 base, u64 *size) return true; } #else -static bool validate_mem_limit(u64 base, u64 *size) +static bool __init validate_mem_limit(u64 base, u64 *size) { return true; } @@ -532,19 +532,18 @@ static int __init early_init_drmem_lmb(struct drmem_lmb *lmb, } #endif /* CONFIG_PPC_PSERIES */ -static int __init early_init_dt_scan_memory_ppc(unsigned long node, - const char *uname, - int depth, void *data) +static int __init early_init_dt_scan_memory_ppc(void) { #ifdef CONFIG_PPC_PSERIES - if (depth == 1 && - strcmp(uname, "ibm,dynamic-reconfiguration-memory") == 0) { + const void *fdt = initial_boot_params; + int node = fdt_path_offset(fdt, "/ibm,dynamic-reconfiguration-memory"); + + if (node > 0) walk_drmem_lmbs_early(node, NULL, early_init_drmem_lmb); - return 0; - } + #endif - - return early_init_dt_scan_memory(node, uname, depth, data); + + return early_init_dt_scan_memory(); } /* @@ -748,8 +747,8 @@ void __init early_init_devtree(void *params) of_scan_flat_dt(early_init_dt_scan_chosen_ppc, boot_command_line); /* Scan memory nodes and rebuild MEMBLOCKs */ - of_scan_flat_dt(early_init_dt_scan_root, NULL); - of_scan_flat_dt(early_init_dt_scan_memory_ppc, NULL); + early_init_dt_scan_root(); + early_init_dt_scan_memory_ppc(); parse_early_param(); @@ -857,8 +856,8 @@ void __init early_get_first_memblock_info(void *params, phys_addr_t *size) * mess the memblock. */ add_mem_to_memblock = 0; - of_scan_flat_dt(early_init_dt_scan_root, NULL); - of_scan_flat_dt(early_init_dt_scan_memory_ppc, NULL); + early_init_dt_scan_root(); + early_init_dt_scan_memory_ppc(); add_mem_to_memblock = 1; if (size) diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index f845065c86..0ac5faacc9 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -672,7 +672,7 @@ static inline int __init prom_getproplen(phandle node, const char *pname) return call_prom("getproplen", 2, 1, node, ADDR(pname)); } -static void add_string(char **str, const char *q) +static void __init add_string(char **str, const char *q) { char *p = *str; @@ -682,7 +682,7 @@ static void add_string(char **str, const char *q) *str = p; } -static char *tohex(unsigned int x) +static char *__init tohex(unsigned int x) { static const char digits[] __initconst = "0123456789abcdef"; static char result[9] __prombss; @@ -728,7 +728,7 @@ static int __init prom_setprop(phandle node, const char *nodename, #define prom_islower(c) ('a' <= (c) && (c) <= 'z') #define prom_toupper(c) (prom_islower(c) ? ((c) - 'a' + 'A') : (c)) -static unsigned long prom_strtoul(const char *cp, const char **endp) +static unsigned long __init prom_strtoul(const char *cp, const char **endp) { unsigned long result = 0, base = 10, value; @@ -753,7 +753,7 @@ static unsigned long prom_strtoul(const char *cp, const char **endp) return result; } -static unsigned long prom_memparse(const char *ptr, const char **retptr) +static unsigned long __init prom_memparse(const char *ptr, const char **retptr) { unsigned long ret = prom_strtoul(ptr, retptr); int shift = 0; @@ -1786,7 +1786,7 @@ static void __init prom_close_stdin(void) } #ifdef CONFIG_PPC_SVM -static int prom_rtas_hcall(uint64_t args) +static int __init prom_rtas_hcall(uint64_t args) { register uint64_t arg1 asm("r3") = H_RTAS; register uint64_t arg2 asm("r4") = args; @@ -3248,7 +3248,7 @@ static void __init prom_check_initrd(unsigned long r3, unsigned long r4) /* * Perform the Enter Secure Mode ultracall. */ -static int enter_secure_mode(unsigned long kbase, unsigned long fdt) +static int __init enter_secure_mode(unsigned long kbase, unsigned long fdt) { register unsigned long r3 asm("r3") = UV_ESM; register unsigned long r4 asm("r4") = kbase; diff --git a/arch/powerpc/kernel/ptrace/ptrace.c b/arch/powerpc/kernel/ptrace/ptrace.c index 7c7093c17c..c43f77e2ac 100644 --- a/arch/powerpc/kernel/ptrace/ptrace.c +++ b/arch/powerpc/kernel/ptrace/ptrace.c @@ -260,8 +260,7 @@ long do_syscall_trace_enter(struct pt_regs *regs) { u32 flags; - flags = READ_ONCE(current_thread_info()->flags) & - (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE); + flags = read_thread_flags() & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE); if (flags) { int rc = tracehook_report_syscall_entry(regs); diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index ff80bbad22..733e6ef367 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -492,8 +492,25 @@ int rtas_call(int token, int nargs, int nret, int *outputs, ...) } EXPORT_SYMBOL(rtas_call); -/* For RTAS_BUSY (-2), delay for 1 millisecond. For an extended busy status - * code of 990n, perform the hinted delay of 10^n (last digit) milliseconds. +/** + * rtas_busy_delay_time() - From an RTAS status value, calculate the + * suggested delay time in milliseconds. + * + * @status: a value returned from rtas_call() or similar APIs which return + * the status of a RTAS function call. + * + * Context: Any context. + * + * Return: + * * 100000 - If @status is 9905. + * * 10000 - If @status is 9904. + * * 1000 - If @status is 9903. + * * 100 - If @status is 9902. + * * 10 - If @status is 9901. + * * 1 - If @status is either 9900 or -2. This is "wrong" for -2, but + * some callers depend on this behavior, and the worst outcome + * is that they will delay for longer than necessary. + * * 0 - If @status is not a busy or extended delay value. */ unsigned int rtas_busy_delay_time(int status) { @@ -513,17 +530,77 @@ unsigned int rtas_busy_delay_time(int status) } EXPORT_SYMBOL(rtas_busy_delay_time); -/* For an RTAS busy status code, perform the hinted delay. */ -unsigned int rtas_busy_delay(int status) +/** + * rtas_busy_delay() - helper for RTAS busy and extended delay statuses + * + * @status: a value returned from rtas_call() or similar APIs which return + * the status of a RTAS function call. + * + * Context: Process context. May sleep or schedule. + * + * Return: + * * true - @status is RTAS_BUSY or an extended delay hint. The + * caller may assume that the CPU has been yielded if necessary, + * and that an appropriate delay for @status has elapsed. + * Generally the caller should reattempt the RTAS call which + * yielded @status. + * + * * false - @status is not @RTAS_BUSY nor an extended delay hint. The + * caller is responsible for handling @status. + */ +bool rtas_busy_delay(int status) { unsigned int ms; + bool ret; - might_sleep(); - ms = rtas_busy_delay_time(status); - if (ms && need_resched()) - msleep(ms); + switch (status) { + case RTAS_EXTENDED_DELAY_MIN...RTAS_EXTENDED_DELAY_MAX: + ret = true; + ms = rtas_busy_delay_time(status); + /* + * The extended delay hint can be as high as 100 seconds. + * Surely any function returning such a status is either + * buggy or isn't going to be significantly slowed by us + * polling at 1HZ. Clamp the sleep time to one second. + */ + ms = clamp(ms, 1U, 1000U); + /* + * The delay hint is an order-of-magnitude suggestion, not + * a minimum. It is fine, possibly even advantageous, for + * us to pause for less time than hinted. For small values, + * use usleep_range() to ensure we don't sleep much longer + * than actually needed. + * + * See Documentation/timers/timers-howto.rst for + * explanation of the threshold used here. In effect we use + * usleep_range() for 9900 and 9901, msleep() for + * 9902-9905. + */ + if (ms <= 20) + usleep_range(ms * 100, ms * 1000); + else + msleep(ms); + break; + case RTAS_BUSY: + ret = true; + /* + * We should call again immediately if there's no other + * work to do. + */ + cond_resched(); + break; + default: + ret = false; + /* + * Not a busy or extended delay status; the caller should + * handle @status itself. Ensure we warn on misuses in + * atomic context regardless. + */ + might_sleep(); + break; + } - return ms; + return ret; } EXPORT_SYMBOL(rtas_busy_delay); @@ -809,13 +886,13 @@ void rtas_os_term(char *str) /** * rtas_activate_firmware() - Activate a new version of firmware. * + * Context: This function may sleep. + * * Activate a new version of partition firmware. The OS must call this * after resuming from a partition hibernation or migration in order * to maintain the ability to perform live firmware updates. It's not * catastrophic for this method to be absent or to fail; just log the * condition in that case. - * - * Context: This function may sleep. */ void rtas_activate_firmware(void) { @@ -890,11 +967,12 @@ int rtas_call_reentrant(int token, int nargs, int nret, int *outputs, ...) #endif /* CONFIG_PPC_PSERIES */ /** - * Find a specific pseries error log in an RTAS extended event log. + * get_pseries_errorlog() - Find a specific pseries error log in an RTAS + * extended event log. * @log: RTAS error/event log * @section_id: two character section identifier * - * Returns a pointer to the specified errorlog or NULL if not found. + * Return: A pointer to the specified errorlog or NULL if not found. */ struct pseries_errorlog *get_pseries_errorlog(struct rtas_error_log *log, uint16_t section_id) diff --git a/arch/powerpc/kernel/rtasd.c b/arch/powerpc/kernel/rtasd.c index 32ee17753e..cf0f42909d 100644 --- a/arch/powerpc/kernel/rtasd.c +++ b/arch/powerpc/kernel/rtasd.c @@ -455,7 +455,7 @@ static void rtas_event_scan(struct work_struct *w) } #ifdef CONFIG_PPC64 -static void retrieve_nvram_error_log(void) +static void __init retrieve_nvram_error_log(void) { unsigned int err_type ; int rc ; @@ -473,12 +473,12 @@ static void retrieve_nvram_error_log(void) } } #else /* CONFIG_PPC64 */ -static void retrieve_nvram_error_log(void) +static void __init retrieve_nvram_error_log(void) { } #endif /* CONFIG_PPC64 */ -static void start_event_scan(void) +static void __init start_event_scan(void) { printk(KERN_DEBUG "RTAS daemon started\n"); pr_debug("rtasd: will sleep for %d milliseconds\n", diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c index 15fb5ea1b9..e159d4093d 100644 --- a/arch/powerpc/kernel/security.c +++ b/arch/powerpc/kernel/security.c @@ -44,7 +44,7 @@ static void enable_barrier_nospec(bool enable) do_barrier_nospec_fixups(enable); } -void setup_barrier_nospec(void) +void __init setup_barrier_nospec(void) { bool enable; @@ -132,7 +132,7 @@ early_param("nospectre_v2", handle_nospectre_v2); #endif /* CONFIG_PPC_FSL_BOOK3E || CONFIG_PPC_BOOK3S_64 */ #ifdef CONFIG_PPC_FSL_BOOK3E -void setup_spectre_v2(void) +void __init setup_spectre_v2(void) { if (no_spectrev2 || cpu_mitigations_off()) do_btb_flush_fixups(); diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index b1e43b69a5..f8da937df9 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -278,9 +278,6 @@ static int show_cpuinfo(struct seq_file *m, void *v) seq_printf(m, "clock\t\t: %lu.%06luMHz\n", proc_freq / 1000000, proc_freq % 1000000); - if (ppc_md.show_percpuinfo != NULL) - ppc_md.show_percpuinfo(m, cpu_id); - /* If we are a Freescale core do a simple check so * we dont have to keep adding cases in the future */ if (PVR_VER(pvr) & 0x8000) { @@ -585,7 +582,7 @@ static __init int add_pcspkr(void) device_initcall(add_pcspkr); #endif /* CONFIG_PCSPKR_PLATFORM */ -void probe_machine(void) +static __init void probe_machine(void) { extern struct machdep_calls __machine_desc_start; extern struct machdep_calls __machine_desc_end; @@ -825,7 +822,7 @@ static void __init smp_setup_pacas(void) set_hard_smp_processor_id(cpu, cpu_to_phys_id[cpu]); } - memblock_free(__pa(cpu_to_phys_id), nr_cpu_ids * sizeof(u32)); + memblock_free(cpu_to_phys_id, nr_cpu_ids * sizeof(u32)); cpu_to_phys_id = NULL; } #endif diff --git a/arch/powerpc/kernel/setup.h b/arch/powerpc/kernel/setup.h index 84058bbc8f..93f22da12a 100644 --- a/arch/powerpc/kernel/setup.h +++ b/arch/powerpc/kernel/setup.h @@ -29,7 +29,7 @@ void setup_tlb_core_data(void); static inline void setup_tlb_core_data(void) { } #endif -#if defined(CONFIG_PPC_BOOK3E) || defined(CONFIG_BOOKE) || defined(CONFIG_40x) +#ifdef CONFIG_BOOKE_OR_40x void exc_lvl_early_init(void); #else static inline void exc_lvl_early_init(void) { } diff --git a/arch/powerpc/kernel/setup_32.c b/arch/powerpc/kernel/setup_32.c index 7ec5c47fce..a6e9d36d7c 100644 --- a/arch/powerpc/kernel/setup_32.c +++ b/arch/powerpc/kernel/setup_32.c @@ -75,7 +75,7 @@ EXPORT_SYMBOL(DMA_MODE_WRITE); notrace void __init machine_init(u64 dt_ptr) { u32 *addr = (u32 *)patch_site_addr(&patch__memset_nocache); - struct ppc_inst insn; + ppc_inst_t insn; /* Configure static keys first, now that we're relocated. */ setup_feature_keys(); @@ -175,7 +175,7 @@ void __init emergency_stack_init(void) } #endif -#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) +#ifdef CONFIG_BOOKE_OR_40x void __init exc_lvl_early_init(void) { unsigned int i, hw_cpu; diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index eaa79a0996..be8577ac93 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -499,7 +499,7 @@ void smp_release_cpus(void) * routines and/or provided to userland */ -static void init_cache_info(struct ppc_cache_info *info, u32 size, u32 lsize, +static void __init init_cache_info(struct ppc_cache_info *info, u32 size, u32 lsize, u32 bsize, u32 sets) { info->size = size; @@ -771,50 +771,6 @@ void __init emergency_stack_init(void) } #ifdef CONFIG_SMP -/** - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu - * @cpu: cpu to allocate for - * @size: size allocation in bytes - * @align: alignment - * - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper - * does the right thing for NUMA regardless of the current - * configuration. - * - * RETURNS: - * Pointer to the allocated area on success, NULL on failure. - */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, - size_t align) -{ - const unsigned long goal = __pa(MAX_DMA_ADDRESS); -#ifdef CONFIG_NUMA - int node = early_cpu_to_node(cpu); - void *ptr; - - if (!node_online(node) || !NODE_DATA(node)) { - ptr = memblock_alloc_from(size, align, goal); - pr_info("cpu %d has no node %d or node-local memory\n", - cpu, node); - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", - cpu, size, __pa(ptr)); - } else { - ptr = memblock_alloc_try_nid(size, align, goal, - MEMBLOCK_ALLOC_ACCESSIBLE, node); - pr_debug("per cpu data for cpu%d %lu bytes on node%d at " - "%016lx\n", cpu, size, node, __pa(ptr)); - } - return ptr; -#else - return memblock_alloc_from(size, align, goal); -#endif -} - -static void __init pcpu_free_bootmem(void *ptr, size_t size) -{ - memblock_free(__pa(ptr), size); -} - static int pcpu_cpu_distance(unsigned int from, unsigned int to) { if (early_cpu_to_node(from) == early_cpu_to_node(to)) @@ -823,53 +779,13 @@ static int pcpu_cpu_distance(unsigned int from, unsigned int to) return REMOTE_DISTANCE; } -unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; -EXPORT_SYMBOL(__per_cpu_offset); - -static void __init pcpu_populate_pte(unsigned long addr) +static __init int pcpu_cpu_to_node(int cpu) { - pgd_t *pgd = pgd_offset_k(addr); - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - - p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { - pud_t *new; - - new = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); - if (!new) - goto err_alloc; - p4d_populate(&init_mm, p4d, new); - } - - pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { - pmd_t *new; - - new = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE); - if (!new) - goto err_alloc; - pud_populate(&init_mm, pud, new); - } - - pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) { - pte_t *new; - - new = memblock_alloc(PTE_TABLE_SIZE, PTE_TABLE_SIZE); - if (!new) - goto err_alloc; - pmd_populate_kernel(&init_mm, pmd, new); - } - - return; - -err_alloc: - panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + return early_cpu_to_node(cpu); } +unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(__per_cpu_offset); void __init setup_per_cpu_areas(void) { @@ -880,18 +796,27 @@ void __init setup_per_cpu_areas(void) int rc = -EINVAL; /* - * Linear mapping is one of 4K, 1M and 16M. For 4K, no need - * to group units. For larger mappings, use 1M atom which - * should be large enough to contain a number of units. + * BookE and BookS radix are historical values and should be revisited. */ - if (mmu_linear_psize == MMU_PAGE_4K) + if (IS_ENABLED(CONFIG_PPC_BOOK3E)) { + atom_size = SZ_1M; + } else if (radix_enabled()) { atom_size = PAGE_SIZE; - else - atom_size = 1 << 20; + } else if (IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) { + /* + * Linear mapping is one of 4K, 1M and 16M. For 4K, no need + * to group units. For larger mappings, use 1M atom which + * should be large enough to contain a number of units. + */ + if (mmu_linear_psize == MMU_PAGE_4K) + atom_size = PAGE_SIZE; + else + atom_size = SZ_1M; + } if (pcpu_chosen_fc != PCPU_FC_PAGE) { rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance, - pcpu_alloc_bootmem, pcpu_free_bootmem); + pcpu_cpu_to_node); if (rc) pr_warn("PERCPU: %s allocator failed (%d), " "falling back to page size\n", @@ -899,8 +824,7 @@ void __init setup_per_cpu_areas(void) } if (rc < 0) - rc = pcpu_page_first_chunk(0, pcpu_alloc_bootmem, pcpu_free_bootmem, - pcpu_populate_pte); + rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); @@ -912,7 +836,7 @@ void __init setup_per_cpu_areas(void) } #endif -#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE +#ifdef CONFIG_MEMORY_HOTPLUG unsigned long memory_block_size_bytes(void) { if (ppc_md.memory_block_size) diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c index f2da879264..d84c434b2b 100644 --- a/arch/powerpc/kernel/signal_32.c +++ b/arch/powerpc/kernel/signal_32.c @@ -527,16 +527,20 @@ static long restore_user_regs(struct pt_regs *regs, regs_set_return_msr(regs, regs->msr & ~(MSR_FP | MSR_FE0 | MSR_FE1)); #ifdef CONFIG_SPE - /* force the process to reload the spe registers from - current->thread when it next does spe instructions */ + /* + * Force the process to reload the spe registers from + * current->thread when it next does spe instructions. + * Since this is user ABI, we must enforce the sizing. + */ + BUILD_BUG_ON(sizeof(current->thread.spe) != ELF_NEVRREG * sizeof(u32)); regs_set_return_msr(regs, regs->msr & ~MSR_SPE); if (msr & MSR_SPE) { /* restore spe registers from the stack */ - unsafe_copy_from_user(current->thread.evr, &sr->mc_vregs, - ELF_NEVRREG * sizeof(u32), failed); + unsafe_copy_from_user(¤t->thread.spe, &sr->mc_vregs, + sizeof(current->thread.spe), failed); current->thread.used_spe = true; } else if (current->thread.used_spe) - memset(current->thread.evr, 0, ELF_NEVRREG * sizeof(u32)); + memset(¤t->thread.spe, 0, sizeof(current->thread.spe)); /* Always get SPEFSCR back */ unsafe_get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs + ELF_NEVRREG, failed); @@ -1048,7 +1052,7 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, if (new_ctx == NULL) return 0; if (!access_ok(new_ctx, ctx_size) || - fault_in_pages_readable((u8 __user *)new_ctx, ctx_size)) + fault_in_readable((char __user *)new_ctx, ctx_size)) return -EFAULT; /* @@ -1239,7 +1243,7 @@ SYSCALL_DEFINE3(debug_setcontext, struct ucontext __user *, ctx, #endif if (!access_ok(ctx, sizeof(*ctx)) || - fault_in_pages_readable((u8 __user *)ctx, sizeof(*ctx))) + fault_in_readable((char __user *)ctx, sizeof(*ctx))) return -EFAULT; /* diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c index bb9c077ac1..d1e1fc0acb 100644 --- a/arch/powerpc/kernel/signal_64.c +++ b/arch/powerpc/kernel/signal_64.c @@ -688,7 +688,7 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, if (new_ctx == NULL) return 0; if (!access_ok(new_ctx, ctx_size) || - fault_in_pages_readable((u8 __user *)new_ctx, ctx_size)) + fault_in_readable((char __user *)new_ctx, ctx_size)) return -EFAULT; /* diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index fb95f92dcf..b7fd6a72aa 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -936,7 +936,8 @@ static struct thread_groups *__init get_thread_groups(int cpu, return tg; } -static int update_mask_from_threadgroup(cpumask_var_t *mask, struct thread_groups *tg, int cpu, int cpu_group_start) +static int __init update_mask_from_threadgroup(cpumask_var_t *mask, struct thread_groups *tg, + int cpu, int cpu_group_start) { int first_thread = cpu_first_thread_sibling(cpu); int i; @@ -1263,7 +1264,7 @@ static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle) paca_ptrs[cpu]->kstack = (unsigned long)task_stack_page(idle) + THREAD_SIZE - STACK_FRAME_OVERHEAD; #endif - idle->cpu = cpu; + task_thread_info(idle)->cpu = cpu; secondary_current = current_set[cpu] = idle; } @@ -1353,18 +1354,13 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) int cpu_to_core_id(int cpu) { struct device_node *np; - const __be32 *reg; int id = -1; np = of_get_cpu_node(cpu, NULL); if (!np) goto out; - reg = of_get_property(np, "reg", NULL); - if (!reg) - goto out; - - id = be32_to_cpup(reg); + id = of_get_cpu_hwid(np, 0); out: of_node_put(np); return id; @@ -1687,7 +1683,7 @@ int setup_profiling_timer(unsigned int multiplier) } #endif -static void fixup_topology(void) +static void __init fixup_topology(void) { int i; diff --git a/arch/powerpc/kernel/stacktrace.c b/arch/powerpc/kernel/stacktrace.c index 9e4a4a7af3..a2443d6172 100644 --- a/arch/powerpc/kernel/stacktrace.c +++ b/arch/powerpc/kernel/stacktrace.c @@ -155,7 +155,7 @@ int __no_sanitize_address arch_stack_walk_reliable(stack_trace_consume_fn consum * Mark stacktraces with kretprobed functions on them * as unreliable. */ - if (ip == (unsigned long)kretprobe_trampoline) + if (ip == (unsigned long)__kretprobe_trampoline) return -EINVAL; #endif diff --git a/arch/powerpc/kernel/swsusp_32.S b/arch/powerpc/kernel/swsusp_32.S index f73f4d72fe..e0cbd63007 100644 --- a/arch/powerpc/kernel/swsusp_32.S +++ b/arch/powerpc/kernel/swsusp_32.S @@ -181,7 +181,7 @@ _GLOBAL(swsusp_arch_resume) #ifdef CONFIG_ALTIVEC /* Stop pending alitvec streams and memory accesses */ BEGIN_FTR_SECTION - DSSALL + PPC_DSSALL END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) #endif sync diff --git a/arch/powerpc/kernel/swsusp_64.c b/arch/powerpc/kernel/swsusp_64.c index aeea97ad85..16ee3baaf0 100644 --- a/arch/powerpc/kernel/swsusp_64.c +++ b/arch/powerpc/kernel/swsusp_64.c @@ -17,8 +17,3 @@ void do_after_copyback(void) touch_softlockup_watchdog(); mb(); } - -void _iommu_save(void) -{ - iommu_save(); -} diff --git a/arch/powerpc/kernel/swsusp_asm64.S b/arch/powerpc/kernel/swsusp_asm64.S index 6d3189830d..9f1903c7f5 100644 --- a/arch/powerpc/kernel/swsusp_asm64.S +++ b/arch/powerpc/kernel/swsusp_asm64.S @@ -128,7 +128,6 @@ END_FW_FTR_SECTION_IFCLR(FW_FEATURE_LPAR) * stack pointer on the stack like a real stackframe */ addi r1,r1,-128 - bl _iommu_save bl swsusp_save /* restore LR */ @@ -142,7 +141,7 @@ END_FW_FTR_SECTION_IFCLR(FW_FEATURE_LPAR) _GLOBAL(swsusp_arch_resume) /* Stop pending alitvec streams and memory accesses */ BEGIN_FTR_SECTION - DSSALL + PPC_DSSALL END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) sync diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index 7bef917cc8..2600b42372 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -528,3 +528,5 @@ 446 common landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 nospu set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index defecb3b1b..d45a415d53 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -214,7 +214,7 @@ static ssize_t __used store_dscr_default(struct device *dev, static DEVICE_ATTR(dscr_default, 0600, show_dscr_default, store_dscr_default); -static void sysfs_create_dscr_default(void) +static void __init sysfs_create_dscr_default(void) { if (cpu_has_feature(CPU_FTR_DSCR)) { int cpu; @@ -744,12 +744,12 @@ static ssize_t show_svm(struct device *dev, struct device_attribute *attr, char } static DEVICE_ATTR(svm, 0444, show_svm, NULL); -static void create_svm_file(void) +static void __init create_svm_file(void) { device_create_file(cpu_subsys.dev_root, &dev_attr_svm); } #else -static void create_svm_file(void) +static void __init create_svm_file(void) { } #endif /* CONFIG_PPC_SVM */ @@ -928,7 +928,8 @@ static int unregister_cpu_online(unsigned int cpu) struct device_attribute *attrs, *pmc_attrs; int i, nattrs; - BUG_ON(!c->hotpluggable); + if (WARN_RATELIMIT(!c->hotpluggable, "cpu %d can't be offlined\n", cpu)) + return -EBUSY; #ifdef CONFIG_PPC64 if (cpu_has_feature(CPU_FTR_SMT)) @@ -1109,7 +1110,7 @@ EXPORT_SYMBOL_GPL(cpu_remove_dev_attr_group); /* NUMA stuff */ #ifdef CONFIG_NUMA -static void register_nodes(void) +static void __init register_nodes(void) { int i; @@ -1133,7 +1134,7 @@ void sysfs_remove_device_from_node(struct device *dev, int nid) EXPORT_SYMBOL_GPL(sysfs_remove_device_from_node); #else -static void register_nodes(void) +static void __init register_nodes(void) { return; } diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 934d8ae66c..cd0b8b71ec 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -88,6 +88,7 @@ static struct clocksource clocksource_timebase = { #define DECREMENTER_DEFAULT_MAX 0x7FFFFFFF u64 decrementer_max = DECREMENTER_DEFAULT_MAX; +EXPORT_SYMBOL_GPL(decrementer_max); /* for KVM HDEC */ static int decrementer_set_next_event(unsigned long evt, struct clock_event_device *dev); @@ -107,6 +108,7 @@ struct clock_event_device decrementer_clockevent = { EXPORT_SYMBOL(decrementer_clockevent); DEFINE_PER_CPU(u64, decrementers_next_tb); +EXPORT_SYMBOL_GPL(decrementers_next_tb); static DEFINE_PER_CPU(struct clock_event_device, decrementers); #define XSEC_PER_SEC (1024*1024) @@ -496,6 +498,16 @@ EXPORT_SYMBOL(profile_pc); * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable... */ #ifdef CONFIG_PPC64 +static inline unsigned long test_irq_work_pending(void) +{ + unsigned long x; + + asm volatile("lbz %0,%1(13)" + : "=r" (x) + : "i" (offsetof(struct paca_struct, irq_work_pending))); + return x; +} + static inline void set_irq_work_pending_flag(void) { asm volatile("stb %0,%1(13)" : : @@ -539,13 +551,44 @@ void arch_irq_work_raise(void) preempt_enable(); } +static void set_dec_or_work(u64 val) +{ + set_dec(val); + /* We may have raced with new irq work */ + if (unlikely(test_irq_work_pending())) + set_dec(1); +} + #else /* CONFIG_IRQ_WORK */ #define test_irq_work_pending() 0 #define clear_irq_work_pending() +static void set_dec_or_work(u64 val) +{ + set_dec(val); +} #endif /* CONFIG_IRQ_WORK */ +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +void timer_rearm_host_dec(u64 now) +{ + u64 *next_tb = this_cpu_ptr(&decrementers_next_tb); + + WARN_ON_ONCE(!arch_irqs_disabled()); + WARN_ON_ONCE(mfmsr() & MSR_EE); + + if (now >= *next_tb) { + local_paca->irq_happened |= PACA_IRQ_DEC; + } else { + now = *next_tb - now; + if (now <= decrementer_max) + set_dec_or_work(now); + } +} +EXPORT_SYMBOL_GPL(timer_rearm_host_dec); +#endif + /* * timer_interrupt - gets called when the decrementer overflows, * with interrupts disabled. @@ -566,22 +609,23 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt) return; } - /* Ensure a positive value is written to the decrementer, or else - * some CPUs will continue to take decrementer exceptions. When the - * PPC_WATCHDOG (decrementer based) is configured, keep this at most - * 31 bits, which is about 4 seconds on most systems, which gives - * the watchdog a chance of catching timer interrupt hard lockups. - */ - if (IS_ENABLED(CONFIG_PPC_WATCHDOG)) - set_dec(0x7fffffff); - else - set_dec(decrementer_max); - - /* Conditionally hard-enable interrupts now that the DEC has been - * bumped to its maximum value - */ - may_hard_irq_enable(); + /* Conditionally hard-enable interrupts. */ + if (should_hard_irq_enable()) { + /* + * Ensure a positive value is written to the decrementer, or + * else some CPUs will continue to take decrementer exceptions. + * When the PPC_WATCHDOG (decrementer based) is configured, + * keep this at most 31 bits, which is about 4 seconds on most + * systems, which gives the watchdog a chance of catching timer + * interrupt hard lockups. + */ + if (IS_ENABLED(CONFIG_PPC_WATCHDOG)) + set_dec(0x7fffffff); + else + set_dec(decrementer_max); + do_hard_irq_enable(); + } #if defined(CONFIG_PPC32) && defined(CONFIG_PPC_PMAC) if (atomic_read(&ppc_n_lost_interrupts) != 0) @@ -605,11 +649,9 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt) __this_cpu_inc(irq_stat.timer_irqs_event); } else { now = *next_tb - now; - if (now <= decrementer_max) - set_dec(now); - /* We may have raced with new irq work */ - if (test_irq_work_pending()) - set_dec(1); + if (now > decrementer_max) + now = decrementer_max; + set_dec_or_work(now); __this_cpu_inc(irq_stat.timer_irqs_others); } @@ -631,8 +673,12 @@ void timer_broadcast_interrupt(void) #endif #ifdef CONFIG_SUSPEND -static void generic_suspend_disable_irqs(void) +/* Overrides the weak version in kernel/power/main.c */ +void arch_suspend_disable_irqs(void) { + if (ppc_md.suspend_disable_irqs) + ppc_md.suspend_disable_irqs(); + /* Disable the decrementer, so that it doesn't interfere * with suspending. */ @@ -642,23 +688,11 @@ static void generic_suspend_disable_irqs(void) set_dec(decrementer_max); } -static void generic_suspend_enable_irqs(void) -{ - local_irq_enable(); -} - -/* Overrides the weak version in kernel/power/main.c */ -void arch_suspend_disable_irqs(void) -{ - if (ppc_md.suspend_disable_irqs) - ppc_md.suspend_disable_irqs(); - generic_suspend_disable_irqs(); -} - /* Overrides the weak version in kernel/power/main.c */ void arch_suspend_enable_irqs(void) { - generic_suspend_enable_irqs(); + local_irq_enable(); + if (ppc_md.suspend_enable_irqs) ppc_md.suspend_enable_irqs(); } @@ -738,7 +772,7 @@ static int __init get_freq(char *name, int cells, unsigned long *val) static void start_cpu_decrementer(void) { -#if defined(CONFIG_BOOKE) || defined(CONFIG_40x) +#ifdef CONFIG_BOOKE_OR_40x unsigned int tcr; /* Clear any pending timer interrupts */ @@ -851,11 +885,7 @@ static int decrementer_set_next_event(unsigned long evt, struct clock_event_device *dev) { __this_cpu_write(decrementers_next_tb, get_tb() + evt); - set_dec(evt); - - /* We may have raced with new irq work */ - if (test_irq_work_pending()) - set_dec(1); + set_dec_or_work(evt); return 0; } diff --git a/arch/powerpc/kernel/tm.S b/arch/powerpc/kernel/tm.S index 2b91f233b0..3beecc3294 100644 --- a/arch/powerpc/kernel/tm.S +++ b/arch/powerpc/kernel/tm.S @@ -226,11 +226,8 @@ _GLOBAL(tm_reclaim) /* Sync the userland GPRs 2-12, 14-31 to thread->regs: */ SAVE_GPR(0, r7) /* user r0 */ - SAVE_GPR(2, r7) /* user r2 */ - SAVE_4GPRS(3, r7) /* user r3-r6 */ - SAVE_GPR(8, r7) /* user r8 */ - SAVE_GPR(9, r7) /* user r9 */ - SAVE_GPR(10, r7) /* user r10 */ + SAVE_GPRS(2, 6, r7) /* user r2-r6 */ + SAVE_GPRS(8, 10, r7) /* user r8-r10 */ ld r3, GPR1(r1) /* user r1 */ ld r4, GPR7(r1) /* user r7 */ ld r5, GPR11(r1) /* user r11 */ @@ -445,12 +442,8 @@ restore_gprs: ld r6, THREAD_TM_PPR(r3) REST_GPR(0, r7) /* GPR0 */ - REST_2GPRS(2, r7) /* GPR2-3 */ - REST_GPR(4, r7) /* GPR4 */ - REST_4GPRS(8, r7) /* GPR8-11 */ - REST_2GPRS(12, r7) /* GPR12-13 */ - - REST_NVGPRS(r7) /* GPR14-31 */ + REST_GPRS(2, 4, r7) /* GPR2-4 */ + REST_GPRS(8, 31, r7) /* GPR8-31 */ /* Load up PPR and DSCR here so we don't run with user values for long */ mtspr SPRN_DSCR, r5 diff --git a/arch/powerpc/kernel/trace/ftrace.c b/arch/powerpc/kernel/trace/ftrace.c index d89c5df4f2..80b6285769 100644 --- a/arch/powerpc/kernel/trace/ftrace.c +++ b/arch/powerpc/kernel/trace/ftrace.c @@ -41,10 +41,10 @@ #define NUM_FTRACE_TRAMPS 8 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS]; -static struct ppc_inst +static ppc_inst_t ftrace_call_replace(unsigned long ip, unsigned long addr, int link) { - struct ppc_inst op; + ppc_inst_t op; addr = ppc_function_entry((void *)addr); @@ -55,9 +55,9 @@ ftrace_call_replace(unsigned long ip, unsigned long addr, int link) } static int -ftrace_modify_code(unsigned long ip, struct ppc_inst old, struct ppc_inst new) +ftrace_modify_code(unsigned long ip, ppc_inst_t old, ppc_inst_t new) { - struct ppc_inst replaced; + ppc_inst_t replaced; /* * Note: @@ -90,24 +90,24 @@ ftrace_modify_code(unsigned long ip, struct ppc_inst old, struct ppc_inst new) */ static int test_24bit_addr(unsigned long ip, unsigned long addr) { - struct ppc_inst op; + ppc_inst_t op; addr = ppc_function_entry((void *)addr); /* use the create_branch to verify that this offset can be branched */ return create_branch(&op, (u32 *)ip, addr, 0) == 0; } -static int is_bl_op(struct ppc_inst op) +static int is_bl_op(ppc_inst_t op) { return (ppc_inst_val(op) & 0xfc000003) == 0x48000001; } -static int is_b_op(struct ppc_inst op) +static int is_b_op(ppc_inst_t op) { return (ppc_inst_val(op) & 0xfc000003) == 0x48000000; } -static unsigned long find_bl_target(unsigned long ip, struct ppc_inst op) +static unsigned long find_bl_target(unsigned long ip, ppc_inst_t op) { int offset; @@ -127,7 +127,7 @@ __ftrace_make_nop(struct module *mod, { unsigned long entry, ptr, tramp; unsigned long ip = rec->ip; - struct ppc_inst op, pop; + ppc_inst_t op, pop; /* read where this goes */ if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { @@ -221,10 +221,9 @@ static int __ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { - struct ppc_inst op; - unsigned int jmp[4]; + ppc_inst_t op; unsigned long ip = rec->ip; - unsigned long tramp; + unsigned long tramp, ptr; if (copy_from_kernel_nofault(&op, (void *)ip, MCOUNT_INSN_SIZE)) return -EFAULT; @@ -238,41 +237,13 @@ __ftrace_make_nop(struct module *mod, /* lets find where the pointer goes */ tramp = find_bl_target(ip, op); - /* - * On PPC32 the trampoline looks like: - * 0x3d, 0x80, 0x00, 0x00 lis r12,sym@ha - * 0x39, 0x8c, 0x00, 0x00 addi r12,r12,sym@l - * 0x7d, 0x89, 0x03, 0xa6 mtctr r12 - * 0x4e, 0x80, 0x04, 0x20 bctr - */ - - pr_devel("ip:%lx jumps to %lx", ip, tramp); - /* Find where the trampoline jumps to */ - if (copy_from_kernel_nofault(jmp, (void *)tramp, sizeof(jmp))) { - pr_err("Failed to read %lx\n", tramp); + if (module_trampoline_target(mod, tramp, &ptr)) { + pr_err("Failed to get trampoline target\n"); return -EFAULT; } - pr_devel(" %08x %08x ", jmp[0], jmp[1]); - - /* verify that this is what we expect it to be */ - if (((jmp[0] & 0xffff0000) != 0x3d800000) || - ((jmp[1] & 0xffff0000) != 0x398c0000) || - (jmp[2] != 0x7d8903a6) || - (jmp[3] != 0x4e800420)) { - pr_err("Not a trampoline\n"); - return -EINVAL; - } - - tramp = (jmp[1] & 0xffff) | - ((jmp[0] & 0xffff) << 16); - if (tramp & 0x8000) - tramp -= 0x10000; - - pr_devel(" %lx ", tramp); - - if (tramp != addr) { + if (ptr != addr) { pr_err("Trampoline location %08lx does not match addr\n", tramp); return -EINVAL; @@ -291,7 +262,7 @@ __ftrace_make_nop(struct module *mod, static unsigned long find_ftrace_tramp(unsigned long ip) { int i; - struct ppc_inst instr; + ppc_inst_t instr; /* * We have the compiler generated long_branch tramps at the end @@ -329,9 +300,9 @@ static int add_ftrace_tramp(unsigned long tramp) static int setup_mcount_compiler_tramp(unsigned long tramp) { int i; - struct ppc_inst op; + ppc_inst_t op; unsigned long ptr; - struct ppc_inst instr; + ppc_inst_t instr; static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS]; /* Is this a known long jump tramp? */ @@ -396,7 +367,7 @@ static int setup_mcount_compiler_tramp(unsigned long tramp) static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) { unsigned long tramp, ip = rec->ip; - struct ppc_inst op; + ppc_inst_t op; /* Read where this goes */ if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { @@ -436,7 +407,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { unsigned long ip = rec->ip; - struct ppc_inst old, new; + ppc_inst_t old, new; /* * If the calling address is more that 24 bits away, @@ -489,7 +460,7 @@ int ftrace_make_nop(struct module *mod, */ #ifndef CONFIG_MPROFILE_KERNEL static int -expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1) +expected_nop_sequence(void *ip, ppc_inst_t op0, ppc_inst_t op1) { /* * We expect to see: @@ -507,7 +478,7 @@ expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1) } #else static int -expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1) +expected_nop_sequence(void *ip, ppc_inst_t op0, ppc_inst_t op1) { /* look for patched "NOP" on ppc64 with -mprofile-kernel */ if (!ppc_inst_equal(op0, ppc_inst(PPC_RAW_NOP()))) @@ -519,8 +490,8 @@ expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1) static int __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { - struct ppc_inst op[2]; - struct ppc_inst instr; + ppc_inst_t op[2]; + ppc_inst_t instr; void *ip = (void *)rec->ip; unsigned long entry, ptr, tramp; struct module *mod = rec->arch.mod; @@ -588,8 +559,10 @@ static int __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { int err; - struct ppc_inst op; + ppc_inst_t op; u32 *ip = (u32 *)rec->ip; + struct module *mod = rec->arch.mod; + unsigned long tramp; /* read where this goes */ if (copy_inst_from_kernel_nofault(&op, ip)) @@ -602,13 +575,23 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) } /* If we never set up a trampoline to ftrace_caller, then bail */ - if (!rec->arch.mod->arch.tramp) { +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + if (!mod->arch.tramp || !mod->arch.tramp_regs) { +#else + if (!mod->arch.tramp) { +#endif pr_err("No ftrace trampoline\n"); return -EINVAL; } +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + if (rec->flags & FTRACE_FL_REGS) + tramp = mod->arch.tramp_regs; + else +#endif + tramp = mod->arch.tramp; /* create the branch to the trampoline */ - err = create_branch(&op, ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK); + err = create_branch(&op, ip, tramp, BRANCH_SET_LINK); if (err) { pr_err("REL24 out of range!\n"); return -EINVAL; @@ -626,7 +609,7 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) { - struct ppc_inst op; + ppc_inst_t op; void *ip = (void *)rec->ip; unsigned long tramp, entry, ptr; @@ -674,7 +657,7 @@ static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { unsigned long ip = rec->ip; - struct ppc_inst old, new; + ppc_inst_t old, new; /* * If the calling address is more that 24 bits away, @@ -713,7 +696,7 @@ static int __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { - struct ppc_inst op; + ppc_inst_t op; unsigned long ip = rec->ip; unsigned long entry, ptr, tramp; struct module *mod = rec->arch.mod; @@ -807,7 +790,7 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { unsigned long ip = rec->ip; - struct ppc_inst old, new; + ppc_inst_t old, new; /* * If the calling address is more that 24 bits away, @@ -847,7 +830,7 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, int ftrace_update_ftrace_func(ftrace_func_t func) { unsigned long ip = (unsigned long)(&ftrace_call); - struct ppc_inst old, new; + ppc_inst_t old, new; int ret; old = ppc_inst_read((u32 *)&ftrace_call); @@ -932,7 +915,7 @@ int ftrace_enable_ftrace_graph_caller(void) unsigned long ip = (unsigned long)(&ftrace_graph_call); unsigned long addr = (unsigned long)(&ftrace_graph_caller); unsigned long stub = (unsigned long)(&ftrace_graph_stub); - struct ppc_inst old, new; + ppc_inst_t old, new; old = ftrace_call_replace(ip, stub, 0); new = ftrace_call_replace(ip, addr, 0); @@ -945,7 +928,7 @@ int ftrace_disable_ftrace_graph_caller(void) unsigned long ip = (unsigned long)(&ftrace_graph_call); unsigned long addr = (unsigned long)(&ftrace_graph_caller); unsigned long stub = (unsigned long)(&ftrace_graph_stub); - struct ppc_inst old, new; + ppc_inst_t old, new; old = ftrace_call_replace(ip, addr, 0); new = ftrace_call_replace(ip, stub, 0); diff --git a/arch/powerpc/kernel/trace/ftrace_32.S b/arch/powerpc/kernel/trace/ftrace_32.S index e023ae59c4..0a02c0cb12 100644 --- a/arch/powerpc/kernel/trace/ftrace_32.S +++ b/arch/powerpc/kernel/trace/ftrace_32.S @@ -9,54 +9,134 @@ #include #include #include +#include _GLOBAL(mcount) _GLOBAL(_mcount) /* * It is required that _mcount on PPC32 must preserve the - * link register. But we have r0 to play with. We use r0 + * link register. But we have r12 to play with. We use r12 * to push the return address back to the caller of mcount * into the ctr register, restore the link register and * then jump back using the ctr register. */ - mflr r0 - mtctr r0 - lwz r0, 4(r1) + mflr r12 + mtctr r12 mtlr r0 bctr +EXPORT_SYMBOL(_mcount) _GLOBAL(ftrace_caller) MCOUNT_SAVE_FRAME /* r3 ends up with link register */ subi r3, r3, MCOUNT_INSN_SIZE + lis r5,function_trace_op@ha + lwz r5,function_trace_op@l(r5) + li r6, 0 .globl ftrace_call ftrace_call: bl ftrace_stub nop + MCOUNT_RESTORE_FRAME +ftrace_caller_common: #ifdef CONFIG_FUNCTION_GRAPH_TRACER .globl ftrace_graph_call ftrace_graph_call: b ftrace_graph_stub _GLOBAL(ftrace_graph_stub) #endif - MCOUNT_RESTORE_FRAME /* old link register ends up in ctr reg */ bctr -EXPORT_SYMBOL(_mcount) _GLOBAL(ftrace_stub) blr +_GLOBAL(ftrace_regs_caller) + /* Save the original return address in A's stack frame */ + stw r0,LRSAVE(r1) + + /* Create our stack frame + pt_regs */ + stwu r1,-INT_FRAME_SIZE(r1) + + /* Save all gprs to pt_regs */ + stw r0, GPR0(r1) + stmw r2, GPR2(r1) + + /* Save previous stack pointer (r1) */ + addi r8, r1, INT_FRAME_SIZE + stw r8, GPR1(r1) + + /* Load special regs for save below */ + mfmsr r8 + mfctr r9 + mfxer r10 + mfcr r11 + + /* Get the _mcount() call site out of LR */ + mflr r7 + /* Save it as pt_regs->nip */ + stw r7, _NIP(r1) + /* Save the read LR in pt_regs->link */ + stw r0, _LINK(r1) + + lis r3,function_trace_op@ha + lwz r5,function_trace_op@l(r3) + + /* Calculate ip from nip-4 into r3 for call below */ + subi r3, r7, MCOUNT_INSN_SIZE + + /* Put the original return address in r4 as parent_ip */ + mr r4, r0 + + /* Save special regs */ + stw r8, _MSR(r1) + stw r9, _CTR(r1) + stw r10, _XER(r1) + stw r11, _CCR(r1) + + /* Load &pt_regs in r6 for call below */ + addi r6, r1, STACK_FRAME_OVERHEAD + + /* ftrace_call(r3, r4, r5, r6) */ +.globl ftrace_regs_call +ftrace_regs_call: + bl ftrace_stub + nop + + /* Load ctr with the possibly modified NIP */ + lwz r3, _NIP(r1) + mtctr r3 + + /* Restore gprs */ + lmw r2, GPR2(r1) + + /* Restore possibly modified LR */ + lwz r0, _LINK(r1) + mtlr r0 + + /* Pop our stack frame */ + addi r1, r1, INT_FRAME_SIZE + + b ftrace_caller_common + #ifdef CONFIG_FUNCTION_GRAPH_TRACER _GLOBAL(ftrace_graph_caller) - addi r5, r1, 48 - /* load r4 with local address */ - lwz r4, 44(r1) - subi r4, r4, MCOUNT_INSN_SIZE + stwu r1,-48(r1) + stw r3, 12(r1) + stw r4, 16(r1) + stw r5, 20(r1) + stw r6, 24(r1) + stw r7, 28(r1) + stw r8, 32(r1) + stw r9, 36(r1) + stw r10,40(r1) - /* Grab the LR out of the caller stack frame */ - lwz r3,52(r1) + addi r5, r1, 48 + mfctr r4 /* ftrace_caller has moved local addr here */ + stw r4, 44(r1) + mflr r3 /* ftrace_caller has restored LR from stack */ + subi r4, r4, MCOUNT_INSN_SIZE bl prepare_ftrace_return nop @@ -66,9 +146,21 @@ _GLOBAL(ftrace_graph_caller) * Change the LR in the callers stack frame to this. */ stw r3,52(r1) + mtlr r3 + lwz r0,44(r1) + mtctr r0 + + lwz r3, 12(r1) + lwz r4, 16(r1) + lwz r5, 20(r1) + lwz r6, 24(r1) + lwz r7, 28(r1) + lwz r8, 32(r1) + lwz r9, 36(r1) + lwz r10,40(r1) + + addi r1, r1, 48 - MCOUNT_RESTORE_FRAME - /* old link register ends up in ctr reg */ bctr _GLOBAL(return_to_handler) diff --git a/arch/powerpc/kernel/trace/ftrace_64_mprofile.S b/arch/powerpc/kernel/trace/ftrace_64_mprofile.S index f9fd5f743e..d636fc755f 100644 --- a/arch/powerpc/kernel/trace/ftrace_64_mprofile.S +++ b/arch/powerpc/kernel/trace/ftrace_64_mprofile.S @@ -41,15 +41,14 @@ _GLOBAL(ftrace_regs_caller) /* Save all gprs to pt_regs */ SAVE_GPR(0, r1) - SAVE_10GPRS(2, r1) + SAVE_GPRS(2, 11, r1) /* Ok to continue? */ lbz r3, PACA_FTRACE_ENABLED(r13) cmpdi r3, 0 beq ftrace_no_trace - SAVE_10GPRS(12, r1) - SAVE_10GPRS(22, r1) + SAVE_GPRS(12, 31, r1) /* Save previous stack pointer (r1) */ addi r8, r1, SWITCH_FRAME_SIZE @@ -108,10 +107,8 @@ ftrace_regs_call: #endif /* Restore gprs */ - REST_GPR(0,r1) - REST_10GPRS(2,r1) - REST_10GPRS(12,r1) - REST_10GPRS(22,r1) + REST_GPR(0, r1) + REST_GPRS(2, 31, r1) /* Restore possibly modified LR */ ld r0, _LINK(r1) @@ -157,7 +154,7 @@ _GLOBAL(ftrace_caller) stdu r1, -SWITCH_FRAME_SIZE(r1) /* Save all gprs to pt_regs */ - SAVE_8GPRS(3, r1) + SAVE_GPRS(3, 10, r1) lbz r3, PACA_FTRACE_ENABLED(r13) cmpdi r3, 0 @@ -194,7 +191,7 @@ ftrace_call: mtctr r3 /* Restore gprs */ - REST_8GPRS(3,r1) + REST_GPRS(3, 10, r1) /* Restore callee's TOC */ ld r2, 24(r1) diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 11741703d2..a08bb7cefd 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -245,7 +245,7 @@ static void oops_end(unsigned long flags, struct pt_regs *regs, if (panic_on_oops) panic("Fatal exception"); - do_exit(signr); + make_task_dead(signr); } NOKPROBE_SYMBOL(oops_end); @@ -792,9 +792,9 @@ int machine_check_generic(struct pt_regs *regs) void die_mce(const char *str, struct pt_regs *regs, long err) { /* - * The machine check wants to kill the interrupted context, but - * do_exit() checks for in_interrupt() and panics in that case, so - * exit the irq/nmi before calling die. + * The machine check wants to kill the interrupted context, + * but make_task_dead() checks for in_interrupt() and panics + * in that case, so exit the irq/nmi before calling die. */ if (in_nmi()) nmi_exit(); diff --git a/arch/powerpc/kernel/udbg_16550.c b/arch/powerpc/kernel/udbg_16550.c index 8513aa4961..d3942de254 100644 --- a/arch/powerpc/kernel/udbg_16550.c +++ b/arch/powerpc/kernel/udbg_16550.c @@ -84,7 +84,7 @@ static int udbg_uart_getc(void) return udbg_uart_in(UART_RBR); } -static void udbg_use_uart(void) +static void __init udbg_use_uart(void) { udbg_putc = udbg_uart_putc; udbg_flush = udbg_uart_flush; @@ -92,7 +92,7 @@ static void udbg_use_uart(void) udbg_getc_poll = udbg_uart_getc_poll; } -void udbg_uart_setup(unsigned int speed, unsigned int clock) +void __init udbg_uart_setup(unsigned int speed, unsigned int clock) { unsigned int dll, base_bauds; @@ -121,7 +121,7 @@ void udbg_uart_setup(unsigned int speed, unsigned int clock) udbg_uart_out(UART_FCR, 0x7); } -unsigned int udbg_probe_uart_speed(unsigned int clock) +unsigned int __init udbg_probe_uart_speed(unsigned int clock) { unsigned int dll, dlm, divisor, prescaler, speed; u8 old_lcr; @@ -172,7 +172,7 @@ static void udbg_uart_out_pio(unsigned int reg, u8 data) outb(data, udbg_uart.pio_base + (reg * udbg_uart_stride)); } -void udbg_uart_init_pio(unsigned long port, unsigned int stride) +void __init udbg_uart_init_pio(unsigned long port, unsigned int stride) { if (!port) return; @@ -194,7 +194,7 @@ static void udbg_uart_out_mmio(unsigned int reg, u8 data) } -void udbg_uart_init_mmio(void __iomem *addr, unsigned int stride) +void __init udbg_uart_init_mmio(void __iomem *addr, unsigned int stride) { if (!addr) return; diff --git a/arch/powerpc/kernel/vecemu.c b/arch/powerpc/kernel/vecemu.c index ae63256944..fd9432875e 100644 --- a/arch/powerpc/kernel/vecemu.c +++ b/arch/powerpc/kernel/vecemu.c @@ -261,7 +261,7 @@ static unsigned int rfin(unsigned int x) int emulate_altivec(struct pt_regs *regs) { - struct ppc_inst instr; + ppc_inst_t instr; unsigned int i, word; unsigned int va, vb, vc, vd; vector128 *vrs; diff --git a/arch/powerpc/kernel/vector.S b/arch/powerpc/kernel/vector.S index ba03eedfdc..5cc24d8cce 100644 --- a/arch/powerpc/kernel/vector.S +++ b/arch/powerpc/kernel/vector.S @@ -47,6 +47,10 @@ EXPORT_SYMBOL(store_vr_state) */ _GLOBAL(load_up_altivec) mfmsr r5 /* grab the current MSR */ +#ifdef CONFIG_PPC_BOOK3S_64 + /* interrupt doesn't set MSR[RI] and HPT can fault on current access */ + ori r5,r5,MSR_RI +#endif oris r5,r5,MSR_VEC@h MTMSRD(r5) /* enable use of AltiVec now */ isync @@ -126,6 +130,12 @@ _GLOBAL(load_up_vsx) andis. r5,r12,MSR_VEC@h beql+ load_up_altivec /* skip if already loaded */ +#ifdef CONFIG_PPC_BOOK3S_64 + /* interrupt doesn't set MSR[RI] and HPT can fault on current access */ + li r5,MSR_RI + mtmsrd r5,1 +#endif + ld r4,PACACURRENT(r13) addi r4,r4,THREAD /* Get THREAD */ li r6,1 diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index 40bdefe9ca..2bcca81813 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -143,6 +143,12 @@ SECTIONS SOFT_MASK_TABLE(8) RESTART_TABLE(8) + .opd : AT(ADDR(.opd) - LOAD_OFFSET) { + __start_opd = .; + KEEP(*(.opd)) + __end_opd = .; + } + . = ALIGN(8); __stf_entry_barrier_fixup : AT(ADDR(__stf_entry_barrier_fixup) - LOAD_OFFSET) { __start___stf_entry_barrier_fixup = .; @@ -316,10 +322,6 @@ SECTIONS #ifdef CONFIG_PPC32 .data : AT(ADDR(.data) - LOAD_OFFSET) { DATA_DATA -#ifdef CONFIG_UBSAN - *(.data..Lubsan_data*) - *(.data..Lubsan_type*) -#endif *(.data.rel*) *(SDATA_MAIN) *(.sdata2) @@ -330,30 +332,18 @@ SECTIONS #else .data : AT(ADDR(.data) - LOAD_OFFSET) { DATA_DATA -#ifdef CONFIG_UBSAN - *(.data..Lubsan_data*) - *(.data..Lubsan_type*) -#endif *(.data.rel*) *(.toc1) *(.branch_lt) } - .opd : AT(ADDR(.opd) - LOAD_OFFSET) { - __start_opd = .; - KEEP(*(.opd)) - __end_opd = .; - } - - . = ALIGN(256); - .got : AT(ADDR(.got) - LOAD_OFFSET) { - __toc_start = .; + .got : AT(ADDR(.got) - LOAD_OFFSET) ALIGN(256) { + *(.got) #ifndef CONFIG_RELOCATABLE __prom_init_toc_start = .; - arch/powerpc/kernel/prom_init.o*(.toc .got) + arch/powerpc/kernel/prom_init.o*(.toc) __prom_init_toc_end = .; #endif - *(.got) *(.toc) } #endif diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c index ad94a2c6b7..bfc27496fe 100644 --- a/arch/powerpc/kernel/watchdog.c +++ b/arch/powerpc/kernel/watchdog.c @@ -85,10 +85,37 @@ static DEFINE_PER_CPU(u64, wd_timer_tb); /* SMP checker bits */ static unsigned long __wd_smp_lock; +static unsigned long __wd_reporting; +static unsigned long __wd_nmi_output; static cpumask_t wd_smp_cpus_pending; static cpumask_t wd_smp_cpus_stuck; static u64 wd_smp_last_reset_tb; +/* + * Try to take the exclusive watchdog action / NMI IPI / printing lock. + * wd_smp_lock must be held. If this fails, we should return and wait + * for the watchdog to kick in again (or another CPU to trigger it). + * + * Importantly, if hardlockup_panic is set, wd_try_report failure should + * not delay the panic, because whichever other CPU is reporting will + * call panic. + */ +static bool wd_try_report(void) +{ + if (__wd_reporting) + return false; + __wd_reporting = 1; + return true; +} + +/* End printing after successful wd_try_report. wd_smp_lock not required. */ +static void wd_end_reporting(void) +{ + smp_mb(); /* End printing "critical section" */ + WARN_ON_ONCE(__wd_reporting == 0); + WRITE_ONCE(__wd_reporting, 0); +} + static inline void wd_smp_lock(unsigned long *flags) { /* @@ -128,101 +155,123 @@ static void wd_lockup_ipi(struct pt_regs *regs) else dump_stack(); + /* + * __wd_nmi_output must be set after we printk from NMI context. + * + * printk from NMI context defers printing to the console to irq_work. + * If that NMI was taken in some code that is hard-locked, then irqs + * are disabled so irq_work will never fire. That can result in the + * hard lockup messages being delayed (indefinitely, until something + * else kicks the console drivers). + * + * Setting __wd_nmi_output will cause another CPU to notice and kick + * the console drivers for us. + * + * xchg is not needed here (it could be a smp_mb and store), but xchg + * gives the memory ordering and atomicity required. + */ + xchg(&__wd_nmi_output, 1); + /* Do not panic from here because that can recurse into NMI IPI layer */ } -static void set_cpumask_stuck(const struct cpumask *cpumask, u64 tb) +static bool set_cpu_stuck(int cpu) { - cpumask_or(&wd_smp_cpus_stuck, &wd_smp_cpus_stuck, cpumask); - cpumask_andnot(&wd_smp_cpus_pending, &wd_smp_cpus_pending, cpumask); + cpumask_set_cpu(cpu, &wd_smp_cpus_stuck); + cpumask_clear_cpu(cpu, &wd_smp_cpus_pending); /* * See wd_smp_clear_cpu_pending() */ smp_mb(); if (cpumask_empty(&wd_smp_cpus_pending)) { - wd_smp_last_reset_tb = tb; + wd_smp_last_reset_tb = get_tb(); cpumask_andnot(&wd_smp_cpus_pending, &wd_cpus_enabled, &wd_smp_cpus_stuck); + return true; } -} -static void set_cpu_stuck(int cpu, u64 tb) -{ - set_cpumask_stuck(cpumask_of(cpu), tb); + return false; } -static void watchdog_smp_panic(int cpu, u64 tb) +static void watchdog_smp_panic(int cpu) { + static cpumask_t wd_smp_cpus_ipi; // protected by reporting unsigned long flags; + u64 tb, last_reset; int c; wd_smp_lock(&flags); /* Double check some things under lock */ - if ((s64)(tb - wd_smp_last_reset_tb) < (s64)wd_smp_panic_timeout_tb) + tb = get_tb(); + last_reset = wd_smp_last_reset_tb; + if ((s64)(tb - last_reset) < (s64)wd_smp_panic_timeout_tb) goto out; if (cpumask_test_cpu(cpu, &wd_smp_cpus_pending)) goto out; - if (cpumask_weight(&wd_smp_cpus_pending) == 0) + if (!wd_try_report()) goto out; + for_each_online_cpu(c) { + if (!cpumask_test_cpu(c, &wd_smp_cpus_pending)) + continue; + if (c == cpu) + continue; // should not happen + + __cpumask_set_cpu(c, &wd_smp_cpus_ipi); + if (set_cpu_stuck(c)) + break; + } + if (cpumask_empty(&wd_smp_cpus_ipi)) { + wd_end_reporting(); + goto out; + } + wd_smp_unlock(&flags); pr_emerg("CPU %d detected hard LOCKUP on other CPUs %*pbl\n", - cpu, cpumask_pr_args(&wd_smp_cpus_pending)); + cpu, cpumask_pr_args(&wd_smp_cpus_ipi)); pr_emerg("CPU %d TB:%lld, last SMP heartbeat TB:%lld (%lldms ago)\n", - cpu, tb, wd_smp_last_reset_tb, - tb_to_ns(tb - wd_smp_last_reset_tb) / 1000000); + cpu, tb, last_reset, tb_to_ns(tb - last_reset) / 1000000); if (!sysctl_hardlockup_all_cpu_backtrace) { /* * Try to trigger the stuck CPUs, unless we are going to * get a backtrace on all of them anyway. */ - for_each_cpu(c, &wd_smp_cpus_pending) { - if (c == cpu) - continue; + for_each_cpu(c, &wd_smp_cpus_ipi) { smp_send_nmi_ipi(c, wd_lockup_ipi, 1000000); + __cpumask_clear_cpu(c, &wd_smp_cpus_ipi); } - } - - /* Take the stuck CPUs out of the watch group */ - set_cpumask_stuck(&wd_smp_cpus_pending, tb); - - wd_smp_unlock(&flags); - - if (sysctl_hardlockup_all_cpu_backtrace) + } else { trigger_allbutself_cpu_backtrace(); - - /* - * Force flush any remote buffers that might be stuck in IRQ context - * and therefore could not run their irq_work. - */ - printk_trigger_flush(); + cpumask_clear(&wd_smp_cpus_ipi); + } if (hardlockup_panic) nmi_panic(NULL, "Hard LOCKUP"); + wd_end_reporting(); + return; out: wd_smp_unlock(&flags); } -static void wd_smp_clear_cpu_pending(int cpu, u64 tb) +static void wd_smp_clear_cpu_pending(int cpu) { if (!cpumask_test_cpu(cpu, &wd_smp_cpus_pending)) { if (unlikely(cpumask_test_cpu(cpu, &wd_smp_cpus_stuck))) { struct pt_regs *regs = get_irq_regs(); unsigned long flags; - wd_smp_lock(&flags); - pr_emerg("CPU %d became unstuck TB:%lld\n", - cpu, tb); + cpu, get_tb()); print_irqtrace_events(current); if (regs) show_regs(regs); else dump_stack(); + wd_smp_lock(&flags); cpumask_clear_cpu(cpu, &wd_smp_cpus_stuck); wd_smp_unlock(&flags); } else { @@ -243,6 +292,22 @@ static void wd_smp_clear_cpu_pending(int cpu, u64 tb) return; } + /* + * All other updates to wd_smp_cpus_pending are performed under + * wd_smp_lock. All of them are atomic except the case where the + * mask becomes empty and is reset. This will not happen here because + * cpu was tested to be in the bitmap (above), and a CPU only clears + * its own bit. _Except_ in the case where another CPU has detected a + * hard lockup on our CPU and takes us out of the pending mask. So in + * normal operation there will be no race here, no problem. + * + * In the lockup case, this atomic clear-bit vs a store that refills + * other bits in the accessed word wll not be a problem. The bit clear + * is atomic so it will not cause the store to get lost, and the store + * will never set this bit so it will not overwrite the bit clear. The + * only way for a stuck CPU to return to the pending bitmap is to + * become unstuck itself. + */ cpumask_clear_cpu(cpu, &wd_smp_cpus_pending); /* @@ -265,7 +330,7 @@ static void wd_smp_clear_cpu_pending(int cpu, u64 tb) */ wd_smp_lock(&flags); if (cpumask_empty(&wd_smp_cpus_pending)) { - wd_smp_last_reset_tb = tb; + wd_smp_last_reset_tb = get_tb(); cpumask_andnot(&wd_smp_cpus_pending, &wd_cpus_enabled, &wd_smp_cpus_stuck); @@ -280,10 +345,21 @@ static void watchdog_timer_interrupt(int cpu) per_cpu(wd_timer_tb, cpu) = tb; - wd_smp_clear_cpu_pending(cpu, tb); + wd_smp_clear_cpu_pending(cpu); if ((s64)(tb - wd_smp_last_reset_tb) >= (s64)wd_smp_panic_timeout_tb) - watchdog_smp_panic(cpu, tb); + watchdog_smp_panic(cpu); + + if (__wd_nmi_output && xchg(&__wd_nmi_output, 0)) { + /* + * Something has called printk from NMI context. It might be + * stuck, so this this triggers a flush that will get that + * printk output to the console. + * + * See wd_lockup_ipi. + */ + printk_trigger_flush(); + } } DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt) @@ -302,12 +378,27 @@ DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt) tb = get_tb(); if (tb - per_cpu(wd_timer_tb, cpu) >= wd_panic_timeout_tb) { + /* + * Taking wd_smp_lock here means it is a soft-NMI lock, which + * means we can't take any regular or irqsafe spin locks while + * holding this lock. This is why timers can't printk while + * holding the lock. + */ wd_smp_lock(&flags); if (cpumask_test_cpu(cpu, &wd_smp_cpus_stuck)) { wd_smp_unlock(&flags); return 0; } - set_cpu_stuck(cpu, tb); + if (!wd_try_report()) { + wd_smp_unlock(&flags); + /* Couldn't report, try again in 100ms */ + mtspr(SPRN_DEC, 100 * tb_ticks_per_usec * 1000); + return 0; + } + + set_cpu_stuck(cpu); + + wd_smp_unlock(&flags); pr_emerg("CPU %d self-detected hard LOCKUP @ %pS\n", cpu, (void *)regs->nip); @@ -318,14 +409,21 @@ DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt) print_irqtrace_events(current); show_regs(regs); - wd_smp_unlock(&flags); + xchg(&__wd_nmi_output, 1); // see wd_lockup_ipi if (sysctl_hardlockup_all_cpu_backtrace) trigger_allbutself_cpu_backtrace(); if (hardlockup_panic) nmi_panic(regs, "Hard LOCKUP"); + + wd_end_reporting(); } + /* + * We are okay to change DEC in soft_nmi_interrupt because the masked + * handler has marked a DEC as pending, so the timer interrupt will be + * replayed as soon as local irqs are enabled again. + */ if (wd_panic_timeout_tb < 0x7fffffff) mtspr(SPRN_DEC, wd_panic_timeout_tb); @@ -361,7 +459,7 @@ void arch_touch_nmi_watchdog(void) tb = get_tb(); if (tb - per_cpu(wd_timer_tb, cpu) >= ticks) { per_cpu(wd_timer_tb, cpu) = tb; - wd_smp_clear_cpu_pending(cpu, tb); + wd_smp_clear_cpu_pending(cpu); } } EXPORT_SYMBOL(arch_touch_nmi_watchdog); @@ -419,7 +517,7 @@ static void stop_watchdog(void *arg) cpumask_clear_cpu(cpu, &wd_cpus_enabled); wd_smp_unlock(&flags); - wd_smp_clear_cpu_pending(cpu, get_tb()); + wd_smp_clear_cpu_pending(cpu); } static int stop_watchdog_on_cpu(unsigned int cpu) diff --git a/arch/powerpc/kexec/core.c b/arch/powerpc/kexec/core.c index 48525e8b57..8b68d9f91a 100644 --- a/arch/powerpc/kexec/core.c +++ b/arch/powerpc/kexec/core.c @@ -48,19 +48,6 @@ void machine_crash_shutdown(struct pt_regs *regs) default_machine_crash_shutdown(regs); } -/* - * Do what every setup is needed on image and the - * reboot code buffer to allow us to avoid allocations - * later. - */ -int machine_kexec_prepare(struct kimage *image) -{ - if (ppc_md.machine_kexec_prepare) - return ppc_md.machine_kexec_prepare(image); - else - return default_machine_kexec_prepare(image); -} - void machine_kexec_cleanup(struct kimage *image) { } @@ -198,7 +185,7 @@ void __init reserve_crashkernel(void) } } -int overlaps_crashkernel(unsigned long start, unsigned long size) +int __init overlaps_crashkernel(unsigned long start, unsigned long size) { return (start + size) > crashk_res.start && start <= crashk_res.end; } diff --git a/arch/powerpc/kexec/core_32.c b/arch/powerpc/kexec/core_32.c index bf9f1f906d..b50aed48d0 100644 --- a/arch/powerpc/kexec/core_32.c +++ b/arch/powerpc/kexec/core_32.c @@ -63,7 +63,7 @@ void default_machine_kexec(struct kimage *image) (*rnk)(page_list, reboot_code_buffer_phys, image->start); } -int default_machine_kexec_prepare(struct kimage *image) +int machine_kexec_prepare(struct kimage *image) { return 0; } diff --git a/arch/powerpc/kexec/core_64.c b/arch/powerpc/kexec/core_64.c index 89c069d664..635b5fc30b 100644 --- a/arch/powerpc/kexec/core_64.c +++ b/arch/powerpc/kexec/core_64.c @@ -32,7 +32,7 @@ #include #include -int default_machine_kexec_prepare(struct kimage *image) +int machine_kexec_prepare(struct kimage *image) { int i; unsigned long begin, end; /* limits of segment */ @@ -378,7 +378,7 @@ void default_machine_kexec(struct kimage *image) /* NOTREACHED */ } -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU /* Values we need to export to the second kernel via the device tree. */ static unsigned long htab_base; static unsigned long htab_size; @@ -420,4 +420,4 @@ static int __init export_htab_values(void) return 0; } late_initcall(export_htab_values); -#endif /* CONFIG_PPC_BOOK3S_64 */ +#endif /* CONFIG_PPC_64S_HASH_MMU */ diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index 5056e175ca..b4981b651d 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -700,6 +700,7 @@ static int update_usable_mem_fdt(void *fdt, struct crash_mem *usable_mem) if (ret) { pr_err("Failed to set linux,usable-memory property for %s node", dn->full_name); + of_node_put(dn); goto out; } } diff --git a/arch/powerpc/kexec/ranges.c b/arch/powerpc/kexec/ranges.c index 6b81c852fe..563e9989a5 100644 --- a/arch/powerpc/kexec/ranges.c +++ b/arch/powerpc/kexec/ranges.c @@ -296,7 +296,7 @@ int add_initrd_mem_range(struct crash_mem **mem_ranges) return ret; } -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_64S_HASH_MMU /** * add_htab_mem_range - Adds htab range to the given memory ranges list, * if it exists diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index ff581d70f2..18e5808544 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -26,6 +26,7 @@ config KVM select KVM_VFIO select IRQ_BYPASS_MANAGER select HAVE_KVM_IRQ_BYPASS + select INTERVAL_TREE config KVM_BOOK3S_HANDLER bool @@ -69,6 +70,7 @@ config KVM_BOOK3S_64 select KVM_BOOK3S_64_HANDLER select KVM select KVM_BOOK3S_PR_POSSIBLE if !KVM_BOOK3S_HV_POSSIBLE + select PPC_64S_HASH_MMU select SPAPR_TCE_IOMMU if IOMMU_SUPPORT && (PPC_PSERIES || PPC_POWERNV) help Support running unmodified book3s_64 and book3s_32 guest kernels @@ -130,6 +132,21 @@ config KVM_BOOK3S_HV_EXIT_TIMING If unsure, say N. +config KVM_BOOK3S_HV_NESTED_PMU_WORKAROUND + bool "Nested L0 host workaround for L1 KVM host PMU handling bug" if EXPERT + depends on KVM_BOOK3S_HV_POSSIBLE + default !EXPERT + help + Old nested HV capable Linux guests have a bug where they don't + reflect the PMU in-use status of their L2 guest to the L0 host + while the L2 PMU registers are live. This can result in loss + of L2 PMU register state, causing perf to not work correctly in + L2 guests. + + Selecting this option for the L0 host implements a workaround for + those buggy L1s which saves the L2 state, at the cost of performance + in all nested-capable guest entry/exit. + config KVM_BOOKE_HV bool diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index b785f67723..6d525285db 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -847,21 +847,19 @@ void kvmppc_core_flush_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot) } int kvmppc_core_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, - enum kvm_mr_change change) + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) { - return kvm->arch.kvm_ops->prepare_memory_region(kvm, memslot, mem, - change); + return kvm->arch.kvm_ops->prepare_memory_region(kvm, old, new, change); } void kvmppc_core_commit_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { - kvm->arch.kvm_ops->commit_memory_region(kvm, mem, old, new, change); + kvm->arch.kvm_ops->commit_memory_region(kvm, old, new, change); } bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) diff --git a/arch/powerpc/kvm/book3s_32_mmu.c b/arch/powerpc/kvm/book3s_32_mmu.c index 3fbd570f9c..0215f32932 100644 --- a/arch/powerpc/kvm/book3s_32_mmu.c +++ b/arch/powerpc/kvm/book3s_32_mmu.c @@ -337,7 +337,7 @@ static void kvmppc_mmu_book3s_32_mtsrin(struct kvm_vcpu *vcpu, u32 srnum, static void kvmppc_mmu_book3s_32_tlbie(struct kvm_vcpu *vcpu, ulong ea, bool large) { - int i; + unsigned long i; struct kvm_vcpu *v; /* flush this VA on all cpus */ diff --git a/arch/powerpc/kvm/book3s_64_entry.S b/arch/powerpc/kvm/book3s_64_entry.S index 983b8c18bc..05e003eb5d 100644 --- a/arch/powerpc/kvm/book3s_64_entry.S +++ b/arch/powerpc/kvm/book3s_64_entry.S @@ -374,11 +374,16 @@ END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) BEGIN_FTR_SECTION mtspr SPRN_DAWRX1,r10 END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) - mtspr SPRN_PID,r10 /* - * Switch to host MMU mode + * Switch to host MMU mode (don't have the real host PID but we aren't + * going back to userspace). */ + hwsync + isync + + mtspr SPRN_PID,r10 + ld r10, HSTATE_KVM_VCPU(r13) ld r10, VCPU_KVM(r10) lwz r10, KVM_HOST_LPID(r10) @@ -389,6 +394,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) ld r10, KVM_HOST_LPCR(r10) mtspr SPRN_LPCR,r10 + isync + /* * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear * MSR_RI in r12 ([H]SRR1) so the handler won't try to return. diff --git a/arch/powerpc/kvm/book3s_64_mmu.c b/arch/powerpc/kvm/book3s_64_mmu.c index feee40cb2b..61290282fd 100644 --- a/arch/powerpc/kvm/book3s_64_mmu.c +++ b/arch/powerpc/kvm/book3s_64_mmu.c @@ -530,7 +530,7 @@ static void kvmppc_mmu_book3s_64_tlbie(struct kvm_vcpu *vcpu, ulong va, bool large) { u64 mask = 0xFFFFFFFFFULL; - long i; + unsigned long i; struct kvm_vcpu *v; dprintk("KVM MMU: tlbie(0x%lx)\n", va); diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index c63e263312..2132329143 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -734,11 +734,11 @@ void kvmppc_rmap_reset(struct kvm *kvm) { struct kvm_memslots *slots; struct kvm_memory_slot *memslot; - int srcu_idx; + int srcu_idx, bkt; srcu_idx = srcu_read_lock(&kvm->srcu); slots = kvm_memslots(kvm); - kvm_for_each_memslot(memslot, slots) { + kvm_for_each_memslot(memslot, bkt, slots) { /* Mutual exclusion with kvm_unmap_hva_range etc. */ spin_lock(&kvm->mmu_lock); /* diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index 6365087f31..d42b4b6d4a 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -295,8 +295,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm, return ret; ret = -ENOMEM; - stt = kzalloc(sizeof(*stt) + npages * sizeof(struct page *), - GFP_KERNEL); + stt = kzalloc(struct_size(stt, pages, npages), GFP_KERNEL); if (!stt) goto fail_acct; diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index a2fd1db29f..84c89f08ae 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -80,6 +80,7 @@ #include #include "book3s.h" +#include "book3s_hv.h" #define CREATE_TRACE_POINTS #include "trace_hv.h" @@ -127,11 +128,6 @@ static bool nested = true; module_param(nested, bool, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(nested, "Enable nested virtualization (only on POWER9)"); -static inline bool nesting_enabled(struct kvm *kvm) -{ - return kvm->arch.nested_enable && kvm_is_radix(kvm); -} - static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); /* @@ -276,22 +272,26 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) * they should never fail.) */ -static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc) +static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc, u64 tb) { unsigned long flags; + WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300)); + spin_lock_irqsave(&vc->stoltb_lock, flags); - vc->preempt_tb = mftb(); + vc->preempt_tb = tb; spin_unlock_irqrestore(&vc->stoltb_lock, flags); } -static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc) +static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc, u64 tb) { unsigned long flags; + WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300)); + spin_lock_irqsave(&vc->stoltb_lock, flags); if (vc->preempt_tb != TB_NIL) { - vc->stolen_tb += mftb() - vc->preempt_tb; + vc->stolen_tb += tb - vc->preempt_tb; vc->preempt_tb = TB_NIL; } spin_unlock_irqrestore(&vc->stoltb_lock, flags); @@ -301,6 +301,12 @@ static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) { struct kvmppc_vcore *vc = vcpu->arch.vcore; unsigned long flags; + u64 now; + + if (cpu_has_feature(CPU_FTR_ARCH_300)) + return; + + now = mftb(); /* * We can test vc->runner without taking the vcore lock, @@ -309,12 +315,12 @@ static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) * ever sets it to NULL. */ if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) - kvmppc_core_end_stolen(vc); + kvmppc_core_end_stolen(vc, now); spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST && vcpu->arch.busy_preempt != TB_NIL) { - vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt; + vcpu->arch.busy_stolen += now - vcpu->arch.busy_preempt; vcpu->arch.busy_preempt = TB_NIL; } spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); @@ -324,13 +330,19 @@ static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu) { struct kvmppc_vcore *vc = vcpu->arch.vcore; unsigned long flags; + u64 now; + + if (cpu_has_feature(CPU_FTR_ARCH_300)) + return; + + now = mftb(); if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) - kvmppc_core_start_stolen(vc); + kvmppc_core_start_stolen(vc, now); spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST) - vcpu->arch.busy_preempt = mftb(); + vcpu->arch.busy_preempt = now; spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); } @@ -675,6 +687,8 @@ static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now) u64 p; unsigned long flags; + WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300)); + spin_lock_irqsave(&vc->stoltb_lock, flags); p = vc->stolen_tb; if (vc->vcore_state != VCORE_INACTIVE && @@ -684,35 +698,30 @@ static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now) return p; } -static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, - struct kvmppc_vcore *vc) +static void __kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, + unsigned int pcpu, u64 now, + unsigned long stolen) { struct dtl_entry *dt; struct lppaca *vpa; - unsigned long stolen; - unsigned long core_stolen; - u64 now; - unsigned long flags; dt = vcpu->arch.dtl_ptr; vpa = vcpu->arch.vpa.pinned_addr; - now = mftb(); - core_stolen = vcore_stolen_time(vc, now); - stolen = core_stolen - vcpu->arch.stolen_logged; - vcpu->arch.stolen_logged = core_stolen; - spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); - stolen += vcpu->arch.busy_stolen; - vcpu->arch.busy_stolen = 0; - spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); + if (!dt || !vpa) return; - memset(dt, 0, sizeof(struct dtl_entry)); + dt->dispatch_reason = 7; - dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid); - dt->timebase = cpu_to_be64(now + vc->tb_offset); + dt->preempt_reason = 0; + dt->processor_id = cpu_to_be16(pcpu + vcpu->arch.ptid); dt->enqueue_to_dispatch_time = cpu_to_be32(stolen); + dt->ready_to_enqueue_time = 0; + dt->waiting_to_ready_time = 0; + dt->timebase = cpu_to_be64(now); + dt->fault_addr = 0; dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu)); dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr); + ++dt; if (dt == vcpu->arch.dtl.pinned_end) dt = vcpu->arch.dtl.pinned_addr; @@ -723,6 +732,27 @@ static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, vcpu->arch.dtl.dirty = true; } +static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, + struct kvmppc_vcore *vc) +{ + unsigned long stolen; + unsigned long core_stolen; + u64 now; + unsigned long flags; + + now = mftb(); + + core_stolen = vcore_stolen_time(vc, now); + stolen = core_stolen - vcpu->arch.stolen_logged; + vcpu->arch.stolen_logged = core_stolen; + spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); + stolen += vcpu->arch.busy_stolen; + vcpu->arch.busy_stolen = 0; + spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); + + __kvmppc_create_dtl_entry(vcpu, vc->pcpu, now + vc->tb_offset, stolen); +} + /* See if there is a doorbell interrupt pending for a vcpu */ static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu) { @@ -731,6 +761,8 @@ static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu) if (vcpu->arch.doorbell_request) return true; + if (cpu_has_feature(CPU_FTR_ARCH_300)) + return false; /* * Ensure that the read of vcore->dpdes comes after the read * of vcpu->doorbell_request. This barrier matches the @@ -900,13 +932,14 @@ static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target) * mode handler is not called but no other threads are in the * source vcore. */ - - spin_lock(&vcore->lock); - if (target->arch.state == KVMPPC_VCPU_RUNNABLE && - vcore->vcore_state != VCORE_INACTIVE && - vcore->runner) - target = vcore->runner; - spin_unlock(&vcore->lock); + if (!cpu_has_feature(CPU_FTR_ARCH_300)) { + spin_lock(&vcore->lock); + if (target->arch.state == KVMPPC_VCPU_RUNNABLE && + vcore->vcore_state != VCORE_INACTIVE && + vcore->runner) + target = vcore->runner; + spin_unlock(&vcore->lock); + } return kvm_vcpu_yield_to(target); } @@ -1421,6 +1454,43 @@ static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu) return RESUME_GUEST; } +/* + * If the lppaca had pmcregs_in_use clear when we exited the guest, then + * HFSCR_PM is cleared for next entry. If the guest then tries to access + * the PMU SPRs, we get this facility unavailable interrupt. Putting HFSCR_PM + * back in the guest HFSCR will cause the next entry to load the PMU SPRs and + * allow the guest access to continue. + */ +static int kvmppc_pmu_unavailable(struct kvm_vcpu *vcpu) +{ + if (!(vcpu->arch.hfscr_permitted & HFSCR_PM)) + return EMULATE_FAIL; + + vcpu->arch.hfscr |= HFSCR_PM; + + return RESUME_GUEST; +} + +static int kvmppc_ebb_unavailable(struct kvm_vcpu *vcpu) +{ + if (!(vcpu->arch.hfscr_permitted & HFSCR_EBB)) + return EMULATE_FAIL; + + vcpu->arch.hfscr |= HFSCR_EBB; + + return RESUME_GUEST; +} + +static int kvmppc_tm_unavailable(struct kvm_vcpu *vcpu) +{ + if (!(vcpu->arch.hfscr_permitted & HFSCR_TM)) + return EMULATE_FAIL; + + vcpu->arch.hfscr |= HFSCR_TM; + + return RESUME_GUEST; +} + static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, struct task_struct *tsk) { @@ -1451,6 +1521,10 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, run->ready_for_interrupt_injection = 1; switch (vcpu->arch.trap) { /* We're good on these - the host merely wanted to get our attention */ + case BOOK3S_INTERRUPT_NESTED_HV_DECREMENTER: + WARN_ON_ONCE(1); /* Should never happen */ + vcpu->arch.trap = BOOK3S_INTERRUPT_HV_DECREMENTER; + fallthrough; case BOOK3S_INTERRUPT_HV_DECREMENTER: vcpu->stat.dec_exits++; r = RESUME_GUEST; @@ -1575,7 +1649,8 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, unsigned long vsid; long err; - if (vcpu->arch.fault_dsisr == HDSISR_CANARY) { + if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG) && + unlikely(vcpu->arch.fault_dsisr == HDSISR_CANARY)) { r = RESUME_GUEST; /* Just retry if it's the canary */ break; } @@ -1702,16 +1777,26 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, * to emulate. * Otherwise, we just generate a program interrupt to the guest. */ - case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: + case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: { + u64 cause = vcpu->arch.hfscr >> 56; + r = EMULATE_FAIL; - if (((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) && - cpu_has_feature(CPU_FTR_ARCH_300)) - r = kvmppc_emulate_doorbell_instr(vcpu); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + if (cause == FSCR_MSGP_LG) + r = kvmppc_emulate_doorbell_instr(vcpu); + if (cause == FSCR_PM_LG) + r = kvmppc_pmu_unavailable(vcpu); + if (cause == FSCR_EBB_LG) + r = kvmppc_ebb_unavailable(vcpu); + if (cause == FSCR_TM_LG) + r = kvmppc_tm_unavailable(vcpu); + } if (r == EMULATE_FAIL) { kvmppc_core_queue_program(vcpu, SRR1_PROGILL); r = RESUME_GUEST; } break; + } case BOOK3S_INTERRUPT_HV_RM_HARD: r = RESUME_PASSTHROUGH; @@ -1767,6 +1852,12 @@ static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu) vcpu->stat.ext_intr_exits++; r = RESUME_GUEST; break; + /* These need to go to the nested HV */ + case BOOK3S_INTERRUPT_NESTED_HV_DECREMENTER: + vcpu->arch.trap = BOOK3S_INTERRUPT_HV_DECREMENTER; + vcpu->stat.dec_exits++; + r = RESUME_HOST; + break; /* SR/HMI/PMI are HV interrupts that host has handled. Resume guest.*/ case BOOK3S_INTERRUPT_HMI: case BOOK3S_INTERRUPT_PERFMON: @@ -1992,7 +2083,7 @@ static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, */ if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) { struct kvm_vcpu *vcpu; - int i; + unsigned long i; kvm_for_each_vcpu(i, vcpu, kvm) { if (vcpu->arch.vcore != vc) @@ -2095,8 +2186,10 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, * either vcore->dpdes or doorbell_request. * On POWER8, doorbell_request is 0. */ - *val = get_reg_val(id, vcpu->arch.vcore->dpdes | - vcpu->arch.doorbell_request); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + *val = get_reg_val(id, vcpu->arch.doorbell_request); + else + *val = get_reg_val(id, vcpu->arch.vcore->dpdes); break; case KVM_REG_PPC_VTB: *val = get_reg_val(id, vcpu->arch.vcore->vtb); @@ -2237,8 +2330,7 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, *val = get_reg_val(id, vcpu->arch.vcore->arch_compat); break; case KVM_REG_PPC_DEC_EXPIRY: - *val = get_reg_val(id, vcpu->arch.dec_expires + - vcpu->arch.vcore->tb_offset); + *val = get_reg_val(id, vcpu->arch.dec_expires); break; case KVM_REG_PPC_ONLINE: *val = get_reg_val(id, vcpu->arch.online); @@ -2334,7 +2426,10 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, vcpu->arch.pspb = set_reg_val(id, *val); break; case KVM_REG_PPC_DPDES: - vcpu->arch.vcore->dpdes = set_reg_val(id, *val); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + vcpu->arch.doorbell_request = set_reg_val(id, *val) & 1; + else + vcpu->arch.vcore->dpdes = set_reg_val(id, *val); break; case KVM_REG_PPC_VTB: vcpu->arch.vcore->vtb = set_reg_val(id, *val); @@ -2490,8 +2585,7 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_DEC_EXPIRY: - vcpu->arch.dec_expires = set_reg_val(id, *val) - - vcpu->arch.vcore->tb_offset; + vcpu->arch.dec_expires = set_reg_val(id, *val); break; case KVM_REG_PPC_ONLINE: i = set_reg_val(id, *val); @@ -2714,6 +2808,11 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) #endif #endif vcpu->arch.mmcr[0] = MMCR0_FC; + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + vcpu->arch.mmcr[0] |= MMCR0_PMCCEXT; + vcpu->arch.mmcra = MMCRA_BHRB_DISABLE; + } + vcpu->arch.ctrl = CTRL_RUNLATCH; /* default to host PVR, since we can't spoof it */ kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR)); @@ -2744,6 +2843,11 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) vcpu->arch.hfscr_permitted = vcpu->arch.hfscr; + /* + * PM, EBB, TM are demand-faulted so start with it clear. + */ + vcpu->arch.hfscr &= ~(HFSCR_PM | HFSCR_EBB | HFSCR_TM); + kvmppc_mmu_book3s_hv_init(vcpu); vcpu->arch.state = KVMPPC_VCPU_NOTREADY; @@ -2868,13 +2972,13 @@ static void kvmppc_set_timer(struct kvm_vcpu *vcpu) unsigned long dec_nsec, now; now = get_tb(); - if (now > vcpu->arch.dec_expires) { + if (now > kvmppc_dec_expires_host_tb(vcpu)) { /* decrementer has already gone negative */ kvmppc_core_queue_dec(vcpu); kvmppc_core_prepare_to_enter(vcpu); return; } - dec_nsec = tb_to_ns(vcpu->arch.dec_expires - now); + dec_nsec = tb_to_ns(kvmppc_dec_expires_host_tb(vcpu) - now); hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL); vcpu->arch.timer_running = 1; } @@ -2882,14 +2986,14 @@ static void kvmppc_set_timer(struct kvm_vcpu *vcpu) extern int __kvmppc_vcore_entry(void); static void kvmppc_remove_runnable(struct kvmppc_vcore *vc, - struct kvm_vcpu *vcpu) + struct kvm_vcpu *vcpu, u64 tb) { u64 now; if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) return; spin_lock_irq(&vcpu->arch.tbacct_lock); - now = mftb(); + now = tb; vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) - vcpu->arch.stolen_logged; vcpu->arch.busy_preempt = now; @@ -2944,30 +3048,59 @@ static void kvmppc_release_hwthread(int cpu) tpaca->kvm_hstate.kvm_split_mode = NULL; } +static DEFINE_PER_CPU(struct kvm *, cpu_in_guest); + static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu) { struct kvm_nested_guest *nested = vcpu->arch.nested; - cpumask_t *cpu_in_guest; + cpumask_t *need_tlb_flush; int i; + if (nested) + need_tlb_flush = &nested->need_tlb_flush; + else + need_tlb_flush = &kvm->arch.need_tlb_flush; + cpu = cpu_first_tlb_thread_sibling(cpu); - if (nested) { - cpumask_set_cpu(cpu, &nested->need_tlb_flush); - cpu_in_guest = &nested->cpu_in_guest; - } else { - cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush); - cpu_in_guest = &kvm->arch.cpu_in_guest; - } - /* - * Make sure setting of bit in need_tlb_flush precedes - * testing of cpu_in_guest bits. The matching barrier on - * the other side is the first smp_mb() in kvmppc_run_core(). - */ - smp_mb(); for (i = cpu; i <= cpu_last_tlb_thread_sibling(cpu); i += cpu_tlb_thread_sibling_step()) - if (cpumask_test_cpu(i, cpu_in_guest)) + cpumask_set_cpu(i, need_tlb_flush); + + /* + * Make sure setting of bit in need_tlb_flush precedes testing of + * cpu_in_guest. The matching barrier on the other side is hwsync + * when switching to guest MMU mode, which happens between + * cpu_in_guest being set to the guest kvm, and need_tlb_flush bit + * being tested. + */ + smp_mb(); + + for (i = cpu; i <= cpu_last_tlb_thread_sibling(cpu); + i += cpu_tlb_thread_sibling_step()) { + struct kvm *running = *per_cpu_ptr(&cpu_in_guest, i); + + if (running == kvm) smp_call_function_single(i, do_nothing, NULL, 1); + } +} + +static void do_migrate_away_vcpu(void *arg) +{ + struct kvm_vcpu *vcpu = arg; + struct kvm *kvm = vcpu->kvm; + + /* + * If the guest has GTSE, it may execute tlbie, so do a eieio; tlbsync; + * ptesync sequence on the old CPU before migrating to a new one, in + * case we interrupted the guest between a tlbie ; eieio ; + * tlbsync; ptesync sequence. + * + * Otherwise, ptesync is sufficient for ordering tlbiel sequences. + */ + if (kvm->arch.lpcr & LPCR_GTSE) + asm volatile("eieio; tlbsync; ptesync"); + else + asm volatile("ptesync"); } static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu) @@ -2993,14 +3126,17 @@ static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu) * can move around between pcpus. To cope with this, when * a vcpu moves from one pcpu to another, we need to tell * any vcpus running on the same core as this vcpu previously - * ran to flush the TLB. The TLB is shared between threads, - * so we use a single bit in .need_tlb_flush for all 4 threads. + * ran to flush the TLB. */ if (prev_cpu != pcpu) { - if (prev_cpu >= 0 && - cpu_first_tlb_thread_sibling(prev_cpu) != - cpu_first_tlb_thread_sibling(pcpu)) - radix_flush_cpu(kvm, prev_cpu, vcpu); + if (prev_cpu >= 0) { + if (cpu_first_tlb_thread_sibling(prev_cpu) != + cpu_first_tlb_thread_sibling(pcpu)) + radix_flush_cpu(kvm, prev_cpu, vcpu); + + smp_call_function_single(prev_cpu, + do_migrate_away_vcpu, vcpu, 1); + } if (nested) nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu; else @@ -3012,7 +3148,6 @@ static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) { int cpu; struct paca_struct *tpaca; - struct kvm *kvm = vc->kvm; cpu = vc->pcpu; if (vcpu) { @@ -3023,7 +3158,6 @@ static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) cpu += vcpu->arch.ptid; vcpu->cpu = vc->pcpu; vcpu->arch.thread_cpu = cpu; - cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest); } tpaca = paca_ptrs[cpu]; tpaca->kvm_hstate.kvm_vcpu = vcpu; @@ -3124,6 +3258,8 @@ static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) { struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); + WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300)); + vc->vcore_state = VCORE_PREEMPT; vc->pcpu = smp_processor_id(); if (vc->num_threads < threads_per_vcore(vc->kvm)) { @@ -3133,14 +3269,16 @@ static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) } /* Start accumulating stolen time */ - kvmppc_core_start_stolen(vc); + kvmppc_core_start_stolen(vc, mftb()); } static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc) { struct preempted_vcore_list *lp; - kvmppc_core_end_stolen(vc); + WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300)); + + kvmppc_core_end_stolen(vc, mftb()); if (!list_empty(&vc->preempt_list)) { lp = &per_cpu(preempted_vcores, vc->pcpu); spin_lock(&lp->lock); @@ -3267,7 +3405,7 @@ static void prepare_threads(struct kvmppc_vcore *vc) vcpu->arch.ret = RESUME_GUEST; else continue; - kvmppc_remove_runnable(vc, vcpu); + kvmppc_remove_runnable(vc, vcpu, mftb()); wake_up(&vcpu->arch.cpu_run); } } @@ -3286,7 +3424,7 @@ static void collect_piggybacks(struct core_info *cip, int target_threads) list_del_init(&pvc->preempt_list); if (pvc->runner == NULL) { pvc->vcore_state = VCORE_INACTIVE; - kvmppc_core_end_stolen(pvc); + kvmppc_core_end_stolen(pvc, mftb()); } spin_unlock(&pvc->lock); continue; @@ -3295,7 +3433,7 @@ static void collect_piggybacks(struct core_info *cip, int target_threads) spin_unlock(&pvc->lock); continue; } - kvmppc_core_end_stolen(pvc); + kvmppc_core_end_stolen(pvc, mftb()); pvc->vcore_state = VCORE_PIGGYBACK; if (cip->total_threads >= target_threads) break; @@ -3339,7 +3477,7 @@ static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) */ spin_unlock(&vc->lock); /* cancel pending dec exception if dec is positive */ - if (now < vcpu->arch.dec_expires && + if (now < kvmppc_dec_expires_host_tb(vcpu) && kvmppc_core_pending_dec(vcpu)) kvmppc_core_dequeue_dec(vcpu); @@ -3362,7 +3500,7 @@ static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) else ++still_running; } else { - kvmppc_remove_runnable(vc, vcpu); + kvmppc_remove_runnable(vc, vcpu, mftb()); wake_up(&vcpu->arch.cpu_run); } } @@ -3371,7 +3509,7 @@ static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) kvmppc_vcore_preempt(vc); } else if (vc->runner) { vc->vcore_state = VCORE_PREEMPT; - kvmppc_core_start_stolen(vc); + kvmppc_core_start_stolen(vc, mftb()); } else { vc->vcore_state = VCORE_INACTIVE; } @@ -3502,7 +3640,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) { for_each_runnable_thread(i, vcpu, vc) { vcpu->arch.ret = -EBUSY; - kvmppc_remove_runnable(vc, vcpu); + kvmppc_remove_runnable(vc, vcpu, mftb()); wake_up(&vcpu->arch.cpu_run); } goto out; @@ -3747,7 +3885,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) kvmppc_release_hwthread(pcpu + i); if (sip && sip->napped[i]) kvmppc_ipi_thread(pcpu + i); - cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest); } spin_unlock(&vc->lock); @@ -3769,210 +3906,136 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) trace_kvmppc_run_core(vc, 1); } -static void load_spr_state(struct kvm_vcpu *vcpu) -{ - mtspr(SPRN_DSCR, vcpu->arch.dscr); - mtspr(SPRN_IAMR, vcpu->arch.iamr); - mtspr(SPRN_PSPB, vcpu->arch.pspb); - mtspr(SPRN_FSCR, vcpu->arch.fscr); - mtspr(SPRN_TAR, vcpu->arch.tar); - mtspr(SPRN_EBBHR, vcpu->arch.ebbhr); - mtspr(SPRN_EBBRR, vcpu->arch.ebbrr); - mtspr(SPRN_BESCR, vcpu->arch.bescr); - mtspr(SPRN_TIDR, vcpu->arch.tid); - mtspr(SPRN_AMR, vcpu->arch.amr); - mtspr(SPRN_UAMOR, vcpu->arch.uamor); - - /* - * DAR, DSISR, and for nested HV, SPRGs must be set with MSR[RI] - * clear (or hstate set appropriately to catch those registers - * being clobbered if we take a MCE or SRESET), so those are done - * later. - */ - - if (!(vcpu->arch.ctrl & 1)) - mtspr(SPRN_CTRLT, mfspr(SPRN_CTRLF) & ~1); -} - -static void store_spr_state(struct kvm_vcpu *vcpu) -{ - vcpu->arch.ctrl = mfspr(SPRN_CTRLF); - - vcpu->arch.iamr = mfspr(SPRN_IAMR); - vcpu->arch.pspb = mfspr(SPRN_PSPB); - vcpu->arch.fscr = mfspr(SPRN_FSCR); - vcpu->arch.tar = mfspr(SPRN_TAR); - vcpu->arch.ebbhr = mfspr(SPRN_EBBHR); - vcpu->arch.ebbrr = mfspr(SPRN_EBBRR); - vcpu->arch.bescr = mfspr(SPRN_BESCR); - vcpu->arch.tid = mfspr(SPRN_TIDR); - vcpu->arch.amr = mfspr(SPRN_AMR); - vcpu->arch.uamor = mfspr(SPRN_UAMOR); - vcpu->arch.dscr = mfspr(SPRN_DSCR); -} - -/* - * Privileged (non-hypervisor) host registers to save. - */ -struct p9_host_os_sprs { - unsigned long dscr; - unsigned long tidr; - unsigned long iamr; - unsigned long amr; - unsigned long fscr; -}; - -static void save_p9_host_os_sprs(struct p9_host_os_sprs *host_os_sprs) -{ - host_os_sprs->dscr = mfspr(SPRN_DSCR); - host_os_sprs->tidr = mfspr(SPRN_TIDR); - host_os_sprs->iamr = mfspr(SPRN_IAMR); - host_os_sprs->amr = mfspr(SPRN_AMR); - host_os_sprs->fscr = mfspr(SPRN_FSCR); -} - -/* vcpu guest regs must already be saved */ -static void restore_p9_host_os_sprs(struct kvm_vcpu *vcpu, - struct p9_host_os_sprs *host_os_sprs) -{ - mtspr(SPRN_PSPB, 0); - mtspr(SPRN_UAMOR, 0); - - mtspr(SPRN_DSCR, host_os_sprs->dscr); - mtspr(SPRN_TIDR, host_os_sprs->tidr); - mtspr(SPRN_IAMR, host_os_sprs->iamr); - - if (host_os_sprs->amr != vcpu->arch.amr) - mtspr(SPRN_AMR, host_os_sprs->amr); - - if (host_os_sprs->fscr != vcpu->arch.fscr) - mtspr(SPRN_FSCR, host_os_sprs->fscr); - - /* Save guest CTRL register, set runlatch to 1 */ - if (!(vcpu->arch.ctrl & 1)) - mtspr(SPRN_CTRLT, 1); -} - static inline bool hcall_is_xics(unsigned long req) { return req == H_EOI || req == H_CPPR || req == H_IPI || req == H_IPOLL || req == H_XIRR || req == H_XIRR_X; } -/* - * Guest entry for POWER9 and later CPUs. - */ -static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, - unsigned long lpcr) +static void vcpu_vpa_increment_dispatch(struct kvm_vcpu *vcpu) { - struct kvmppc_vcore *vc = vcpu->arch.vcore; - struct p9_host_os_sprs host_os_sprs; - s64 dec; - u64 tb; - int trap, save_pmu; - - WARN_ON_ONCE(vcpu->arch.ceded); - - dec = mfspr(SPRN_DEC); - tb = mftb(); - if (dec < 0) - return BOOK3S_INTERRUPT_HV_DECREMENTER; - local_paca->kvm_hstate.dec_expires = dec + tb; - if (local_paca->kvm_hstate.dec_expires < time_limit) - time_limit = local_paca->kvm_hstate.dec_expires; - - save_p9_host_os_sprs(&host_os_sprs); - - kvmhv_save_host_pmu(); /* saves it to PACA kvm_hstate */ - - kvmppc_subcore_enter_guest(); - - vc->entry_exit_map = 1; - vc->in_guest = 1; - - if (vcpu->arch.vpa.pinned_addr) { - struct lppaca *lp = vcpu->arch.vpa.pinned_addr; + struct lppaca *lp = vcpu->arch.vpa.pinned_addr; + if (lp) { u32 yield_count = be32_to_cpu(lp->yield_count) + 1; lp->yield_count = cpu_to_be32(yield_count); vcpu->arch.vpa.dirty = 1; } +} - if (cpu_has_feature(CPU_FTR_TM) || - cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) - kvmppc_restore_tm_hv(vcpu, vcpu->arch.shregs.msr, true); +/* call our hypervisor to load up HV regs and go */ +static int kvmhv_vcpu_entry_p9_nested(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb) +{ + struct kvmppc_vcore *vc = vcpu->arch.vcore; + unsigned long host_psscr; + unsigned long msr; + struct hv_guest_state hvregs; + struct p9_host_os_sprs host_os_sprs; + s64 dec; + int trap; -#ifdef CONFIG_PPC_PSERIES - if (kvmhv_on_pseries()) { - barrier(); - if (vcpu->arch.vpa.pinned_addr) { - struct lppaca *lp = vcpu->arch.vpa.pinned_addr; - get_lppaca()->pmcregs_in_use = lp->pmcregs_in_use; - } else { - get_lppaca()->pmcregs_in_use = 1; - } - barrier(); - } -#endif - kvmhv_load_guest_pmu(vcpu); + msr = mfmsr(); - msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX); - load_fp_state(&vcpu->arch.fp); -#ifdef CONFIG_ALTIVEC - load_vr_state(&vcpu->arch.vr); -#endif - mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); - - load_spr_state(vcpu); + save_p9_host_os_sprs(&host_os_sprs); /* - * When setting DEC, we must always deal with irq_work_raise via NMI vs - * setting DEC. The problem occurs right as we switch into guest mode - * if a NMI hits and sets pending work and sets DEC, then that will - * apply to the guest and not bring us back to the host. + * We need to save and restore the guest visible part of the + * psscr (i.e. using SPRN_PSSCR_PR) since the hypervisor + * doesn't do this for us. Note only required if pseries since + * this is done in kvmhv_vcpu_entry_p9() below otherwise. + */ + host_psscr = mfspr(SPRN_PSSCR_PR); + + kvmppc_msr_hard_disable_set_facilities(vcpu, msr); + if (lazy_irq_pending()) + return 0; + + if (unlikely(load_vcpu_state(vcpu, &host_os_sprs))) + msr = mfmsr(); /* TM restore can update msr */ + + if (vcpu->arch.psscr != host_psscr) + mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr); + + kvmhv_save_hv_regs(vcpu, &hvregs); + hvregs.lpcr = lpcr; + vcpu->arch.regs.msr = vcpu->arch.shregs.msr; + hvregs.version = HV_GUEST_STATE_VERSION; + if (vcpu->arch.nested) { + hvregs.lpid = vcpu->arch.nested->shadow_lpid; + hvregs.vcpu_token = vcpu->arch.nested_vcpu_id; + } else { + hvregs.lpid = vcpu->kvm->arch.lpid; + hvregs.vcpu_token = vcpu->vcpu_id; + } + hvregs.hdec_expiry = time_limit; + + /* + * When setting DEC, we must always deal with irq_work_raise + * via NMI vs setting DEC. The problem occurs right as we + * switch into guest mode if a NMI hits and sets pending work + * and sets DEC, then that will apply to the guest and not + * bring us back to the host. * - * irq_work_raise could check a flag (or possibly LPCR[HDICE] for - * example) and set HDEC to 1? That wouldn't solve the nested hv - * case which needs to abort the hcall or zero the time limit. + * irq_work_raise could check a flag (or possibly LPCR[HDICE] + * for example) and set HDEC to 1? That wouldn't solve the + * nested hv case which needs to abort the hcall or zero the + * time limit. * * XXX: Another day's problem. */ - mtspr(SPRN_DEC, vcpu->arch.dec_expires - mftb()); + mtspr(SPRN_DEC, kvmppc_dec_expires_host_tb(vcpu) - *tb); + + mtspr(SPRN_DAR, vcpu->arch.shregs.dar); + mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr); + switch_pmu_to_guest(vcpu, &host_os_sprs); + trap = plpar_hcall_norets(H_ENTER_NESTED, __pa(&hvregs), + __pa(&vcpu->arch.regs)); + kvmhv_restore_hv_return_state(vcpu, &hvregs); + switch_pmu_to_host(vcpu, &host_os_sprs); + vcpu->arch.shregs.msr = vcpu->arch.regs.msr; + vcpu->arch.shregs.dar = mfspr(SPRN_DAR); + vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR); + vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR); + + store_vcpu_state(vcpu); + + dec = mfspr(SPRN_DEC); + if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */ + dec = (s32) dec; + *tb = mftb(); + vcpu->arch.dec_expires = dec + (*tb + vc->tb_offset); + + timer_rearm_host_dec(*tb); + + restore_p9_host_os_sprs(vcpu, &host_os_sprs); + if (vcpu->arch.psscr != host_psscr) + mtspr(SPRN_PSSCR_PR, host_psscr); + + return trap; +} + +/* + * Guest entry for POWER9 and later CPUs. + */ +static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, + unsigned long lpcr, u64 *tb) +{ + u64 next_timer; + int trap; + + next_timer = timer_get_next_tb(); + if (*tb >= next_timer) + return BOOK3S_INTERRUPT_HV_DECREMENTER; + if (next_timer < time_limit) + time_limit = next_timer; + else if (*tb >= time_limit) /* nested time limit */ + return BOOK3S_INTERRUPT_NESTED_HV_DECREMENTER; + + vcpu->arch.ceded = 0; + + vcpu_vpa_increment_dispatch(vcpu); if (kvmhv_on_pseries()) { - /* - * We need to save and restore the guest visible part of the - * psscr (i.e. using SPRN_PSSCR_PR) since the hypervisor - * doesn't do this for us. Note only required if pseries since - * this is done in kvmhv_vcpu_entry_p9() below otherwise. - */ - unsigned long host_psscr; - /* call our hypervisor to load up HV regs and go */ - struct hv_guest_state hvregs; - - host_psscr = mfspr(SPRN_PSSCR_PR); - mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr); - kvmhv_save_hv_regs(vcpu, &hvregs); - hvregs.lpcr = lpcr; - vcpu->arch.regs.msr = vcpu->arch.shregs.msr; - hvregs.version = HV_GUEST_STATE_VERSION; - if (vcpu->arch.nested) { - hvregs.lpid = vcpu->arch.nested->shadow_lpid; - hvregs.vcpu_token = vcpu->arch.nested_vcpu_id; - } else { - hvregs.lpid = vcpu->kvm->arch.lpid; - hvregs.vcpu_token = vcpu->vcpu_id; - } - hvregs.hdec_expiry = time_limit; - mtspr(SPRN_DAR, vcpu->arch.shregs.dar); - mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr); - trap = plpar_hcall_norets(H_ENTER_NESTED, __pa(&hvregs), - __pa(&vcpu->arch.regs)); - kvmhv_restore_hv_return_state(vcpu, &hvregs); - vcpu->arch.shregs.msr = vcpu->arch.regs.msr; - vcpu->arch.shregs.dar = mfspr(SPRN_DAR); - vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR); - vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR); - mtspr(SPRN_PSSCR_PR, host_psscr); + trap = kvmhv_vcpu_entry_p9_nested(vcpu, time_limit, lpcr, tb); /* H_CEDE has to be handled now, not later */ if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested && @@ -3981,9 +4044,16 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, kvmppc_set_gpr(vcpu, 3, 0); trap = 0; } + } else { + struct kvm *kvm = vcpu->kvm; + kvmppc_xive_push_vcpu(vcpu); - trap = kvmhv_vcpu_entry_p9(vcpu, time_limit, lpcr); + + __this_cpu_write(cpu_in_guest, kvm); + trap = kvmhv_vcpu_entry_p9(vcpu, time_limit, lpcr, tb); + __this_cpu_write(cpu_in_guest, NULL); + if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested && !(vcpu->arch.shregs.msr & MSR_PR)) { unsigned long req = kvmppc_get_gpr(vcpu, 3); @@ -4008,65 +4078,11 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, } kvmppc_xive_pull_vcpu(vcpu); - if (kvm_is_radix(vcpu->kvm)) + if (kvm_is_radix(kvm)) vcpu->arch.slb_max = 0; } - dec = mfspr(SPRN_DEC); - if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */ - dec = (s32) dec; - tb = mftb(); - vcpu->arch.dec_expires = dec + tb; - vcpu->cpu = -1; - vcpu->arch.thread_cpu = -1; - - store_spr_state(vcpu); - - restore_p9_host_os_sprs(vcpu, &host_os_sprs); - - msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX); - store_fp_state(&vcpu->arch.fp); -#ifdef CONFIG_ALTIVEC - store_vr_state(&vcpu->arch.vr); -#endif - vcpu->arch.vrsave = mfspr(SPRN_VRSAVE); - - if (cpu_has_feature(CPU_FTR_TM) || - cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) - kvmppc_save_tm_hv(vcpu, vcpu->arch.shregs.msr, true); - - save_pmu = 1; - if (vcpu->arch.vpa.pinned_addr) { - struct lppaca *lp = vcpu->arch.vpa.pinned_addr; - u32 yield_count = be32_to_cpu(lp->yield_count) + 1; - lp->yield_count = cpu_to_be32(yield_count); - vcpu->arch.vpa.dirty = 1; - save_pmu = lp->pmcregs_in_use; - } - /* Must save pmu if this guest is capable of running nested guests */ - save_pmu |= nesting_enabled(vcpu->kvm); - - kvmhv_save_guest_pmu(vcpu, save_pmu); -#ifdef CONFIG_PPC_PSERIES - if (kvmhv_on_pseries()) { - barrier(); - get_lppaca()->pmcregs_in_use = ppc_get_pmu_inuse(); - barrier(); - } -#endif - - vc->entry_exit_map = 0x101; - vc->in_guest = 0; - - mtspr(SPRN_DEC, local_paca->kvm_hstate.dec_expires - mftb()); - /* We may have raced with new irq work */ - if (test_irq_work_pending()) - set_dec(1); - mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso); - - kvmhv_load_host_pmu(); - - kvmppc_subcore_exit_guest(); + vcpu_vpa_increment_dispatch(vcpu); return trap; } @@ -4131,6 +4147,13 @@ static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu) return false; } +static bool kvmppc_vcpu_check_block(struct kvm_vcpu *vcpu) +{ + if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu)) + return true; + return false; +} + /* * Check to see if any of the runnable vcpus on the vcore have pending * exceptions or are no longer ceded @@ -4141,7 +4164,7 @@ static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc) int i; for_each_runnable_thread(i, vcpu, vc) { - if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu)) + if (kvmppc_vcpu_check_block(vcpu)) return 1; } @@ -4158,6 +4181,8 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) int do_sleep = 1; u64 block_ns; + WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300)); + /* Poll for pending exceptions and ceded state */ cur = start_poll = ktime_get(); if (vc->halt_poll_ns) { @@ -4354,7 +4379,7 @@ static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu) for_each_runnable_thread(i, v, vc) { kvmppc_core_prepare_to_enter(v); if (signal_pending(v->arch.run_task)) { - kvmppc_remove_runnable(vc, v); + kvmppc_remove_runnable(vc, v, mftb()); v->stat.signal_exits++; v->run->exit_reason = KVM_EXIT_INTR; v->arch.ret = -EINTR; @@ -4395,7 +4420,7 @@ static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu) kvmppc_vcore_end_preempt(vc); if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { - kvmppc_remove_runnable(vc, vcpu); + kvmppc_remove_runnable(vc, vcpu, mftb()); vcpu->stat.signal_exits++; run->exit_reason = KVM_EXIT_INTR; vcpu->arch.ret = -EINTR; @@ -4416,12 +4441,15 @@ static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu) int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr) { + struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); struct kvm_run *run = vcpu->run; int trap, r, pcpu; int srcu_idx; struct kvmppc_vcore *vc; struct kvm *kvm = vcpu->kvm; struct kvm_nested_guest *nested = vcpu->arch.nested; + unsigned long flags; + u64 tb; trace_kvmppc_run_vcpu_enter(vcpu); @@ -4432,16 +4460,11 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, vc = vcpu->arch.vcore; vcpu->arch.ceded = 0; vcpu->arch.run_task = current; - vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb()); vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; - vcpu->arch.busy_preempt = TB_NIL; vcpu->arch.last_inst = KVM_INST_FETCH_FAILED; - vc->runnable_threads[0] = vcpu; - vc->n_runnable = 1; - vc->runner = vcpu; /* See if the MMU is ready to go */ - if (!kvm->arch.mmu_ready) { + if (unlikely(!kvm->arch.mmu_ready)) { r = kvmhv_setup_mmu(vcpu); if (r) { run->exit_reason = KVM_EXIT_FAIL_ENTRY; @@ -4456,29 +4479,21 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, kvmppc_update_vpas(vcpu); - init_vcore_to_run(vc); - vc->preempt_tb = TB_NIL; - preempt_disable(); pcpu = smp_processor_id(); - vc->pcpu = pcpu; if (kvm_is_radix(kvm)) kvmppc_prepare_radix_vcpu(vcpu, pcpu); - local_irq_disable(); - hard_irq_disable(); + /* flags save not required, but irq_pmu has no disable/enable API */ + powerpc_local_irq_pmu_save(flags); + if (signal_pending(current)) goto sigpend; - if (lazy_irq_pending() || need_resched() || !kvm->arch.mmu_ready) + if (need_resched() || !kvm->arch.mmu_ready) goto out; if (!nested) { kvmppc_core_prepare_to_enter(vcpu); - if (vcpu->arch.doorbell_request) { - vc->dpdes = 1; - smp_wmb(); - vcpu->arch.doorbell_request = 0; - } if (test_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions)) lpcr |= LPCR_MER; @@ -4489,17 +4504,24 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, goto out; } - kvmppc_clear_host_core(pcpu); + if (vcpu->arch.timer_running) { + hrtimer_try_to_cancel(&vcpu->arch.dec_timer); + vcpu->arch.timer_running = 0; + } + + tb = mftb(); + + vcpu->cpu = pcpu; + vcpu->arch.thread_cpu = pcpu; + vc->pcpu = pcpu; + local_paca->kvm_hstate.kvm_vcpu = vcpu; + local_paca->kvm_hstate.ptid = 0; + local_paca->kvm_hstate.fake_suspend = 0; + + __kvmppc_create_dtl_entry(vcpu, pcpu, tb + vc->tb_offset, 0); - local_paca->kvm_hstate.napping = 0; - local_paca->kvm_hstate.kvm_split_mode = NULL; - kvmppc_start_thread(vcpu, vc); - kvmppc_create_dtl_entry(vcpu, vc); trace_kvm_guest_enter(vcpu); - vc->vcore_state = VCORE_RUNNING; - trace_kvmppc_run_core(vc, 0); - guest_enter_irqoff(); srcu_idx = srcu_read_lock(&kvm->srcu); @@ -4509,7 +4531,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, /* Tell lockdep that we're about to enable interrupts */ trace_hardirqs_on(); - trap = kvmhv_p9_guest_entry(vcpu, time_limit, lpcr); + trap = kvmhv_p9_guest_entry(vcpu, time_limit, lpcr, &tb); vcpu->arch.trap = trap; trace_hardirqs_off(); @@ -4520,8 +4542,6 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, set_irq_happened(trap); - kvmppc_set_host_core(pcpu); - context_tracking_guest_exit(); if (!vtime_accounting_enabled_this_cpu()) { local_irq_enable(); @@ -4537,9 +4557,10 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, } vtime_account_guest_exit(); - local_irq_enable(); + vcpu->cpu = -1; + vcpu->arch.thread_cpu = -1; - cpumask_clear_cpu(pcpu, &kvm->arch.cpu_in_guest); + powerpc_local_irq_pmu_restore(flags); preempt_enable(); @@ -4549,7 +4570,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, * by L2 and the L1 decrementer is provided in hdec_expires */ if (kvmppc_core_pending_dec(vcpu) && - ((get_tb() < vcpu->arch.dec_expires) || + ((tb < kvmppc_dec_expires_host_tb(vcpu)) || (trap == BOOK3S_INTERRUPT_SYSCALL && kvmppc_get_gpr(vcpu, 3) == H_ENTER_NESTED))) kvmppc_core_dequeue_dec(vcpu); @@ -4564,28 +4585,31 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, } vcpu->arch.ret = r; - if (is_kvmppc_resume_guest(r) && vcpu->arch.ceded && - !kvmppc_vcpu_woken(vcpu)) { + if (is_kvmppc_resume_guest(r) && !kvmppc_vcpu_check_block(vcpu)) { kvmppc_set_timer(vcpu); - while (vcpu->arch.ceded && !kvmppc_vcpu_woken(vcpu)) { + + prepare_to_rcuwait(wait); + for (;;) { + set_current_state(TASK_INTERRUPTIBLE); if (signal_pending(current)) { vcpu->stat.signal_exits++; run->exit_reason = KVM_EXIT_INTR; vcpu->arch.ret = -EINTR; break; } - spin_lock(&vc->lock); - kvmppc_vcore_blocked(vc); - spin_unlock(&vc->lock); + + if (kvmppc_vcpu_check_block(vcpu)) + break; + + trace_kvmppc_vcore_blocked(vc, 0); + schedule(); + trace_kvmppc_vcore_blocked(vc, 1); } + finish_rcuwait(wait); } vcpu->arch.ceded = 0; - vc->vcore_state = VCORE_INACTIVE; - trace_kvmppc_run_core(vc, 1); - done: - kvmppc_remove_runnable(vc, vcpu); trace_kvmppc_run_vcpu_exit(vcpu); return vcpu->arch.ret; @@ -4595,7 +4619,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, run->exit_reason = KVM_EXIT_INTR; vcpu->arch.ret = -EINTR; out: - local_irq_enable(); + powerpc_local_irq_pmu_restore(flags); preempt_enable(); goto done; } @@ -4605,23 +4629,25 @@ static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu) struct kvm_run *run = vcpu->run; int r; int srcu_idx; - unsigned long ebb_regs[3] = {}; /* shut up GCC */ - unsigned long user_tar = 0; - unsigned int user_vrsave; struct kvm *kvm; + unsigned long msr; if (!vcpu->arch.sane) { run->exit_reason = KVM_EXIT_INTERNAL_ERROR; return -EINVAL; } + /* No need to go into the guest when all we'll do is come back out */ + if (signal_pending(current)) { + run->exit_reason = KVM_EXIT_INTR; + return -EINTR; + } + +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM /* * Don't allow entry with a suspended transaction, because * the guest entry/exit code will lose it. - * If the guest has TM enabled, save away their TM-related SPRs - * (they will get restored by the TM unavailable interrupt). */ -#ifdef CONFIG_PPC_TRANSACTIONAL_MEM if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs && (current->thread.regs->msr & MSR_TM)) { if (MSR_TM_ACTIVE(current->thread.regs->msr)) { @@ -4629,12 +4655,6 @@ static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu) run->fail_entry.hardware_entry_failure_reason = 0; return -EINVAL; } - /* Enable TM so we can read the TM SPRs */ - mtmsr(mfmsr() | MSR_TM); - current->thread.tm_tfhar = mfspr(SPRN_TFHAR); - current->thread.tm_tfiar = mfspr(SPRN_TFIAR); - current->thread.tm_texasr = mfspr(SPRN_TEXASR); - current->thread.regs->msr &= ~MSR_TM; } #endif @@ -4649,29 +4669,30 @@ static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu) kvmppc_core_prepare_to_enter(vcpu); - /* No need to go into the guest when all we'll do is come back out */ - if (signal_pending(current)) { - run->exit_reason = KVM_EXIT_INTR; - return -EINTR; - } - kvm = vcpu->kvm; atomic_inc(&kvm->arch.vcpus_running); /* Order vcpus_running vs. mmu_ready, see kvmppc_alloc_reset_hpt */ smp_mb(); - flush_all_to_thread(current); + msr = 0; + if (IS_ENABLED(CONFIG_PPC_FPU)) + msr |= MSR_FP; + if (cpu_has_feature(CPU_FTR_ALTIVEC)) + msr |= MSR_VEC; + if (cpu_has_feature(CPU_FTR_VSX)) + msr |= MSR_VSX; + if ((cpu_has_feature(CPU_FTR_TM) || + cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) && + (vcpu->arch.hfscr & HFSCR_TM)) + msr |= MSR_TM; + msr = msr_check_and_set(msr); - /* Save userspace EBB and other register values */ - if (cpu_has_feature(CPU_FTR_ARCH_207S)) { - ebb_regs[0] = mfspr(SPRN_EBBHR); - ebb_regs[1] = mfspr(SPRN_EBBRR); - ebb_regs[2] = mfspr(SPRN_BESCR); - user_tar = mfspr(SPRN_TAR); - } - user_vrsave = mfspr(SPRN_VRSAVE); + kvmppc_save_user_regs(); - vcpu->arch.waitp = &vcpu->arch.vcore->wait; + kvmppc_save_current_sprs(); + + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + vcpu->arch.waitp = &vcpu->arch.vcore->wait; vcpu->arch.pgdir = kvm->mm->pgd; vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; @@ -4710,15 +4731,6 @@ static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu) } } while (is_kvmppc_resume_guest(r)); - /* Restore userspace EBB and other register values */ - if (cpu_has_feature(CPU_FTR_ARCH_207S)) { - mtspr(SPRN_EBBHR, ebb_regs[0]); - mtspr(SPRN_EBBRR, ebb_regs[1]); - mtspr(SPRN_BESCR, ebb_regs[2]); - mtspr(SPRN_TAR, user_tar); - } - mtspr(SPRN_VRSAVE, user_vrsave); - vcpu->arch.state = KVMPPC_VCPU_NOTREADY; atomic_dec(&kvm->arch.vcpus_running); @@ -4785,8 +4797,8 @@ static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm, { struct kvm_memslots *slots; struct kvm_memory_slot *memslot; - int i, r; - unsigned long n; + int r; + unsigned long n, i; unsigned long *buf, *p; struct kvm_vcpu *vcpu; @@ -4853,41 +4865,38 @@ static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *slot) } static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm, - struct kvm_memory_slot *slot, - const struct kvm_userspace_memory_region *mem, - enum kvm_mr_change change) + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) { - unsigned long npages = mem->memory_size >> PAGE_SHIFT; - if (change == KVM_MR_CREATE) { - unsigned long size = array_size(npages, sizeof(*slot->arch.rmap)); + unsigned long size = array_size(new->npages, sizeof(*new->arch.rmap)); if ((size >> PAGE_SHIFT) > totalram_pages()) return -ENOMEM; - slot->arch.rmap = vzalloc(size); - if (!slot->arch.rmap) + new->arch.rmap = vzalloc(size); + if (!new->arch.rmap) return -ENOMEM; + } else if (change != KVM_MR_DELETE) { + new->arch.rmap = old->arch.rmap; } return 0; } static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { - unsigned long npages = mem->memory_size >> PAGE_SHIFT; - /* - * If we are making a new memslot, it might make + * If we are creating or modifying a memslot, it might make * some address that was previously cached as emulated * MMIO be no longer emulated MMIO, so invalidate * all the caches of emulated MMIO translations. */ - if (npages) + if (change != KVM_MR_DELETE) atomic64_inc(&kvm->arch.mmio_update); /* @@ -5075,6 +5084,8 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) */ int kvmppc_switch_mmu_to_hpt(struct kvm *kvm) { + unsigned long lpcr, lpcr_mask; + if (nesting_enabled(kvm)) kvmhv_release_all_nested(kvm); kvmppc_rmap_reset(kvm); @@ -5084,8 +5095,13 @@ int kvmppc_switch_mmu_to_hpt(struct kvm *kvm) kvm->arch.radix = 0; spin_unlock(&kvm->mmu_lock); kvmppc_free_radix(kvm); - kvmppc_update_lpcr(kvm, LPCR_VPM1, - LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR); + + lpcr = LPCR_VPM1; + lpcr_mask = LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR; + if (cpu_has_feature(CPU_FTR_ARCH_31)) + lpcr_mask |= LPCR_HAIL; + kvmppc_update_lpcr(kvm, lpcr, lpcr_mask); + return 0; } @@ -5095,6 +5111,7 @@ int kvmppc_switch_mmu_to_hpt(struct kvm *kvm) */ int kvmppc_switch_mmu_to_radix(struct kvm *kvm) { + unsigned long lpcr, lpcr_mask; int err; err = kvmppc_init_vm_radix(kvm); @@ -5106,8 +5123,17 @@ int kvmppc_switch_mmu_to_radix(struct kvm *kvm) kvm->arch.radix = 1; spin_unlock(&kvm->mmu_lock); kvmppc_free_hpt(&kvm->arch.hpt); - kvmppc_update_lpcr(kvm, LPCR_UPRT | LPCR_GTSE | LPCR_HR, - LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR); + + lpcr = LPCR_UPRT | LPCR_GTSE | LPCR_HR; + lpcr_mask = LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR; + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + lpcr_mask |= LPCR_HAIL; + if (cpu_has_feature(CPU_FTR_HVMODE) && + (kvm->arch.host_lpcr & LPCR_HAIL)) + lpcr |= LPCR_HAIL; + } + kvmppc_update_lpcr(kvm, lpcr, lpcr_mask); + return 0; } @@ -5129,6 +5155,9 @@ void kvmppc_alloc_host_rm_ops(void) int cpu, core; int size; + if (cpu_has_feature(CPU_FTR_ARCH_300)) + return; + /* Not the first time here ? */ if (kvmppc_host_rm_ops_hv != NULL) return; @@ -5271,6 +5300,10 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm) kvm->arch.mmu_ready = 1; lpcr &= ~LPCR_VPM1; lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR; + if (cpu_has_feature(CPU_FTR_HVMODE) && + cpu_has_feature(CPU_FTR_ARCH_31) && + (kvm->arch.host_lpcr & LPCR_HAIL)) + lpcr |= LPCR_HAIL; ret = kvmppc_init_vm_radix(kvm); if (ret) { kvmppc_free_lpid(kvm->arch.lpid); @@ -5864,7 +5897,7 @@ static int kvmhv_svm_off(struct kvm *kvm) int mmu_was_ready; int srcu_idx; int ret = 0; - int i; + unsigned long i; if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) return ret; @@ -5886,11 +5919,12 @@ static int kvmhv_svm_off(struct kvm *kvm) for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { struct kvm_memory_slot *memslot; struct kvm_memslots *slots = __kvm_memslots(kvm, i); + int bkt; if (!slots) continue; - kvm_for_each_memslot(memslot, slots) { + kvm_for_each_memslot(memslot, bkt, slots) { kvmppc_uvmem_drop_pages(memslot, kvm, true); uv_unregister_mem_slot(kvm->arch.lpid, memslot->id); } @@ -6066,9 +6100,11 @@ static int kvmppc_book3s_init_hv(void) if (r) return r; - r = kvm_init_subcore_bitmap(); - if (r) - return r; + if (!cpu_has_feature(CPU_FTR_ARCH_300)) { + r = kvm_init_subcore_bitmap(); + if (r) + return r; + } /* * We need a way of accessing the XICS interrupt controller, diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index 70b7a8f971..7d6d91338c 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -649,6 +649,8 @@ void kvmppc_guest_entry_inject_int(struct kvm_vcpu *vcpu) int ext; unsigned long lpcr; + WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300)); + /* Insert EXTERNAL bit into LPCR at the MER bit position */ ext = (vcpu->arch.pending_exceptions >> BOOK3S_IRQPRIO_EXTERNAL) & 1; lpcr = mfspr(SPRN_LPCR); @@ -682,60 +684,23 @@ static void flush_guest_tlb(struct kvm *kvm) unsigned long rb, set; rb = PPC_BIT(52); /* IS = 2 */ - if (kvm_is_radix(kvm)) { - /* R=1 PRS=1 RIC=2 */ + for (set = 0; set < kvm->arch.tlb_sets; ++set) { + /* R=0 PRS=0 RIC=0 */ asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) - : : "r" (rb), "i" (1), "i" (1), "i" (2), + : : "r" (rb), "i" (0), "i" (0), "i" (0), "r" (0) : "memory"); - for (set = 1; set < kvm->arch.tlb_sets; ++set) { - rb += PPC_BIT(51); /* increment set number */ - /* R=1 PRS=1 RIC=0 */ - asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) - : : "r" (rb), "i" (1), "i" (1), "i" (0), - "r" (0) : "memory"); - } - asm volatile("ptesync": : :"memory"); - // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. - asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory"); - } else { - for (set = 0; set < kvm->arch.tlb_sets; ++set) { - /* R=0 PRS=0 RIC=0 */ - asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) - : : "r" (rb), "i" (0), "i" (0), "i" (0), - "r" (0) : "memory"); - rb += PPC_BIT(51); /* increment set number */ - } - asm volatile("ptesync": : :"memory"); - // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. - if (cpu_has_feature(CPU_FTR_ARCH_300)) - asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); + rb += PPC_BIT(51); /* increment set number */ } + asm volatile("ptesync": : :"memory"); } -void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu, - struct kvm_nested_guest *nested) +void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu) { - cpumask_t *need_tlb_flush; - - /* - * On POWER9, individual threads can come in here, but the - * TLB is shared between the 4 threads in a core, hence - * invalidating on one thread invalidates for all. - * Thus we make all 4 threads use the same bit. - */ - if (cpu_has_feature(CPU_FTR_ARCH_300)) - pcpu = cpu_first_tlb_thread_sibling(pcpu); - - if (nested) - need_tlb_flush = &nested->need_tlb_flush; - else - need_tlb_flush = &kvm->arch.need_tlb_flush; - - if (cpumask_test_cpu(pcpu, need_tlb_flush)) { + if (cpumask_test_cpu(pcpu, &kvm->arch.need_tlb_flush)) { flush_guest_tlb(kvm); /* Clear the bit after the TLB flush */ - cpumask_clear_cpu(pcpu, need_tlb_flush); + cpumask_clear_cpu(pcpu, &kvm->arch.need_tlb_flush); } } EXPORT_SYMBOL_GPL(kvmppc_check_need_tlb_flush); diff --git a/arch/powerpc/kvm/book3s_hv_hmi.c b/arch/powerpc/kvm/book3s_hv_hmi.c index 9af6604763..1ec50c6967 100644 --- a/arch/powerpc/kvm/book3s_hv_hmi.c +++ b/arch/powerpc/kvm/book3s_hv_hmi.c @@ -20,10 +20,15 @@ void wait_for_subcore_guest_exit(void) /* * NULL bitmap pointer indicates that KVM module hasn't - * been loaded yet and hence no guests are running. + * been loaded yet and hence no guests are running, or running + * on POWER9 or newer CPU. + * * If no KVM is in use, no need to co-ordinate among threads * as all of them will always be in host and no one is going * to modify TB other than the opal hmi handler. + * + * POWER9 and newer don't need this synchronisation. + * * Hence, just return from here. */ if (!local_paca->sibling_subcore_state) diff --git a/arch/powerpc/kvm/book3s_hv_interrupts.S b/arch/powerpc/kvm/book3s_hv_interrupts.S index 4444f83cb1..59d89e4b15 100644 --- a/arch/powerpc/kvm/book3s_hv_interrupts.S +++ b/arch/powerpc/kvm/book3s_hv_interrupts.S @@ -104,7 +104,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) mtlr r0 blr -_GLOBAL(kvmhv_save_host_pmu) +/* + * void kvmhv_save_host_pmu(void) + */ +kvmhv_save_host_pmu: BEGIN_FTR_SECTION /* Work around P8 PMAE bug */ li r3, -1 @@ -138,14 +141,6 @@ BEGIN_FTR_SECTION std r8, HSTATE_MMCR2(r13) std r9, HSTATE_SIER(r13) END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) -BEGIN_FTR_SECTION - mfspr r5, SPRN_MMCR3 - mfspr r6, SPRN_SIER2 - mfspr r7, SPRN_SIER3 - std r5, HSTATE_MMCR3(r13) - std r6, HSTATE_SIER2(r13) - std r7, HSTATE_SIER3(r13) -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31) mfspr r3, SPRN_PMC1 mfspr r5, SPRN_PMC2 mfspr r6, SPRN_PMC3 diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c index 6c4e0e9310..9d373f8963 100644 --- a/arch/powerpc/kvm/book3s_hv_nested.c +++ b/arch/powerpc/kvm/book3s_hv_nested.c @@ -358,6 +358,7 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) /* convert TB values/offsets to host (L0) values */ hdec_exp = l2_hv.hdec_expiry - vc->tb_offset; vc->tb_offset += l2_hv.tb_offset; + vcpu->arch.dec_expires += l2_hv.tb_offset; /* set L1 state to L2 state */ vcpu->arch.nested = l2; @@ -374,11 +375,6 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) vcpu->arch.ret = RESUME_GUEST; vcpu->arch.trap = 0; do { - if (mftb() >= hdec_exp) { - vcpu->arch.trap = BOOK3S_INTERRUPT_HV_DECREMENTER; - r = RESUME_HOST; - break; - } r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr); } while (is_kvmppc_resume_guest(r)); @@ -399,6 +395,8 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) if (l2_regs.msr & MSR_TS_MASK) vcpu->arch.shregs.msr |= MSR_TS_S; vc->tb_offset = saved_l1_hv.tb_offset; + /* XXX: is this always the same delta as saved_l1_hv.tb_offset? */ + vcpu->arch.dec_expires -= l2_hv.tb_offset; restore_hv_regs(vcpu, &saved_l1_hv); vcpu->arch.purr += delta_purr; vcpu->arch.spurr += delta_spurr; @@ -749,7 +747,7 @@ void kvmhv_release_all_nested(struct kvm *kvm) struct kvm_nested_guest *gp; struct kvm_nested_guest *freelist = NULL; struct kvm_memory_slot *memslot; - int srcu_idx; + int srcu_idx, bkt; spin_lock(&kvm->mmu_lock); for (i = 0; i <= kvm->arch.max_nested_lpid; i++) { @@ -770,7 +768,7 @@ void kvmhv_release_all_nested(struct kvm *kvm) } srcu_idx = srcu_read_lock(&kvm->srcu); - kvm_for_each_memslot(memslot, kvm_memslots(kvm)) + kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm)) kvmhv_free_memslot_nest_rmap(memslot); srcu_read_unlock(&kvm->srcu, srcu_idx); } diff --git a/arch/powerpc/kvm/book3s_hv_p9_entry.c b/arch/powerpc/kvm/book3s_hv_p9_entry.c index 961b3d7048..a28e5b3daa 100644 --- a/arch/powerpc/kvm/book3s_hv_p9_entry.c +++ b/arch/powerpc/kvm/book3s_hv_p9_entry.c @@ -4,8 +4,439 @@ #include #include #include +#include #include +#include "book3s_hv.h" + +static void freeze_pmu(unsigned long mmcr0, unsigned long mmcra) +{ + if (!(mmcr0 & MMCR0_FC)) + goto do_freeze; + if (mmcra & MMCRA_SAMPLE_ENABLE) + goto do_freeze; + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + if (!(mmcr0 & MMCR0_PMCCEXT)) + goto do_freeze; + if (!(mmcra & MMCRA_BHRB_DISABLE)) + goto do_freeze; + } + return; + +do_freeze: + mmcr0 = MMCR0_FC; + mmcra = 0; + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + mmcr0 |= MMCR0_PMCCEXT; + mmcra = MMCRA_BHRB_DISABLE; + } + + mtspr(SPRN_MMCR0, mmcr0); + mtspr(SPRN_MMCRA, mmcra); + isync(); +} + +void switch_pmu_to_guest(struct kvm_vcpu *vcpu, + struct p9_host_os_sprs *host_os_sprs) +{ + struct lppaca *lp; + int load_pmu = 1; + + lp = vcpu->arch.vpa.pinned_addr; + if (lp) + load_pmu = lp->pmcregs_in_use; + + /* Save host */ + if (ppc_get_pmu_inuse()) { + /* + * It might be better to put PMU handling (at least for the + * host) in the perf subsystem because it knows more about what + * is being used. + */ + + /* POWER9, POWER10 do not implement HPMC or SPMC */ + + host_os_sprs->mmcr0 = mfspr(SPRN_MMCR0); + host_os_sprs->mmcra = mfspr(SPRN_MMCRA); + + freeze_pmu(host_os_sprs->mmcr0, host_os_sprs->mmcra); + + host_os_sprs->pmc1 = mfspr(SPRN_PMC1); + host_os_sprs->pmc2 = mfspr(SPRN_PMC2); + host_os_sprs->pmc3 = mfspr(SPRN_PMC3); + host_os_sprs->pmc4 = mfspr(SPRN_PMC4); + host_os_sprs->pmc5 = mfspr(SPRN_PMC5); + host_os_sprs->pmc6 = mfspr(SPRN_PMC6); + host_os_sprs->mmcr1 = mfspr(SPRN_MMCR1); + host_os_sprs->mmcr2 = mfspr(SPRN_MMCR2); + host_os_sprs->sdar = mfspr(SPRN_SDAR); + host_os_sprs->siar = mfspr(SPRN_SIAR); + host_os_sprs->sier1 = mfspr(SPRN_SIER); + + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + host_os_sprs->mmcr3 = mfspr(SPRN_MMCR3); + host_os_sprs->sier2 = mfspr(SPRN_SIER2); + host_os_sprs->sier3 = mfspr(SPRN_SIER3); + } + } + +#ifdef CONFIG_PPC_PSERIES + /* After saving PMU, before loading guest PMU, flip pmcregs_in_use */ + if (kvmhv_on_pseries()) { + barrier(); + get_lppaca()->pmcregs_in_use = load_pmu; + barrier(); + } +#endif + + /* + * Load guest. If the VPA said the PMCs are not in use but the guest + * tried to access them anyway, HFSCR[PM] will be set by the HFAC + * fault so we can make forward progress. + */ + if (load_pmu || (vcpu->arch.hfscr & HFSCR_PM)) { + mtspr(SPRN_PMC1, vcpu->arch.pmc[0]); + mtspr(SPRN_PMC2, vcpu->arch.pmc[1]); + mtspr(SPRN_PMC3, vcpu->arch.pmc[2]); + mtspr(SPRN_PMC4, vcpu->arch.pmc[3]); + mtspr(SPRN_PMC5, vcpu->arch.pmc[4]); + mtspr(SPRN_PMC6, vcpu->arch.pmc[5]); + mtspr(SPRN_MMCR1, vcpu->arch.mmcr[1]); + mtspr(SPRN_MMCR2, vcpu->arch.mmcr[2]); + mtspr(SPRN_SDAR, vcpu->arch.sdar); + mtspr(SPRN_SIAR, vcpu->arch.siar); + mtspr(SPRN_SIER, vcpu->arch.sier[0]); + + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + mtspr(SPRN_MMCR3, vcpu->arch.mmcr[3]); + mtspr(SPRN_SIER2, vcpu->arch.sier[1]); + mtspr(SPRN_SIER3, vcpu->arch.sier[2]); + } + + /* Set MMCRA then MMCR0 last */ + mtspr(SPRN_MMCRA, vcpu->arch.mmcra); + mtspr(SPRN_MMCR0, vcpu->arch.mmcr[0]); + /* No isync necessary because we're starting counters */ + + if (!vcpu->arch.nested && + (vcpu->arch.hfscr_permitted & HFSCR_PM)) + vcpu->arch.hfscr |= HFSCR_PM; + } +} +EXPORT_SYMBOL_GPL(switch_pmu_to_guest); + +void switch_pmu_to_host(struct kvm_vcpu *vcpu, + struct p9_host_os_sprs *host_os_sprs) +{ + struct lppaca *lp; + int save_pmu = 1; + + lp = vcpu->arch.vpa.pinned_addr; + if (lp) + save_pmu = lp->pmcregs_in_use; + if (IS_ENABLED(CONFIG_KVM_BOOK3S_HV_NESTED_PMU_WORKAROUND)) { + /* + * Save pmu if this guest is capable of running nested guests. + * This is option is for old L1s that do not set their + * lppaca->pmcregs_in_use properly when entering their L2. + */ + save_pmu |= nesting_enabled(vcpu->kvm); + } + + if (save_pmu) { + vcpu->arch.mmcr[0] = mfspr(SPRN_MMCR0); + vcpu->arch.mmcra = mfspr(SPRN_MMCRA); + + freeze_pmu(vcpu->arch.mmcr[0], vcpu->arch.mmcra); + + vcpu->arch.pmc[0] = mfspr(SPRN_PMC1); + vcpu->arch.pmc[1] = mfspr(SPRN_PMC2); + vcpu->arch.pmc[2] = mfspr(SPRN_PMC3); + vcpu->arch.pmc[3] = mfspr(SPRN_PMC4); + vcpu->arch.pmc[4] = mfspr(SPRN_PMC5); + vcpu->arch.pmc[5] = mfspr(SPRN_PMC6); + vcpu->arch.mmcr[1] = mfspr(SPRN_MMCR1); + vcpu->arch.mmcr[2] = mfspr(SPRN_MMCR2); + vcpu->arch.sdar = mfspr(SPRN_SDAR); + vcpu->arch.siar = mfspr(SPRN_SIAR); + vcpu->arch.sier[0] = mfspr(SPRN_SIER); + + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + vcpu->arch.mmcr[3] = mfspr(SPRN_MMCR3); + vcpu->arch.sier[1] = mfspr(SPRN_SIER2); + vcpu->arch.sier[2] = mfspr(SPRN_SIER3); + } + + } else if (vcpu->arch.hfscr & HFSCR_PM) { + /* + * The guest accessed PMC SPRs without specifying they should + * be preserved, or it cleared pmcregs_in_use after the last + * access. Just ensure they are frozen. + */ + freeze_pmu(mfspr(SPRN_MMCR0), mfspr(SPRN_MMCRA)); + + /* + * Demand-fault PMU register access in the guest. + * + * This is used to grab the guest's VPA pmcregs_in_use value + * and reflect it into the host's VPA in the case of a nested + * hypervisor. + * + * It also avoids having to zero-out SPRs after each guest + * exit to avoid side-channels when. + * + * This is cleared here when we exit the guest, so later HFSCR + * interrupt handling can add it back to run the guest with + * PM enabled next time. + */ + if (!vcpu->arch.nested) + vcpu->arch.hfscr &= ~HFSCR_PM; + } /* otherwise the PMU should still be frozen */ + +#ifdef CONFIG_PPC_PSERIES + if (kvmhv_on_pseries()) { + barrier(); + get_lppaca()->pmcregs_in_use = ppc_get_pmu_inuse(); + barrier(); + } +#endif + + if (ppc_get_pmu_inuse()) { + mtspr(SPRN_PMC1, host_os_sprs->pmc1); + mtspr(SPRN_PMC2, host_os_sprs->pmc2); + mtspr(SPRN_PMC3, host_os_sprs->pmc3); + mtspr(SPRN_PMC4, host_os_sprs->pmc4); + mtspr(SPRN_PMC5, host_os_sprs->pmc5); + mtspr(SPRN_PMC6, host_os_sprs->pmc6); + mtspr(SPRN_MMCR1, host_os_sprs->mmcr1); + mtspr(SPRN_MMCR2, host_os_sprs->mmcr2); + mtspr(SPRN_SDAR, host_os_sprs->sdar); + mtspr(SPRN_SIAR, host_os_sprs->siar); + mtspr(SPRN_SIER, host_os_sprs->sier1); + + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + mtspr(SPRN_MMCR3, host_os_sprs->mmcr3); + mtspr(SPRN_SIER2, host_os_sprs->sier2); + mtspr(SPRN_SIER3, host_os_sprs->sier3); + } + + /* Set MMCRA then MMCR0 last */ + mtspr(SPRN_MMCRA, host_os_sprs->mmcra); + mtspr(SPRN_MMCR0, host_os_sprs->mmcr0); + isync(); + } +} +EXPORT_SYMBOL_GPL(switch_pmu_to_host); + +static void load_spr_state(struct kvm_vcpu *vcpu, + struct p9_host_os_sprs *host_os_sprs) +{ + /* TAR is very fast */ + mtspr(SPRN_TAR, vcpu->arch.tar); + +#ifdef CONFIG_ALTIVEC + if (cpu_has_feature(CPU_FTR_ALTIVEC) && + current->thread.vrsave != vcpu->arch.vrsave) + mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); +#endif + + if (vcpu->arch.hfscr & HFSCR_EBB) { + if (current->thread.ebbhr != vcpu->arch.ebbhr) + mtspr(SPRN_EBBHR, vcpu->arch.ebbhr); + if (current->thread.ebbrr != vcpu->arch.ebbrr) + mtspr(SPRN_EBBRR, vcpu->arch.ebbrr); + if (current->thread.bescr != vcpu->arch.bescr) + mtspr(SPRN_BESCR, vcpu->arch.bescr); + } + + if (cpu_has_feature(CPU_FTR_P9_TIDR) && + current->thread.tidr != vcpu->arch.tid) + mtspr(SPRN_TIDR, vcpu->arch.tid); + if (host_os_sprs->iamr != vcpu->arch.iamr) + mtspr(SPRN_IAMR, vcpu->arch.iamr); + if (host_os_sprs->amr != vcpu->arch.amr) + mtspr(SPRN_AMR, vcpu->arch.amr); + if (vcpu->arch.uamor != 0) + mtspr(SPRN_UAMOR, vcpu->arch.uamor); + if (current->thread.fscr != vcpu->arch.fscr) + mtspr(SPRN_FSCR, vcpu->arch.fscr); + if (current->thread.dscr != vcpu->arch.dscr) + mtspr(SPRN_DSCR, vcpu->arch.dscr); + if (vcpu->arch.pspb != 0) + mtspr(SPRN_PSPB, vcpu->arch.pspb); + + /* + * DAR, DSISR, and for nested HV, SPRGs must be set with MSR[RI] + * clear (or hstate set appropriately to catch those registers + * being clobbered if we take a MCE or SRESET), so those are done + * later. + */ + + if (!(vcpu->arch.ctrl & 1)) + mtspr(SPRN_CTRLT, 0); +} + +static void store_spr_state(struct kvm_vcpu *vcpu) +{ + vcpu->arch.tar = mfspr(SPRN_TAR); + +#ifdef CONFIG_ALTIVEC + if (cpu_has_feature(CPU_FTR_ALTIVEC)) + vcpu->arch.vrsave = mfspr(SPRN_VRSAVE); +#endif + + if (vcpu->arch.hfscr & HFSCR_EBB) { + vcpu->arch.ebbhr = mfspr(SPRN_EBBHR); + vcpu->arch.ebbrr = mfspr(SPRN_EBBRR); + vcpu->arch.bescr = mfspr(SPRN_BESCR); + } + + if (cpu_has_feature(CPU_FTR_P9_TIDR)) + vcpu->arch.tid = mfspr(SPRN_TIDR); + vcpu->arch.iamr = mfspr(SPRN_IAMR); + vcpu->arch.amr = mfspr(SPRN_AMR); + vcpu->arch.uamor = mfspr(SPRN_UAMOR); + vcpu->arch.fscr = mfspr(SPRN_FSCR); + vcpu->arch.dscr = mfspr(SPRN_DSCR); + vcpu->arch.pspb = mfspr(SPRN_PSPB); + + vcpu->arch.ctrl = mfspr(SPRN_CTRLF); +} + +/* Returns true if current MSR and/or guest MSR may have changed */ +bool load_vcpu_state(struct kvm_vcpu *vcpu, + struct p9_host_os_sprs *host_os_sprs) +{ + bool ret = false; + +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM + if (cpu_has_feature(CPU_FTR_TM) || + cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { + unsigned long guest_msr = vcpu->arch.shregs.msr; + if (MSR_TM_ACTIVE(guest_msr)) { + kvmppc_restore_tm_hv(vcpu, guest_msr, true); + ret = true; + } else if (vcpu->arch.hfscr & HFSCR_TM) { + mtspr(SPRN_TEXASR, vcpu->arch.texasr); + mtspr(SPRN_TFHAR, vcpu->arch.tfhar); + mtspr(SPRN_TFIAR, vcpu->arch.tfiar); + } + } +#endif + + load_spr_state(vcpu, host_os_sprs); + + load_fp_state(&vcpu->arch.fp); +#ifdef CONFIG_ALTIVEC + load_vr_state(&vcpu->arch.vr); +#endif + + return ret; +} +EXPORT_SYMBOL_GPL(load_vcpu_state); + +void store_vcpu_state(struct kvm_vcpu *vcpu) +{ + store_spr_state(vcpu); + + store_fp_state(&vcpu->arch.fp); +#ifdef CONFIG_ALTIVEC + store_vr_state(&vcpu->arch.vr); +#endif + +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM + if (cpu_has_feature(CPU_FTR_TM) || + cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { + unsigned long guest_msr = vcpu->arch.shregs.msr; + if (MSR_TM_ACTIVE(guest_msr)) { + kvmppc_save_tm_hv(vcpu, guest_msr, true); + } else if (vcpu->arch.hfscr & HFSCR_TM) { + vcpu->arch.texasr = mfspr(SPRN_TEXASR); + vcpu->arch.tfhar = mfspr(SPRN_TFHAR); + vcpu->arch.tfiar = mfspr(SPRN_TFIAR); + + if (!vcpu->arch.nested) { + vcpu->arch.load_tm++; /* see load_ebb comment */ + if (!vcpu->arch.load_tm) + vcpu->arch.hfscr &= ~HFSCR_TM; + } + } + } +#endif +} +EXPORT_SYMBOL_GPL(store_vcpu_state); + +void save_p9_host_os_sprs(struct p9_host_os_sprs *host_os_sprs) +{ + host_os_sprs->iamr = mfspr(SPRN_IAMR); + host_os_sprs->amr = mfspr(SPRN_AMR); +} +EXPORT_SYMBOL_GPL(save_p9_host_os_sprs); + +/* vcpu guest regs must already be saved */ +void restore_p9_host_os_sprs(struct kvm_vcpu *vcpu, + struct p9_host_os_sprs *host_os_sprs) +{ + /* + * current->thread.xxx registers must all be restored to host + * values before a potential context switch, othrewise the context + * switch itself will overwrite current->thread.xxx with the values + * from the guest SPRs. + */ + + mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso); + + if (cpu_has_feature(CPU_FTR_P9_TIDR) && + current->thread.tidr != vcpu->arch.tid) + mtspr(SPRN_TIDR, current->thread.tidr); + if (host_os_sprs->iamr != vcpu->arch.iamr) + mtspr(SPRN_IAMR, host_os_sprs->iamr); + if (vcpu->arch.uamor != 0) + mtspr(SPRN_UAMOR, 0); + if (host_os_sprs->amr != vcpu->arch.amr) + mtspr(SPRN_AMR, host_os_sprs->amr); + if (current->thread.fscr != vcpu->arch.fscr) + mtspr(SPRN_FSCR, current->thread.fscr); + if (current->thread.dscr != vcpu->arch.dscr) + mtspr(SPRN_DSCR, current->thread.dscr); + if (vcpu->arch.pspb != 0) + mtspr(SPRN_PSPB, 0); + + /* Save guest CTRL register, set runlatch to 1 */ + if (!(vcpu->arch.ctrl & 1)) + mtspr(SPRN_CTRLT, 1); + +#ifdef CONFIG_ALTIVEC + if (cpu_has_feature(CPU_FTR_ALTIVEC) && + vcpu->arch.vrsave != current->thread.vrsave) + mtspr(SPRN_VRSAVE, current->thread.vrsave); +#endif + if (vcpu->arch.hfscr & HFSCR_EBB) { + if (vcpu->arch.bescr != current->thread.bescr) + mtspr(SPRN_BESCR, current->thread.bescr); + if (vcpu->arch.ebbhr != current->thread.ebbhr) + mtspr(SPRN_EBBHR, current->thread.ebbhr); + if (vcpu->arch.ebbrr != current->thread.ebbrr) + mtspr(SPRN_EBBRR, current->thread.ebbrr); + + if (!vcpu->arch.nested) { + /* + * This is like load_fp in context switching, turn off + * the facility after it wraps the u8 to try avoiding + * saving and restoring the registers each partition + * switch. + */ + vcpu->arch.load_ebb++; + if (!vcpu->arch.load_ebb) + vcpu->arch.hfscr &= ~HFSCR_EBB; + } + } + + if (vcpu->arch.tar != current->thread.tar) + mtspr(SPRN_TAR, current->thread.tar); +} +EXPORT_SYMBOL_GPL(restore_p9_host_os_sprs); + #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING static void __start_timing(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next) { @@ -56,10 +487,22 @@ static void __accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator #define accumulate_time(vcpu, next) do {} while (0) #endif -static inline void mfslb(unsigned int idx, u64 *slbee, u64 *slbev) +static inline u64 mfslbv(unsigned int idx) { - asm volatile("slbmfev %0,%1" : "=r" (*slbev) : "r" (idx)); - asm volatile("slbmfee %0,%1" : "=r" (*slbee) : "r" (idx)); + u64 slbev; + + asm volatile("slbmfev %0,%1" : "=r" (slbev) : "r" (idx)); + + return slbev; +} + +static inline u64 mfslbe(unsigned int idx) +{ + u64 slbee; + + asm volatile("slbmfee %0,%1" : "=r" (slbee) : "r" (idx)); + + return slbee; } static inline void mtslb(u64 slbee, u64 slbev) @@ -100,17 +543,19 @@ static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u6 lpid = nested ? nested->shadow_lpid : kvm->arch.lpid; /* - * All the isync()s are overkill but trivially follow the ISA - * requirements. Some can likely be replaced with justification - * comment for why they are not needed. + * Prior memory accesses to host PID Q3 must be completed before we + * start switching, and stores must be drained to avoid not-my-LPAR + * logic (see switch_mmu_to_host). */ + asm volatile("hwsync" ::: "memory"); isync(); mtspr(SPRN_LPID, lpid); - isync(); mtspr(SPRN_LPCR, lpcr); - isync(); mtspr(SPRN_PID, vcpu->arch.pid); - isync(); + /* + * isync not required here because we are HRFID'ing to guest before + * any guest context access, which is context synchronising. + */ } static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr) @@ -120,25 +565,41 @@ static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpid = kvm->arch.lpid; + /* + * See switch_mmu_to_guest_radix. ptesync should not be required here + * even if the host is in HPT mode because speculative accesses would + * not cause RC updates (we are in real mode). + */ + asm volatile("hwsync" ::: "memory"); + isync(); mtspr(SPRN_LPID, lpid); mtspr(SPRN_LPCR, lpcr); mtspr(SPRN_PID, vcpu->arch.pid); for (i = 0; i < vcpu->arch.slb_max; i++) mtslb(vcpu->arch.slb[i].orige, vcpu->arch.slb[i].origv); - - isync(); + /* + * isync not required here, see switch_mmu_to_guest_radix. + */ } static void switch_mmu_to_host(struct kvm *kvm, u32 pid) { + /* + * The guest has exited, so guest MMU context is no longer being + * non-speculatively accessed, but a hwsync is needed before the + * mtLPIDR / mtPIDR switch, in order to ensure all stores are drained, + * so the not-my-LPAR tlbie logic does not overlook them. + */ + asm volatile("hwsync" ::: "memory"); isync(); mtspr(SPRN_PID, pid); - isync(); mtspr(SPRN_LPID, kvm->arch.host_lpid); - isync(); mtspr(SPRN_LPCR, kvm->arch.host_lpcr); - isync(); + /* + * isync is not required after the switch, because mtmsrd with L=0 + * is performed after this switch, which is context synchronising. + */ if (!radix_enabled()) slb_restore_bolted_realmode(); @@ -171,8 +632,10 @@ static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu) */ for (i = 0; i < vcpu->arch.slb_nr; i++) { u64 slbee, slbev; - mfslb(i, &slbee, &slbev); + + slbee = mfslbe(i); if (slbee & SLB_ESID_V) { + slbev = mfslbv(i); vcpu->arch.slb[nr].orige = slbee | i; vcpu->arch.slb[nr].origv = slbev; nr++; @@ -183,15 +646,128 @@ static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu) } } -int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr) +static void flush_guest_tlb(struct kvm *kvm) { + unsigned long rb, set; + + rb = PPC_BIT(52); /* IS = 2 */ + if (kvm_is_radix(kvm)) { + /* R=1 PRS=1 RIC=2 */ + asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) + : : "r" (rb), "i" (1), "i" (1), "i" (2), + "r" (0) : "memory"); + for (set = 1; set < kvm->arch.tlb_sets; ++set) { + rb += PPC_BIT(51); /* increment set number */ + /* R=1 PRS=1 RIC=0 */ + asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) + : : "r" (rb), "i" (1), "i" (1), "i" (0), + "r" (0) : "memory"); + } + asm volatile("ptesync": : :"memory"); + // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. + asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory"); + } else { + for (set = 0; set < kvm->arch.tlb_sets; ++set) { + /* R=0 PRS=0 RIC=0 */ + asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) + : : "r" (rb), "i" (0), "i" (0), "i" (0), + "r" (0) : "memory"); + rb += PPC_BIT(51); /* increment set number */ + } + asm volatile("ptesync": : :"memory"); + // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now. + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); + } +} + +static void check_need_tlb_flush(struct kvm *kvm, int pcpu, + struct kvm_nested_guest *nested) +{ + cpumask_t *need_tlb_flush; + bool all_set = true; + int i; + + if (nested) + need_tlb_flush = &nested->need_tlb_flush; + else + need_tlb_flush = &kvm->arch.need_tlb_flush; + + if (likely(!cpumask_test_cpu(pcpu, need_tlb_flush))) + return; + + /* + * Individual threads can come in here, but the TLB is shared between + * the 4 threads in a core, hence invalidating on one thread + * invalidates for all, so only invalidate the first time (if all bits + * were set. The others must still execute a ptesync. + * + * If a race occurs and two threads do the TLB flush, that is not a + * problem, just sub-optimal. + */ + for (i = cpu_first_tlb_thread_sibling(pcpu); + i <= cpu_last_tlb_thread_sibling(pcpu); + i += cpu_tlb_thread_sibling_step()) { + if (!cpumask_test_cpu(i, need_tlb_flush)) { + all_set = false; + break; + } + } + if (all_set) + flush_guest_tlb(kvm); + else + asm volatile("ptesync" ::: "memory"); + + /* Clear the bit after the TLB flush */ + cpumask_clear_cpu(pcpu, need_tlb_flush); +} + +unsigned long kvmppc_msr_hard_disable_set_facilities(struct kvm_vcpu *vcpu, unsigned long msr) +{ + unsigned long msr_needed = 0; + + msr &= ~MSR_EE; + + /* MSR bits may have been cleared by context switch so must recheck */ + if (IS_ENABLED(CONFIG_PPC_FPU)) + msr_needed |= MSR_FP; + if (cpu_has_feature(CPU_FTR_ALTIVEC)) + msr_needed |= MSR_VEC; + if (cpu_has_feature(CPU_FTR_VSX)) + msr_needed |= MSR_VSX; + if ((cpu_has_feature(CPU_FTR_TM) || + cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) && + (vcpu->arch.hfscr & HFSCR_TM)) + msr_needed |= MSR_TM; + + /* + * This could be combined with MSR[RI] clearing, but that expands + * the unrecoverable window. It would be better to cover unrecoverable + * with KVM bad interrupt handling rather than use MSR[RI] at all. + * + * Much more difficult and less worthwhile to combine with IR/DR + * disable. + */ + if ((msr & msr_needed) != msr_needed) { + msr |= msr_needed; + __mtmsrd(msr, 0); + } else { + __hard_irq_disable(); + } + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + + return msr; +} +EXPORT_SYMBOL_GPL(kvmppc_msr_hard_disable_set_facilities); + +int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb) +{ + struct p9_host_os_sprs host_os_sprs; struct kvm *kvm = vcpu->kvm; struct kvm_nested_guest *nested = vcpu->arch.nested; struct kvmppc_vcore *vc = vcpu->arch.vcore; - s64 hdec; - u64 tb, purr, spurr; + s64 hdec, dec; + u64 purr, spurr; u64 *exsave; - bool ri_set; int trap; unsigned long msr; unsigned long host_hfscr; @@ -199,11 +775,13 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc unsigned long host_dawr0; unsigned long host_dawrx0; unsigned long host_psscr; + unsigned long host_hpsscr; unsigned long host_pidr; unsigned long host_dawr1; unsigned long host_dawrx1; + unsigned long dpdes; - hdec = time_limit - mftb(); + hdec = time_limit - *tb; if (hdec < 0) return BOOK3S_INTERRUPT_HV_DECREMENTER; @@ -214,51 +792,84 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc vcpu->arch.ceded = 0; - if (vc->tb_offset) { - u64 new_tb = mftb() + vc->tb_offset; - mtspr(SPRN_TBU40, new_tb); - tb = mftb(); - if ((tb & 0xffffff) < (new_tb & 0xffffff)) - mtspr(SPRN_TBU40, new_tb + 0x1000000); - vc->tb_offset_applied = vc->tb_offset; - } - - msr = mfmsr(); + /* Save MSR for restore, with EE clear. */ + msr = mfmsr() & ~MSR_EE; host_hfscr = mfspr(SPRN_HFSCR); host_ciabr = mfspr(SPRN_CIABR); - host_dawr0 = mfspr(SPRN_DAWR0); - host_dawrx0 = mfspr(SPRN_DAWRX0); - host_psscr = mfspr(SPRN_PSSCR); + host_psscr = mfspr(SPRN_PSSCR_PR); + if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) + host_hpsscr = mfspr(SPRN_PSSCR); host_pidr = mfspr(SPRN_PID); - if (cpu_has_feature(CPU_FTR_DAWR1)) { - host_dawr1 = mfspr(SPRN_DAWR1); - host_dawrx1 = mfspr(SPRN_DAWRX1); - } - if (vc->pcr) - mtspr(SPRN_PCR, vc->pcr | PCR_MASK); - mtspr(SPRN_DPDES, vc->dpdes); - mtspr(SPRN_VTB, vc->vtb); + if (dawr_enabled()) { + host_dawr0 = mfspr(SPRN_DAWR0); + host_dawrx0 = mfspr(SPRN_DAWRX0); + if (cpu_has_feature(CPU_FTR_DAWR1)) { + host_dawr1 = mfspr(SPRN_DAWR1); + host_dawrx1 = mfspr(SPRN_DAWRX1); + } + } local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR); local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR); + + save_p9_host_os_sprs(&host_os_sprs); + + msr = kvmppc_msr_hard_disable_set_facilities(vcpu, msr); + if (lazy_irq_pending()) { + trap = 0; + goto out; + } + + if (unlikely(load_vcpu_state(vcpu, &host_os_sprs))) + msr = mfmsr(); /* MSR may have been updated */ + + if (vc->tb_offset) { + u64 new_tb = *tb + vc->tb_offset; + mtspr(SPRN_TBU40, new_tb); + if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { + new_tb += 0x1000000; + mtspr(SPRN_TBU40, new_tb); + } + *tb = new_tb; + vc->tb_offset_applied = vc->tb_offset; + } + + mtspr(SPRN_VTB, vc->vtb); mtspr(SPRN_PURR, vcpu->arch.purr); mtspr(SPRN_SPURR, vcpu->arch.spurr); + if (vc->pcr) + mtspr(SPRN_PCR, vc->pcr | PCR_MASK); + if (vcpu->arch.doorbell_request) { + vcpu->arch.doorbell_request = 0; + mtspr(SPRN_DPDES, 1); + } + if (dawr_enabled()) { - mtspr(SPRN_DAWR0, vcpu->arch.dawr0); - mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0); + if (vcpu->arch.dawr0 != host_dawr0) + mtspr(SPRN_DAWR0, vcpu->arch.dawr0); + if (vcpu->arch.dawrx0 != host_dawrx0) + mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0); if (cpu_has_feature(CPU_FTR_DAWR1)) { - mtspr(SPRN_DAWR1, vcpu->arch.dawr1); - mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1); + if (vcpu->arch.dawr1 != host_dawr1) + mtspr(SPRN_DAWR1, vcpu->arch.dawr1); + if (vcpu->arch.dawrx1 != host_dawrx1) + mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1); } } - mtspr(SPRN_CIABR, vcpu->arch.ciabr); - mtspr(SPRN_IC, vcpu->arch.ic); + if (vcpu->arch.ciabr != host_ciabr) + mtspr(SPRN_CIABR, vcpu->arch.ciabr); - mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC | - (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); + + if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { + mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC | + (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); + } else { + if (vcpu->arch.psscr != host_psscr) + mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr); + } mtspr(SPRN_HFSCR, vcpu->arch.hfscr); @@ -276,18 +887,34 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc * HDSI which should correctly update the HDSISR the second time HDSI * entry. * - * Just do this on all p9 processors for now. + * The "radix prefetch bug" test can be used to test for this bug, as + * it also exists fo DD2.1 and below. */ - mtspr(SPRN_HDSISR, HDSISR_CANARY); + if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) + mtspr(SPRN_HDSISR, HDSISR_CANARY); mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0); mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1); mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2); mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3); - mtspr(SPRN_AMOR, ~0UL); + /* + * It might be preferable to load_vcpu_state here, in order to get the + * GPR/FP register loads executing in parallel with the previous mtSPR + * instructions, but for now that can't be done because the TM handling + * in load_vcpu_state can change some SPRs and vcpu state (nip, msr). + * But TM could be split out if this would be a significant benefit. + */ - local_paca->kvm_hstate.in_guest = KVM_GUEST_MODE_HV_P9; + /* + * MSR[RI] does not need to be cleared (and is not, for radix guests + * with no prefetch bug), because in_guest is set. If we take a SRESET + * or MCE with in_guest set but still in HV mode, then + * kvmppc_p9_bad_interrupt handles the interrupt, which effectively + * clears MSR[RI] and doesn't return. + */ + WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_HV_P9); + barrier(); /* Open in_guest critical section */ /* * Hash host, hash guest, or radix guest with prefetch bug, all have @@ -299,17 +926,13 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc save_clear_host_mmu(kvm); - if (kvm_is_radix(kvm)) { + if (kvm_is_radix(kvm)) switch_mmu_to_guest_radix(kvm, vcpu, lpcr); - if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) - __mtmsrd(0, 1); /* clear RI */ - - } else { + else switch_mmu_to_guest_hpt(kvm, vcpu, lpcr); - } /* TLBIEL uses LPID=LPIDR, so run this after setting guest LPID */ - kvmppc_check_need_tlb_flush(kvm, vc->pcpu, nested); + check_need_tlb_flush(kvm, vc->pcpu, nested); /* * P9 suppresses the HDEC exception when LPCR[HDICE] = 0, @@ -317,6 +940,8 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc */ mtspr(SPRN_HDEC, hdec); + mtspr(SPRN_DEC, vcpu->arch.dec_expires - *tb); + #ifdef CONFIG_PPC_TRANSACTIONAL_MEM tm_return_to_guest: #endif @@ -327,7 +952,9 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc accumulate_time(vcpu, &vcpu->arch.guest_time); + switch_pmu_to_guest(vcpu, &host_os_sprs); kvmppc_p9_enter_guest(vcpu); + switch_pmu_to_host(vcpu, &host_os_sprs); accumulate_time(vcpu, &vcpu->arch.rm_intr); @@ -340,36 +967,27 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc /* 0x2 bit for HSRR is only used by PR and P7/8 HV paths, clear it */ trap = local_paca->kvm_hstate.scratch0 & ~0x2; - /* HSRR interrupts leave MSR[RI] unchanged, SRR interrupts clear it. */ - ri_set = false; - if (likely(trap > BOOK3S_INTERRUPT_MACHINE_CHECK)) { - if (trap != BOOK3S_INTERRUPT_SYSCALL && - (vcpu->arch.shregs.msr & MSR_RI)) - ri_set = true; + if (likely(trap > BOOK3S_INTERRUPT_MACHINE_CHECK)) exsave = local_paca->exgen; - } else if (trap == BOOK3S_INTERRUPT_SYSTEM_RESET) { + else if (trap == BOOK3S_INTERRUPT_SYSTEM_RESET) exsave = local_paca->exnmi; - } else { /* trap == 0x200 */ + else /* trap == 0x200 */ exsave = local_paca->exmc; - } vcpu->arch.regs.gpr[1] = local_paca->kvm_hstate.scratch1; vcpu->arch.regs.gpr[3] = local_paca->kvm_hstate.scratch2; /* - * Only set RI after reading machine check regs (DAR, DSISR, SRR0/1) - * and hstate scratch (which we need to move into exsave to make - * re-entrant vs SRESET/MCE) + * After reading machine check regs (DAR, DSISR, SRR0/1) and hstate + * scratch (which we need to move into exsave to make re-entrant vs + * SRESET/MCE), register state is protected from reentrancy. However + * timebase, MMU, among other state is still set to guest, so don't + * enable MSR[RI] here. It gets enabled at the end, after in_guest + * is cleared. + * + * It is possible an NMI could come in here, which is why it is + * important to save the above state early so it can be debugged. */ - if (ri_set) { - if (unlikely(!(mfmsr() & MSR_RI))) { - __mtmsrd(MSR_RI, 1); - WARN_ON_ONCE(1); - } - } else { - WARN_ON_ONCE(mfmsr() & MSR_RI); - __mtmsrd(MSR_RI, 1); - } vcpu->arch.regs.gpr[9] = exsave[EX_R9/sizeof(u64)]; vcpu->arch.regs.gpr[10] = exsave[EX_R10/sizeof(u64)]; @@ -388,7 +1006,7 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc kvmppc_realmode_machine_check(vcpu); } else if (unlikely(trap == BOOK3S_INTERRUPT_HMI)) { - kvmppc_realmode_hmi_handler(); + kvmppc_p9_realmode_hmi_handler(vcpu); } else if (trap == BOOK3S_INTERRUPT_H_EMUL_ASSIST) { vcpu->arch.emul_inst = mfspr(SPRN_HEIR); @@ -427,13 +1045,6 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc */ mtspr(SPRN_HSRR0, vcpu->arch.regs.nip); mtspr(SPRN_HSRR1, vcpu->arch.shregs.msr); - - /* - * tm_return_to_guest re-loads SRR0/1, DAR, - * DSISR after RI is cleared, in case they had - * been clobbered by a MCE. - */ - __mtmsrd(0, 1); /* clear RI */ goto tm_return_to_guest; } } @@ -445,43 +1056,101 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc /* Advance host PURR/SPURR by the amount used by guest */ purr = mfspr(SPRN_PURR); spurr = mfspr(SPRN_SPURR); - mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr + - purr - vcpu->arch.purr); - mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr + - spurr - vcpu->arch.spurr); + local_paca->kvm_hstate.host_purr += purr - vcpu->arch.purr; + local_paca->kvm_hstate.host_spurr += spurr - vcpu->arch.spurr; vcpu->arch.purr = purr; vcpu->arch.spurr = spurr; vcpu->arch.ic = mfspr(SPRN_IC); vcpu->arch.pid = mfspr(SPRN_PID); - vcpu->arch.psscr = mfspr(SPRN_PSSCR) & PSSCR_GUEST_VIS; + vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR); vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0); vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1); vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2); vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3); - /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */ - mtspr(SPRN_PSSCR, host_psscr | - (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); - mtspr(SPRN_HFSCR, host_hfscr); - mtspr(SPRN_CIABR, host_ciabr); - mtspr(SPRN_DAWR0, host_dawr0); - mtspr(SPRN_DAWRX0, host_dawrx0); - if (cpu_has_feature(CPU_FTR_DAWR1)) { - mtspr(SPRN_DAWR1, host_dawr1); - mtspr(SPRN_DAWRX1, host_dawrx1); + dpdes = mfspr(SPRN_DPDES); + if (dpdes) + vcpu->arch.doorbell_request = 1; + + vc->vtb = mfspr(SPRN_VTB); + + dec = mfspr(SPRN_DEC); + if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */ + dec = (s32) dec; + *tb = mftb(); + vcpu->arch.dec_expires = dec + *tb; + + if (vc->tb_offset_applied) { + u64 new_tb = *tb - vc->tb_offset_applied; + mtspr(SPRN_TBU40, new_tb); + if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { + new_tb += 0x1000000; + mtspr(SPRN_TBU40, new_tb); + } + *tb = new_tb; + vc->tb_offset_applied = 0; } - if (kvm_is_radix(kvm)) { - /* - * Since this is radix, do a eieio; tlbsync; ptesync sequence - * in case we interrupted the guest between a tlbie and a - * ptesync. - */ - asm volatile("eieio; tlbsync; ptesync"); + save_clear_guest_mmu(kvm, vcpu); + switch_mmu_to_host(kvm, host_pidr); + + /* + * Enable MSR here in order to have facilities enabled to save + * guest registers. This enables MMU (if we were in realmode), so + * only switch MMU on after the MMU is switched to host, to avoid + * the P9_RADIX_PREFETCH_BUG or hash guest context. + */ + if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && + vcpu->arch.shregs.msr & MSR_TS_MASK) + msr |= MSR_TS_S; + __mtmsrd(msr, 0); + + store_vcpu_state(vcpu); + + mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr); + mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr); + + if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) { + /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */ + mtspr(SPRN_PSSCR, host_hpsscr | + (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); } + mtspr(SPRN_HFSCR, host_hfscr); + if (vcpu->arch.ciabr != host_ciabr) + mtspr(SPRN_CIABR, host_ciabr); + + if (dawr_enabled()) { + if (vcpu->arch.dawr0 != host_dawr0) + mtspr(SPRN_DAWR0, host_dawr0); + if (vcpu->arch.dawrx0 != host_dawrx0) + mtspr(SPRN_DAWRX0, host_dawrx0); + if (cpu_has_feature(CPU_FTR_DAWR1)) { + if (vcpu->arch.dawr1 != host_dawr1) + mtspr(SPRN_DAWR1, host_dawr1); + if (vcpu->arch.dawrx1 != host_dawrx1) + mtspr(SPRN_DAWRX1, host_dawrx1); + } + } + + if (dpdes) + mtspr(SPRN_DPDES, 0); + if (vc->pcr) + mtspr(SPRN_PCR, PCR_MASK); + + /* HDEC must be at least as large as DEC, so decrementer_max fits */ + mtspr(SPRN_HDEC, decrementer_max); + + timer_rearm_host_dec(*tb); + + restore_p9_host_os_sprs(vcpu, &host_os_sprs); + + barrier(); /* Close in_guest critical section */ + WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_NONE); + /* Interrupts are recoverable at this point */ + /* * cp_abort is required if the processor supports local copy-paste * to clear the copy buffer that was under control of the guest. @@ -489,37 +1158,7 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc if (cpu_has_feature(CPU_FTR_ARCH_31)) asm volatile(PPC_CP_ABORT); - vc->dpdes = mfspr(SPRN_DPDES); - vc->vtb = mfspr(SPRN_VTB); - mtspr(SPRN_DPDES, 0); - if (vc->pcr) - mtspr(SPRN_PCR, PCR_MASK); - - if (vc->tb_offset_applied) { - u64 new_tb = mftb() - vc->tb_offset_applied; - mtspr(SPRN_TBU40, new_tb); - tb = mftb(); - if ((tb & 0xffffff) < (new_tb & 0xffffff)) - mtspr(SPRN_TBU40, new_tb + 0x1000000); - vc->tb_offset_applied = 0; - } - - mtspr(SPRN_HDEC, 0x7fffffff); - - save_clear_guest_mmu(kvm, vcpu); - switch_mmu_to_host(kvm, host_pidr); - local_paca->kvm_hstate.in_guest = KVM_GUEST_MODE_NONE; - - /* - * If we are in real mode, only switch MMU on after the MMU is - * switched to host, to avoid the P9_RADIX_PREFETCH_BUG. - */ - if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && - vcpu->arch.shregs.msr & MSR_TS_MASK) - msr |= MSR_TS_S; - - __mtmsrd(msr, 0); - +out: end_timing(vcpu); return trap; diff --git a/arch/powerpc/kvm/book3s_hv_ras.c b/arch/powerpc/kvm/book3s_hv_ras.c index d4bca93b79..ccfd969656 100644 --- a/arch/powerpc/kvm/book3s_hv_ras.c +++ b/arch/powerpc/kvm/book3s_hv_ras.c @@ -136,6 +136,60 @@ void kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu) vcpu->arch.mce_evt = mce_evt; } + +long kvmppc_p9_realmode_hmi_handler(struct kvm_vcpu *vcpu) +{ + struct kvmppc_vcore *vc = vcpu->arch.vcore; + long ret = 0; + + /* + * Unapply and clear the offset first. That way, if the TB was not + * resynced then it will remain in host-offset, and if it was resynced + * then it is brought into host-offset. Then the tb offset is + * re-applied before continuing with the KVM exit. + * + * This way, we don't need to actually know whether not OPAL resynced + * the timebase or do any of the complicated dance that the P7/8 + * path requires. + */ + if (vc->tb_offset_applied) { + u64 new_tb = mftb() - vc->tb_offset_applied; + mtspr(SPRN_TBU40, new_tb); + if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { + new_tb += 0x1000000; + mtspr(SPRN_TBU40, new_tb); + } + vc->tb_offset_applied = 0; + } + + local_paca->hmi_irqs++; + + if (hmi_handle_debugtrig(NULL) >= 0) { + ret = 1; + goto out; + } + + if (ppc_md.hmi_exception_early) + ppc_md.hmi_exception_early(NULL); + +out: + if (vc->tb_offset) { + u64 new_tb = mftb() + vc->tb_offset; + mtspr(SPRN_TBU40, new_tb); + if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { + new_tb += 0x1000000; + mtspr(SPRN_TBU40, new_tb); + } + vc->tb_offset_applied = vc->tb_offset; + } + + return ret; +} + +/* + * The following subcore HMI handling is all only for pre-POWER9 CPUs. + */ + /* Check if dynamic split is in force and return subcore size accordingly. */ static inline int kvmppc_cur_subcore_size(void) { diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c index 632b254507..2257fb18cb 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c +++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c @@ -55,12 +55,6 @@ static int global_invalidates(struct kvm *kvm) smp_wmb(); cpumask_setall(&kvm->arch.need_tlb_flush); cpu = local_paca->kvm_hstate.kvm_vcore->pcpu; - /* - * On POWER9, threads are independent but the TLB is shared, - * so use the bit for the first thread to represent the core. - */ - if (cpu_has_feature(CPU_FTR_ARCH_300)) - cpu = cpu_first_tlb_thread_sibling(cpu); cpumask_clear_cpu(cpu, &kvm->arch.need_tlb_flush); } @@ -207,6 +201,15 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, if (kvm_is_radix(kvm)) return H_FUNCTION; + /* + * The HPTE gets used by compute_tlbie_rb() to set TLBIE bits, so + * these functions should work together -- must ensure a guest can not + * cause problems with the TLBIE that KVM executes. + */ + if ((pteh >> HPTE_V_SSIZE_SHIFT) & 0x2) { + /* B=0b1x is a reserved value, disallow it. */ + return H_PARAMETER; + } psize = kvmppc_actual_pgsz(pteh, ptel); if (!psize) return H_PARAMETER; diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index 32a4b4d412..d185dee260 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -778,17 +778,14 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) /* Restore AMR and UAMOR, set AMOR to all 1s */ ld r5,VCPU_AMR(r4) ld r6,VCPU_UAMOR(r4) - li r7,-1 mtspr SPRN_AMR,r5 mtspr SPRN_UAMOR,r6 - mtspr SPRN_AMOR,r7 - /* Restore state of CTRL run bit; assume 1 on entry */ + /* Restore state of CTRL run bit; the host currently has it set to 1 */ lwz r5,VCPU_CTRL(r4) andi. r5,r5,1 bne 4f - mfspr r6,SPRN_CTRLF - clrrdi r6,r6,1 + li r6,0 mtspr SPRN_CTRLT,r6 4: /* Secondary threads wait for primary to have done partition switch */ @@ -817,10 +814,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) * Set the decrementer to the guest decrementer. */ ld r8,VCPU_DEC_EXPIRES(r4) - /* r8 is a host timebase value here, convert to guest TB */ - ld r5,HSTATE_KVM_VCORE(r13) - ld r6,VCORE_TB_OFFSET_APPL(r5) - add r8,r8,r6 mftb r7 subf r3,r7,r8 mtspr SPRN_DEC,r3 @@ -1195,9 +1188,6 @@ guest_bypass: mftb r6 extsw r5,r5 16: add r5,r5,r6 - /* r5 is a guest timebase value here, convert to host TB */ - ld r4,VCORE_TB_OFFSET_APPL(r3) - subf r5,r4,r5 std r5,VCPU_DEC_EXPIRES(r9) /* Increment exit count, poke other threads to exit */ @@ -1211,12 +1201,12 @@ guest_bypass: stw r0, VCPU_CPU(r9) stw r0, VCPU_THREAD_CPU(r9) - /* Save guest CTRL register, set runlatch to 1 */ + /* Save guest CTRL register, set runlatch to 1 if it was clear */ mfspr r6,SPRN_CTRLF stw r6,VCPU_CTRL(r9) andi. r0,r6,1 bne 4f - ori r6,r6,1 + li r6,1 mtspr SPRN_CTRLT,r6 4: /* @@ -2163,9 +2153,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_TM) /* save expiry time of guest decrementer */ add r3, r3, r5 ld r4, HSTATE_KVM_VCPU(r13) - ld r5, HSTATE_KVM_VCORE(r13) - ld r6, VCORE_TB_OFFSET_APPL(r5) - subf r3, r6, r3 /* convert to host TB value */ std r3, VCPU_DEC_EXPIRES(r4) #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING @@ -2186,8 +2173,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_TM) * Also clear the runlatch bit before napping. */ kvm_do_nap: - mfspr r0, SPRN_CTRLF - clrrdi r0, r0, 1 + li r0,0 mtspr SPRN_CTRLT, r0 li r0,1 @@ -2206,8 +2192,7 @@ kvm_nap_sequence: /* desired LPCR value in r5 */ bl isa206_idle_insn_mayloss - mfspr r0, SPRN_CTRLF - ori r0, r0, 1 + li r0,1 mtspr SPRN_CTRLT, r0 mtspr SPRN_SRR1, r3 @@ -2264,9 +2249,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_TM) /* Restore guest decrementer */ ld r3, VCPU_DEC_EXPIRES(r4) - ld r5, HSTATE_KVM_VCORE(r13) - ld r6, VCORE_TB_OFFSET_APPL(r5) - add r3, r3, r6 /* convert host TB to guest TB value */ mftb r7 subf r3, r7, r3 mtspr SPRN_DEC, r3 @@ -2711,8 +2693,7 @@ kvmppc_bad_host_intr: std r0, GPR0(r1) std r9, GPR1(r1) std r2, GPR2(r1) - SAVE_4GPRS(3, r1) - SAVE_2GPRS(7, r1) + SAVE_GPRS(3, 8, r1) srdi r0, r12, 32 clrldi r12, r12, 32 std r0, _CCR(r1) @@ -2735,7 +2716,7 @@ kvmppc_bad_host_intr: ld r9, HSTATE_SCRATCH2(r13) ld r12, HSTATE_SCRATCH0(r13) GET_SCRATCH0(r0) - SAVE_4GPRS(9, r1) + SAVE_GPRS(9, 12, r1) std r0, GPR13(r1) SAVE_NVGPRS(r1) ld r5, HSTATE_CFAR(r13) @@ -2778,10 +2759,11 @@ kvmppc_msr_interrupt: blr /* + * void kvmhv_load_guest_pmu(struct kvm_vcpu *vcpu) + * * Load up guest PMU state. R3 points to the vcpu struct. */ -_GLOBAL(kvmhv_load_guest_pmu) -EXPORT_SYMBOL_GPL(kvmhv_load_guest_pmu) +kvmhv_load_guest_pmu: mr r4, r3 mflr r0 li r3, 1 @@ -2815,27 +2797,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) mtspr SPRN_MMCRA, r6 mtspr SPRN_SIAR, r7 mtspr SPRN_SDAR, r8 -BEGIN_FTR_SECTION - ld r5, VCPU_MMCR + 24(r4) - ld r6, VCPU_SIER + 8(r4) - ld r7, VCPU_SIER + 16(r4) - mtspr SPRN_MMCR3, r5 - mtspr SPRN_SIER2, r6 - mtspr SPRN_SIER3, r7 -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31) BEGIN_FTR_SECTION ld r5, VCPU_MMCR + 16(r4) ld r6, VCPU_SIER(r4) mtspr SPRN_MMCR2, r5 mtspr SPRN_SIER, r6 -BEGIN_FTR_SECTION_NESTED(96) lwz r7, VCPU_PMC + 24(r4) lwz r8, VCPU_PMC + 28(r4) ld r9, VCPU_MMCRS(r4) mtspr SPRN_SPMC1, r7 mtspr SPRN_SPMC2, r8 mtspr SPRN_MMCRS, r9 -END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) mtspr SPRN_MMCR0, r3 isync @@ -2843,10 +2815,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) blr /* + * void kvmhv_load_host_pmu(void) + * * Reload host PMU state saved in the PACA by kvmhv_save_host_pmu. */ -_GLOBAL(kvmhv_load_host_pmu) -EXPORT_SYMBOL_GPL(kvmhv_load_host_pmu) +kvmhv_load_host_pmu: mflr r0 lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */ cmpwi r4, 0 @@ -2884,25 +2857,18 @@ BEGIN_FTR_SECTION mtspr SPRN_MMCR2, r8 mtspr SPRN_SIER, r9 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) -BEGIN_FTR_SECTION - ld r5, HSTATE_MMCR3(r13) - ld r6, HSTATE_SIER2(r13) - ld r7, HSTATE_SIER3(r13) - mtspr SPRN_MMCR3, r5 - mtspr SPRN_SIER2, r6 - mtspr SPRN_SIER3, r7 -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31) mtspr SPRN_MMCR0, r3 isync mtlr r0 23: blr /* + * void kvmhv_save_guest_pmu(struct kvm_vcpu *vcpu, bool pmu_in_use) + * * Save guest PMU state into the vcpu struct. * r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA) */ -_GLOBAL(kvmhv_save_guest_pmu) -EXPORT_SYMBOL_GPL(kvmhv_save_guest_pmu) +kvmhv_save_guest_pmu: mr r9, r3 mr r8, r4 BEGIN_FTR_SECTION @@ -2951,14 +2917,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) BEGIN_FTR_SECTION std r10, VCPU_MMCR + 16(r9) END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) -BEGIN_FTR_SECTION - mfspr r5, SPRN_MMCR3 - mfspr r6, SPRN_SIER2 - mfspr r7, SPRN_SIER3 - std r5, VCPU_MMCR + 24(r9) - std r6, VCPU_SIER + 8(r9) - std r7, VCPU_SIER + 16(r9) -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31) std r7, VCPU_SIAR(r9) std r8, VCPU_SDAR(r9) mfspr r3, SPRN_PMC1 @@ -2976,7 +2934,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31) BEGIN_FTR_SECTION mfspr r5, SPRN_SIER std r5, VCPU_SIER(r9) -BEGIN_FTR_SECTION_NESTED(96) mfspr r6, SPRN_SPMC1 mfspr r7, SPRN_SPMC2 mfspr r8, SPRN_MMCRS @@ -2985,7 +2942,6 @@ BEGIN_FTR_SECTION_NESTED(96) std r8, VCPU_MMCRS(r9) lis r4, 0x8000 mtspr SPRN_MMCRS, r4 -END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96) END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 22: blr diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index a7061ee3b1..e414ca4483 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -459,7 +459,7 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) struct kvm_memslots *slots; struct kvm_memory_slot *memslot, *m; int ret = H_SUCCESS; - int srcu_idx; + int srcu_idx, bkt; kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START; @@ -478,7 +478,7 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) /* register the memslot */ slots = kvm_memslots(kvm); - kvm_for_each_memslot(memslot, slots) { + kvm_for_each_memslot(memslot, bkt, slots) { ret = __kvmppc_uvmem_memslot_create(kvm, memslot); if (ret) break; @@ -486,7 +486,7 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) if (ret) { slots = kvm_memslots(kvm); - kvm_for_each_memslot(m, slots) { + kvm_for_each_memslot(m, bkt, slots) { if (m == memslot) break; __kvmppc_uvmem_memslot_delete(kvm, memslot); @@ -560,7 +560,7 @@ static int __kvmppc_svm_page_out(struct vm_area_struct *vma, gpa, 0, page_shift); if (ret == U_SUCCESS) - *mig.dst = migrate_pfn(pfn) | MIGRATE_PFN_LOCKED; + *mig.dst = migrate_pfn(pfn); else { unlock_page(dpage); __free_page(dpage); @@ -647,7 +647,7 @@ void kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *slot, unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm) { - int srcu_idx; + int srcu_idx, bkt; struct kvm_memory_slot *memslot; /* @@ -662,7 +662,7 @@ unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm) srcu_idx = srcu_read_lock(&kvm->srcu); - kvm_for_each_memslot(memslot, kvm_memslots(kvm)) + kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm)) kvmppc_uvmem_drop_pages(memslot, kvm, false); srcu_read_unlock(&kvm->srcu, srcu_idx); @@ -774,7 +774,7 @@ static int kvmppc_svm_page_in(struct vm_area_struct *vma, } } - *mig.dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; + *mig.dst = migrate_pfn(page_to_pfn(dpage)); migrate_vma_pages(&mig); out_finalize: migrate_vma_finalize(&mig); @@ -821,7 +821,7 @@ unsigned long kvmppc_h_svm_init_done(struct kvm *kvm) { struct kvm_memslots *slots; struct kvm_memory_slot *memslot; - int srcu_idx; + int srcu_idx, bkt; long ret = H_SUCCESS; if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) @@ -830,7 +830,7 @@ unsigned long kvmppc_h_svm_init_done(struct kvm *kvm) /* migrate any unmoved normal pfn to device pfns*/ srcu_idx = srcu_read_lock(&kvm->srcu); slots = kvm_memslots(kvm); - kvm_for_each_memslot(memslot, slots) { + kvm_for_each_memslot(memslot, bkt, slots) { ret = kvmppc_uv_migrate_mem_slot(kvm, memslot); if (ret) { /* diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c index 6bc9425acb..34a801c360 100644 --- a/arch/powerpc/kvm/book3s_pr.c +++ b/arch/powerpc/kvm/book3s_pr.c @@ -428,7 +428,7 @@ static int kvmppc_core_check_requests_pr(struct kvm_vcpu *vcpu) /************* MMU Notifiers *************/ static bool do_kvm_unmap_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - long i; + unsigned long i; struct kvm_vcpu *vcpu; kvm_for_each_vcpu(i, vcpu, kvm) @@ -492,7 +492,7 @@ static void kvmppc_set_msr_pr(struct kvm_vcpu *vcpu, u64 msr) if (msr & MSR_POW) { if (!vcpu->arch.pending_exceptions) { - kvm_vcpu_block(vcpu); + kvm_vcpu_halt(vcpu); kvm_clear_request(KVM_REQ_UNHALT, vcpu); vcpu->stat.generic.halt_wakeup++; @@ -1899,16 +1899,15 @@ static void kvmppc_core_flush_memslot_pr(struct kvm *kvm, } static int kvmppc_core_prepare_memory_region_pr(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, - enum kvm_mr_change change) + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) { return 0; } static void kvmppc_core_commit_memory_region_pr(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { diff --git a/arch/powerpc/kvm/book3s_xics.c b/arch/powerpc/kvm/book3s_xics.c index ebd5d920de..9cc466006e 100644 --- a/arch/powerpc/kvm/book3s_xics.c +++ b/arch/powerpc/kvm/book3s_xics.c @@ -942,8 +942,8 @@ static int xics_debug_show(struct seq_file *m, void *private) struct kvmppc_xics *xics = m->private; struct kvm *kvm = xics->kvm; struct kvm_vcpu *vcpu; - int icsid, i; - unsigned long flags; + int icsid; + unsigned long flags, i; unsigned long t_rm_kick_vcpu, t_rm_check_resend; unsigned long t_rm_notify_eoi; unsigned long t_reject, t_check_resend; @@ -1340,7 +1340,7 @@ static int xics_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) static void kvmppc_xics_release(struct kvm_device *dev) { struct kvmppc_xics *xics = dev->private; - int i; + unsigned long i; struct kvm *kvm = xics->kvm; struct kvm_vcpu *vcpu; diff --git a/arch/powerpc/kvm/book3s_xics.h b/arch/powerpc/kvm/book3s_xics.h index 6231f76bdd..8e4c79e2fc 100644 --- a/arch/powerpc/kvm/book3s_xics.h +++ b/arch/powerpc/kvm/book3s_xics.h @@ -116,7 +116,7 @@ static inline struct kvmppc_icp *kvmppc_xics_find_server(struct kvm *kvm, u32 nr) { struct kvm_vcpu *vcpu = NULL; - int i; + unsigned long i; kvm_for_each_vcpu(i, vcpu, kvm) { if (vcpu->arch.icp && nr == vcpu->arch.icp->server_num) diff --git a/arch/powerpc/kvm/book3s_xive.c b/arch/powerpc/kvm/book3s_xive.c index a18db9e16e..e216c06807 100644 --- a/arch/powerpc/kvm/book3s_xive.c +++ b/arch/powerpc/kvm/book3s_xive.c @@ -368,7 +368,8 @@ static int xive_check_provisioning(struct kvm *kvm, u8 prio) { struct kvmppc_xive *xive = kvm->arch.xive; struct kvm_vcpu *vcpu; - int i, rc; + unsigned long i; + int rc; lockdep_assert_held(&xive->lock); @@ -439,7 +440,8 @@ static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio) int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio) { struct kvm_vcpu *vcpu; - int i, rc; + unsigned long i; + int rc; /* Locate target server */ vcpu = kvmppc_xive_find_server(kvm, *server); @@ -1519,7 +1521,8 @@ static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q) static void xive_pre_save_scan(struct kvmppc_xive *xive) { struct kvm_vcpu *vcpu = NULL; - int i, j; + unsigned long i; + int j; /* * See comment in xive_get_source() about how this @@ -1700,7 +1703,7 @@ static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq) { struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu = NULL; - int i; + unsigned long i; kvm_for_each_vcpu(i, vcpu, kvm) { struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; @@ -1928,7 +1931,7 @@ int kvmppc_xive_set_nr_servers(struct kvmppc_xive *xive, u64 addr) pr_devel("%s nr_servers=%u\n", __func__, nr_servers); - if (!nr_servers || nr_servers > KVM_MAX_VCPU_ID) + if (!nr_servers || nr_servers > KVM_MAX_VCPU_IDS) return -EINVAL; mutex_lock(&xive->lock); @@ -2037,7 +2040,7 @@ static void kvmppc_xive_release(struct kvm_device *dev) struct kvmppc_xive *xive = dev->private; struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; - int i; + unsigned long i; pr_devel("Releasing xive device\n"); @@ -2291,7 +2294,7 @@ static int xive_debug_show(struct seq_file *m, void *private) u64 t_vm_h_cppr = 0; u64 t_vm_h_eoi = 0; u64 t_vm_h_ipi = 0; - unsigned int i; + unsigned long i; if (!kvm) return 0; diff --git a/arch/powerpc/kvm/book3s_xive.h b/arch/powerpc/kvm/book3s_xive.h index e6a9651c6f..09d0657596 100644 --- a/arch/powerpc/kvm/book3s_xive.h +++ b/arch/powerpc/kvm/book3s_xive.h @@ -199,7 +199,7 @@ struct kvmppc_xive_vcpu { static inline struct kvm_vcpu *kvmppc_xive_find_server(struct kvm *kvm, u32 nr) { struct kvm_vcpu *vcpu = NULL; - int i; + unsigned long i; kvm_for_each_vcpu(i, vcpu, kvm) { if (vcpu->arch.xive_vcpu && nr == vcpu->arch.xive_vcpu->server_num) @@ -240,7 +240,7 @@ static inline u32 kvmppc_xive_vp(struct kvmppc_xive *xive, u32 server) static inline bool kvmppc_xive_vp_in_use(struct kvm *kvm, u32 vp_id) { struct kvm_vcpu *vcpu = NULL; - int i; + unsigned long i; kvm_for_each_vcpu(i, vcpu, kvm) { if (vcpu->arch.xive_vcpu && vp_id == vcpu->arch.xive_vcpu->vp_id) diff --git a/arch/powerpc/kvm/book3s_xive_native.c b/arch/powerpc/kvm/book3s_xive_native.c index 99db9ac499..561a5bfe04 100644 --- a/arch/powerpc/kvm/book3s_xive_native.c +++ b/arch/powerpc/kvm/book3s_xive_native.c @@ -807,7 +807,7 @@ static int kvmppc_xive_reset(struct kvmppc_xive *xive) { struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; - unsigned int i; + unsigned long i; pr_devel("%s\n", __func__); @@ -916,7 +916,7 @@ static int kvmppc_xive_native_eq_sync(struct kvmppc_xive *xive) { struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; - unsigned int i; + unsigned long i; pr_devel("%s\n", __func__); @@ -1017,7 +1017,7 @@ static void kvmppc_xive_native_release(struct kvm_device *dev) struct kvmppc_xive *xive = dev->private; struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; - int i; + unsigned long i; pr_devel("Releasing xive native device\n"); @@ -1214,7 +1214,7 @@ static int xive_native_debug_show(struct seq_file *m, void *private) struct kvmppc_xive *xive = m->private; struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; - unsigned int i; + unsigned long i; if (!kvm) return 0; diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index 8c15c90dd3..06c5830a93 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -718,7 +718,7 @@ int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu) if (vcpu->arch.shared->msr & MSR_WE) { local_irq_enable(); - kvm_vcpu_block(vcpu); + kvm_vcpu_halt(vcpu); kvm_clear_request(KVM_REQ_UNHALT, vcpu); hard_irq_disable(); @@ -1821,16 +1821,15 @@ void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) } int kvmppc_core_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, enum kvm_mr_change change) { return 0; } void kvmppc_core_commit_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { diff --git a/arch/powerpc/kvm/e500_emulate.c b/arch/powerpc/kvm/e500_emulate.c index 64eb833e9f..051102d50c 100644 --- a/arch/powerpc/kvm/e500_emulate.c +++ b/arch/powerpc/kvm/e500_emulate.c @@ -65,7 +65,7 @@ static int kvmppc_e500_emul_msgsnd(struct kvm_vcpu *vcpu, int rb) ulong param = vcpu->arch.regs.gpr[rb]; int prio = dbell2prio(rb); int pir = param & PPC_DBELL_PIR_MASK; - int i; + unsigned long i; struct kvm_vcpu *cvcpu; if (prio < 0) diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index b4e6f70b97..2ad0ccd202 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -236,7 +236,7 @@ int kvmppc_kvm_pv(struct kvm_vcpu *vcpu) break; case EV_HCALL_TOKEN(EV_IDLE): r = EV_SUCCESS; - kvm_vcpu_block(vcpu); + kvm_vcpu_halt(vcpu); kvm_clear_request(KVM_REQ_UNHALT, vcpu); break; default: @@ -463,9 +463,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) void kvm_arch_destroy_vm(struct kvm *kvm) { - unsigned int i; - struct kvm_vcpu *vcpu; - #ifdef CONFIG_KVM_XICS /* * We call kick_all_cpus_sync() to ensure that all @@ -476,14 +473,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kick_all_cpus_sync(); #endif - kvm_for_each_vcpu(i, vcpu, kvm) - kvm_vcpu_destroy(vcpu); + kvm_destroy_vcpus(kvm); mutex_lock(&kvm->lock); - for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) - kvm->vcpus[i] = NULL; - - atomic_set(&kvm->online_vcpus, 0); kvmppc_core_destroy_vm(kvm); @@ -641,15 +633,15 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) * implementations just count online CPUs. */ if (hv_enabled) - r = num_present_cpus(); + r = min_t(unsigned int, num_present_cpus(), KVM_MAX_VCPUS); else - r = num_online_cpus(); + r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS); break; case KVM_CAP_MAX_VCPUS: r = KVM_MAX_VCPUS; break; case KVM_CAP_MAX_VCPU_ID: - r = KVM_MAX_VCPU_ID; + r = KVM_MAX_VCPU_IDS; break; #ifdef CONFIG_PPC_BOOK3S_64 case KVM_CAP_PPC_GET_SMMU_INFO: @@ -706,20 +698,19 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) } int kvm_arch_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, enum kvm_mr_change change) { - return kvmppc_core_prepare_memory_region(kvm, memslot, mem, change); + return kvmppc_core_prepare_memory_region(kvm, old, new, change); } void kvm_arch_commit_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { - kvmppc_core_commit_memory_region(kvm, mem, old, new, change); + kvmppc_core_commit_memory_region(kvm, old, new, change); } void kvm_arch_flush_shadow_memslot(struct kvm *kvm, @@ -762,7 +753,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) if (err) goto out_vcpu_uninit; - vcpu->arch.waitp = &vcpu->wait; + rcuwait_init(&vcpu->arch.wait); + vcpu->arch.waitp = &vcpu->arch.wait; kvmppc_create_vcpu_debugfs(vcpu, vcpu->vcpu_id); return 0; @@ -1094,7 +1086,7 @@ static inline u64 sp_to_dp(u32 fprs) preempt_disable(); enable_kernel_fp(); - asm ("lfs%U1%X1 0,%1; stfd%U0%X0 0,%0" : "=m"UPD_CONSTR (fprd) : "m"UPD_CONSTR (fprs) + asm ("lfs%U1%X1 0,%1; stfd%U0%X0 0,%0" : "=m<>" (fprd) : "m<>" (fprs) : "fr0"); preempt_enable(); return fprd; @@ -1106,7 +1098,7 @@ static inline u32 dp_to_sp(u64 fprd) preempt_disable(); enable_kernel_fp(); - asm ("lfd%U1%X1 0,%1; stfs%U0%X0 0,%0" : "=m"UPD_CONSTR (fprs) : "m"UPD_CONSTR (fprd) + asm ("lfd%U1%X1 0,%1; stfs%U0%X0 0,%0" : "=m<>" (fprs) : "m<>" (fprd) : "fr0"); preempt_enable(); return fprs; diff --git a/arch/powerpc/mm/book3s32/Makefile b/arch/powerpc/mm/book3s32/Makefile index 15f4773643..50dd8f6bdf 100644 --- a/arch/powerpc/mm/book3s32/Makefile +++ b/arch/powerpc/mm/book3s32/Makefile @@ -9,5 +9,4 @@ endif obj-y += mmu.o mmu_context.o obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o -obj-$(CONFIG_PPC_KUEP) += kuep.o obj-$(CONFIG_PPC_KUAP) += kuap.o diff --git a/arch/powerpc/mm/book3s32/kuap.c b/arch/powerpc/mm/book3s32/kuap.c index 0f920f09af..28676cabb0 100644 --- a/arch/powerpc/mm/book3s32/kuap.c +++ b/arch/powerpc/mm/book3s32/kuap.c @@ -20,8 +20,11 @@ EXPORT_SYMBOL(kuap_unlock_all_ool); void setup_kuap(bool disabled) { - if (!disabled) + if (!disabled) { kuap_lock_all_ool(); + init_mm.context.sr0 |= SR_KS; + current->thread.sr0 |= SR_KS; + } if (smp_processor_id() != boot_cpuid) return; diff --git a/arch/powerpc/mm/book3s32/mmu_context.c b/arch/powerpc/mm/book3s32/mmu_context.c index e2708e387d..269a3eb25a 100644 --- a/arch/powerpc/mm/book3s32/mmu_context.c +++ b/arch/powerpc/mm/book3s32/mmu_context.c @@ -69,6 +69,12 @@ EXPORT_SYMBOL_GPL(__init_new_context); int init_new_context(struct task_struct *t, struct mm_struct *mm) { mm->context.id = __init_new_context(); + mm->context.sr0 = CTX_TO_VSID(mm->context.id, 0); + + if (!kuep_is_disabled()) + mm->context.sr0 |= SR_NX; + if (!kuap_is_disabled()) + mm->context.sr0 |= SR_KS; return 0; } @@ -108,20 +114,13 @@ void __init mmu_context_init(void) void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { long id = next->context.id; - unsigned long val; if (id < 0) panic("mm_struct %p has no context ID", next); isync(); - val = CTX_TO_VSID(id, 0); - if (!kuep_is_disabled()) - val |= SR_NX; - if (!kuap_is_disabled()) - val |= SR_KS; - - update_user_segments(val); + update_user_segments(next->context.sr0); if (IS_ENABLED(CONFIG_BDI_SWITCH)) abatron_pteptrs[1] = next->pgd; diff --git a/arch/powerpc/mm/book3s64/Makefile b/arch/powerpc/mm/book3s64/Makefile index 1b56d3af47..2d50cac499 100644 --- a/arch/powerpc/mm/book3s64/Makefile +++ b/arch/powerpc/mm/book3s64/Makefile @@ -2,20 +2,23 @@ ccflags-y := $(NO_MINIMAL_TOC) +obj-y += mmu_context.o pgtable.o trace.o +ifdef CONFIG_PPC_64S_HASH_MMU CFLAGS_REMOVE_slb.o = $(CC_FLAGS_FTRACE) - -obj-y += hash_pgtable.o hash_utils.o slb.o \ - mmu_context.o pgtable.o hash_tlb.o -obj-$(CONFIG_PPC_NATIVE) += hash_native.o -obj-$(CONFIG_PPC_RADIX_MMU) += radix_pgtable.o radix_tlb.o +obj-y += hash_pgtable.o hash_utils.o hash_tlb.o slb.o +obj-$(CONFIG_PPC_HASH_MMU_NATIVE) += hash_native.o obj-$(CONFIG_PPC_4K_PAGES) += hash_4k.o obj-$(CONFIG_PPC_64K_PAGES) += hash_64k.o -obj-$(CONFIG_HUGETLB_PAGE) += hash_hugetlbpage.o +obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += hash_hugepage.o +obj-$(CONFIG_PPC_SUBPAGE_PROT) += subpage_prot.o +endif + +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o + +obj-$(CONFIG_PPC_RADIX_MMU) += radix_pgtable.o radix_tlb.o ifdef CONFIG_HUGETLB_PAGE obj-$(CONFIG_PPC_RADIX_MMU) += radix_hugetlbpage.o endif -obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += hash_hugepage.o -obj-$(CONFIG_PPC_SUBPAGE_PROT) += subpage_prot.o obj-$(CONFIG_SPAPR_TCE_IOMMU) += iommu_api.o obj-$(CONFIG_PPC_PKEY) += pkeys.o diff --git a/arch/powerpc/mm/book3s64/hash_native.c b/arch/powerpc/mm/book3s64/hash_native.c index d8279bfe68..623a7b7ab3 100644 --- a/arch/powerpc/mm/book3s64/hash_native.c +++ b/arch/powerpc/mm/book3s64/hash_native.c @@ -43,110 +43,6 @@ static DEFINE_RAW_SPINLOCK(native_tlbie_lock); -static inline void tlbiel_hash_set_isa206(unsigned int set, unsigned int is) -{ - unsigned long rb; - - rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); - - asm volatile("tlbiel %0" : : "r" (rb)); -} - -/* - * tlbiel instruction for hash, set invalidation - * i.e., r=1 and is=01 or is=10 or is=11 - */ -static __always_inline void tlbiel_hash_set_isa300(unsigned int set, unsigned int is, - unsigned int pid, - unsigned int ric, unsigned int prs) -{ - unsigned long rb; - unsigned long rs; - unsigned int r = 0; /* hash format */ - - rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); - rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); - - asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) - : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "i"(r) - : "memory"); -} - - -static void tlbiel_all_isa206(unsigned int num_sets, unsigned int is) -{ - unsigned int set; - - asm volatile("ptesync": : :"memory"); - - for (set = 0; set < num_sets; set++) - tlbiel_hash_set_isa206(set, is); - - ppc_after_tlbiel_barrier(); -} - -static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) -{ - unsigned int set; - - asm volatile("ptesync": : :"memory"); - - /* - * Flush the partition table cache if this is HV mode. - */ - if (early_cpu_has_feature(CPU_FTR_HVMODE)) - tlbiel_hash_set_isa300(0, is, 0, 2, 0); - - /* - * Now invalidate the process table cache. UPRT=0 HPT modes (what - * current hardware implements) do not use the process table, but - * add the flushes anyway. - * - * From ISA v3.0B p. 1078: - * The following forms are invalid. - * * PRS=1, R=0, and RIC!=2 (The only process-scoped - * HPT caching is of the Process Table.) - */ - tlbiel_hash_set_isa300(0, is, 0, 2, 1); - - /* - * Then flush the sets of the TLB proper. Hash mode uses - * partition scoped TLB translations, which may be flushed - * in !HV mode. - */ - for (set = 0; set < num_sets; set++) - tlbiel_hash_set_isa300(set, is, 0, 0, 0); - - ppc_after_tlbiel_barrier(); - - asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); -} - -void hash__tlbiel_all(unsigned int action) -{ - unsigned int is; - - switch (action) { - case TLB_INVAL_SCOPE_GLOBAL: - is = 3; - break; - case TLB_INVAL_SCOPE_LPID: - is = 2; - break; - default: - BUG(); - } - - if (early_cpu_has_feature(CPU_FTR_ARCH_300)) - tlbiel_all_isa300(POWER9_TLB_SETS_HASH, is); - else if (early_cpu_has_feature(CPU_FTR_ARCH_207S)) - tlbiel_all_isa206(POWER8_TLB_SETS, is); - else if (early_cpu_has_feature(CPU_FTR_ARCH_206)) - tlbiel_all_isa206(POWER7_TLB_SETS, is); - else - WARN(1, "%s called on pre-POWER7 CPU\n", __func__); -} - static inline unsigned long ___tlbie(unsigned long vpn, int psize, int apsize, int ssize) { @@ -267,7 +163,7 @@ static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize) va |= ssize << 8; sllp = get_sllp_encoding(apsize); va |= sllp << 5; - asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,0", %1) + asm volatile(ASM_FTR_IFSET("tlbiel %0", PPC_TLBIEL_v205(%0, 0), %1) : : "r" (va), "i" (CPU_FTR_ARCH_206) : "memory"); break; @@ -286,7 +182,7 @@ static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize) */ va |= (vpn & 0xfe); va |= 1; /* L */ - asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,1", %1) + asm volatile(ASM_FTR_IFSET("tlbiel %0", PPC_TLBIEL_v205(%0, 1), %1) : : "r" (va), "i" (CPU_FTR_ARCH_206) : "memory"); break; diff --git a/arch/powerpc/mm/book3s64/hash_pgtable.c b/arch/powerpc/mm/book3s64/hash_pgtable.c index ad5eff097d..7ce8914992 100644 --- a/arch/powerpc/mm/book3s64/hash_pgtable.c +++ b/arch/powerpc/mm/book3s64/hash_pgtable.c @@ -16,7 +16,6 @@ #include -#define CREATE_TRACE_POINTS #include #if H_PGTABLE_RANGE > (USER_VSID_RANGE * (TASK_SIZE_USER64 / TASK_CONTEXT_SIZE)) diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c index c145776d3a..7abf82a698 100644 --- a/arch/powerpc/mm/book3s64/hash_utils.c +++ b/arch/powerpc/mm/book3s64/hash_utils.c @@ -99,8 +99,6 @@ */ static unsigned long _SDR1; -struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; -EXPORT_SYMBOL_GPL(mmu_psize_defs); u8 hpte_page_sizes[1 << LP_BITS]; EXPORT_SYMBOL_GPL(hpte_page_sizes); @@ -114,9 +112,6 @@ EXPORT_SYMBOL_GPL(mmu_linear_psize); int mmu_virtual_psize = MMU_PAGE_4K; int mmu_vmalloc_psize = MMU_PAGE_4K; EXPORT_SYMBOL_GPL(mmu_vmalloc_psize); -#ifdef CONFIG_SPARSEMEM_VMEMMAP -int mmu_vmemmap_psize = MMU_PAGE_4K; -#endif int mmu_io_psize = MMU_PAGE_4K; int mmu_kernel_ssize = MMU_SEGSIZE_256M; EXPORT_SYMBOL_GPL(mmu_kernel_ssize); @@ -175,6 +170,110 @@ static struct mmu_psize_def mmu_psize_defaults_gp[] = { }, }; +static inline void tlbiel_hash_set_isa206(unsigned int set, unsigned int is) +{ + unsigned long rb; + + rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); + + asm volatile("tlbiel %0" : : "r" (rb)); +} + +/* + * tlbiel instruction for hash, set invalidation + * i.e., r=1 and is=01 or is=10 or is=11 + */ +static __always_inline void tlbiel_hash_set_isa300(unsigned int set, unsigned int is, + unsigned int pid, + unsigned int ric, unsigned int prs) +{ + unsigned long rb; + unsigned long rs; + unsigned int r = 0; /* hash format */ + + rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); + rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); + + asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) + : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "i"(r) + : "memory"); +} + + +static void tlbiel_all_isa206(unsigned int num_sets, unsigned int is) +{ + unsigned int set; + + asm volatile("ptesync": : :"memory"); + + for (set = 0; set < num_sets; set++) + tlbiel_hash_set_isa206(set, is); + + ppc_after_tlbiel_barrier(); +} + +static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) +{ + unsigned int set; + + asm volatile("ptesync": : :"memory"); + + /* + * Flush the partition table cache if this is HV mode. + */ + if (early_cpu_has_feature(CPU_FTR_HVMODE)) + tlbiel_hash_set_isa300(0, is, 0, 2, 0); + + /* + * Now invalidate the process table cache. UPRT=0 HPT modes (what + * current hardware implements) do not use the process table, but + * add the flushes anyway. + * + * From ISA v3.0B p. 1078: + * The following forms are invalid. + * * PRS=1, R=0, and RIC!=2 (The only process-scoped + * HPT caching is of the Process Table.) + */ + tlbiel_hash_set_isa300(0, is, 0, 2, 1); + + /* + * Then flush the sets of the TLB proper. Hash mode uses + * partition scoped TLB translations, which may be flushed + * in !HV mode. + */ + for (set = 0; set < num_sets; set++) + tlbiel_hash_set_isa300(set, is, 0, 0, 0); + + ppc_after_tlbiel_barrier(); + + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); +} + +void hash__tlbiel_all(unsigned int action) +{ + unsigned int is; + + switch (action) { + case TLB_INVAL_SCOPE_GLOBAL: + is = 3; + break; + case TLB_INVAL_SCOPE_LPID: + is = 2; + break; + default: + BUG(); + } + + if (early_cpu_has_feature(CPU_FTR_ARCH_300)) + tlbiel_all_isa300(POWER9_TLB_SETS_HASH, is); + else if (early_cpu_has_feature(CPU_FTR_ARCH_207S)) + tlbiel_all_isa206(POWER8_TLB_SETS, is); + else if (early_cpu_has_feature(CPU_FTR_ARCH_206)) + tlbiel_all_isa206(POWER7_TLB_SETS, is); + else + WARN(1, "%s called on pre-POWER7 CPU\n", __func__); +} + /* * 'R' and 'C' update notes: * - Under pHyp or KVM, the updatepp path will not set C, thus it *will* @@ -563,7 +662,7 @@ static int __init htab_dt_scan_hugepage_blocks(unsigned long node, } #endif /* CONFIG_HUGETLB_PAGE */ -static void mmu_psize_set_default_penc(void) +static void __init mmu_psize_set_default_penc(void) { int bpsize, apsize; for (bpsize = 0; bpsize < MMU_PAGE_COUNT; bpsize++) @@ -573,7 +672,7 @@ static void mmu_psize_set_default_penc(void) #ifdef CONFIG_PPC_64K_PAGES -static bool might_have_hea(void) +static bool __init might_have_hea(void) { /* * The HEA ethernet adapter requires awareness of the @@ -644,7 +743,7 @@ static void __init htab_scan_page_sizes(void) * low-order N bits as the encoding for the 2^(12+N) byte page size * (if it exists). */ -static void init_hpte_page_sizes(void) +static void __init init_hpte_page_sizes(void) { long int ap, bp; long int shift, penc; @@ -1091,7 +1190,7 @@ void __init hash__early_init_mmu(void) ps3_early_mm_init(); else if (firmware_has_feature(FW_FEATURE_LPAR)) hpte_init_pseries(); - else if (IS_ENABLED(CONFIG_PPC_NATIVE)) + else if (IS_ENABLED(CONFIG_PPC_HASH_MMU_NATIVE)) hpte_init_native(); if (!mmu_hash_ops.hpte_insert) @@ -1988,7 +2087,7 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi) mmu_kernel_ssize, 0); } -void __kernel_map_pages(struct page *page, int numpages, int enable) +void hash__kernel_map_pages(struct page *page, int numpages, int enable) { unsigned long flags, vaddr, lmi; int i; diff --git a/arch/powerpc/mm/book3s64/mmu_context.c b/arch/powerpc/mm/book3s64/mmu_context.c index c10fc8a72f..c766e4c26e 100644 --- a/arch/powerpc/mm/book3s64/mmu_context.c +++ b/arch/powerpc/mm/book3s64/mmu_context.c @@ -31,7 +31,8 @@ static int alloc_context_id(int min_id, int max_id) return ida_alloc_range(&mmu_context_ida, min_id, max_id, GFP_KERNEL); } -void hash__reserve_context_id(int id) +#ifdef CONFIG_PPC_64S_HASH_MMU +void __init hash__reserve_context_id(int id) { int result = ida_alloc_range(&mmu_context_ida, id, id, GFP_KERNEL); @@ -50,7 +51,9 @@ int hash__alloc_context_id(void) return alloc_context_id(MIN_USER_CONTEXT, max); } EXPORT_SYMBOL_GPL(hash__alloc_context_id); +#endif +#ifdef CONFIG_PPC_64S_HASH_MMU static int realloc_context_ids(mm_context_t *ctx) { int i, id; @@ -150,6 +153,13 @@ void hash__setup_new_exec(void) slb_setup_new_exec(); } +#else +static inline int hash__init_new_context(struct mm_struct *mm) +{ + BUILD_BUG(); + return 0; +} +#endif static int radix__init_new_context(struct mm_struct *mm) { @@ -175,7 +185,9 @@ static int radix__init_new_context(struct mm_struct *mm) */ asm volatile("ptesync;isync" : : : "memory"); +#ifdef CONFIG_PPC_64S_HASH_MMU mm->context.hash_context = NULL; +#endif return index; } @@ -213,14 +225,22 @@ EXPORT_SYMBOL_GPL(__destroy_context); static void destroy_contexts(mm_context_t *ctx) { - int index, context_id; + if (radix_enabled()) { + ida_free(&mmu_context_ida, ctx->id); + } else { +#ifdef CONFIG_PPC_64S_HASH_MMU + int index, context_id; - for (index = 0; index < ARRAY_SIZE(ctx->extended_id); index++) { - context_id = ctx->extended_id[index]; - if (context_id) - ida_free(&mmu_context_ida, context_id); + for (index = 0; index < ARRAY_SIZE(ctx->extended_id); index++) { + context_id = ctx->extended_id[index]; + if (context_id) + ida_free(&mmu_context_ida, context_id); + } + kfree(ctx->hash_context); +#else + BUILD_BUG(); // radix_enabled() should be constant true +#endif } - kfree(ctx->hash_context); } static void pmd_frag_destroy(void *pmd_frag) diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 9e16c7b1a6..79ce3c22a2 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -22,6 +22,13 @@ #include "internal.h" +struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; +EXPORT_SYMBOL_GPL(mmu_psize_defs); + +#ifdef CONFIG_SPARSEMEM_VMEMMAP +int mmu_vmemmap_psize = MMU_PAGE_4K; +#endif + unsigned long __pmd_frag_nr; EXPORT_SYMBOL(__pmd_frag_nr); unsigned long __pmd_frag_size_shift; @@ -207,17 +214,12 @@ void __init mmu_partition_table_init(void) unsigned long patb_size = 1UL << PATB_SIZE_SHIFT; unsigned long ptcr; - BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 36), "Partition table size too large."); /* Initialize the Partition Table with no entries */ partition_tb = memblock_alloc(patb_size, patb_size); if (!partition_tb) panic("%s: Failed to allocate %lu bytes align=0x%lx\n", __func__, patb_size, patb_size); - /* - * update partition table control register, - * 64 K size. - */ ptcr = __pa(partition_tb) | (PATB_SIZE_SHIFT - 12); set_ptcr_when_no_uv(ptcr); powernv_set_nmmu_ptcr(ptcr); @@ -526,3 +528,23 @@ static int __init pgtable_debugfs_setup(void) return 0; } arch_initcall(pgtable_debugfs_setup); + +#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_HAS_MEMREMAP_COMPAT_ALIGN) +/* + * Override the generic version in mm/memremap.c. + * + * With hash translation, the direct-map range is mapped with just one + * page size selected by htab_init_page_sizes(). Consult + * mmu_psize_defs[] to determine the minimum page size alignment. +*/ +unsigned long memremap_compat_align(void) +{ + if (!radix_enabled()) { + unsigned int shift = mmu_psize_defs[mmu_linear_psize].shift; + return max(SUBSECTION_SIZE, 1UL << shift); + } + + return SUBSECTION_SIZE; +} +EXPORT_SYMBOL_GPL(memremap_compat_align); +#endif diff --git a/arch/powerpc/mm/book3s64/pkeys.c b/arch/powerpc/mm/book3s64/pkeys.c index a2d9ad1387..753e62ba67 100644 --- a/arch/powerpc/mm/book3s64/pkeys.c +++ b/arch/powerpc/mm/book3s64/pkeys.c @@ -66,7 +66,7 @@ static int __init dt_scan_storage_keys(unsigned long node, return 1; } -static int scan_pkey_feature(void) +static int __init scan_pkey_feature(void) { int ret; int pkeys_total = 0; diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 795d18a84f..def04631a7 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -33,7 +33,6 @@ #include -unsigned int mmu_pid_bits; unsigned int mmu_base_pid; unsigned long radix_mem_block_size __ro_after_init; @@ -335,7 +334,7 @@ static void __init radix_init_pgtable(void) u64 i; /* We don't support slb for radix */ - mmu_slb_size = 0; + slb_set_size(0); /* * Create the linear mapping @@ -357,18 +356,13 @@ static void __init radix_init_pgtable(void) -1, PAGE_KERNEL)); } - /* Find out how many PID bits are supported */ if (!cpu_has_feature(CPU_FTR_HVMODE) && cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) { /* * Older versions of KVM on these machines perfer if the * guest only uses the low 19 PID bits. */ - if (!mmu_pid_bits) - mmu_pid_bits = 19; - } else { - if (!mmu_pid_bits) - mmu_pid_bits = 20; + mmu_pid_bits = 19; } mmu_base_pid = 1; @@ -449,11 +443,6 @@ static int __init radix_dt_scan_page_sizes(unsigned long node, if (type == NULL || strcmp(type, "cpu") != 0) return 0; - /* Find MMU PID size */ - prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size); - if (prop && size == 4) - mmu_pid_bits = be32_to_cpup(prop); - /* Grab page size encodings */ prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size); if (!prop) @@ -510,7 +499,7 @@ static int __init probe_memory_block_size(unsigned long node, const char *uname, return 1; } -static unsigned long radix_memory_block_size(void) +static unsigned long __init radix_memory_block_size(void) { unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE; @@ -528,7 +517,7 @@ static unsigned long radix_memory_block_size(void) #else /* CONFIG_MEMORY_HOTPLUG */ -static unsigned long radix_memory_block_size(void) +static unsigned long __init radix_memory_block_size(void) { return 1UL * 1024 * 1024 * 1024; } @@ -572,22 +561,11 @@ void __init radix__early_init_devtree(void) return; } -static void radix_init_amor(void) -{ - /* - * In HV mode, we init AMOR (Authority Mask Override Register) so that - * the hypervisor and guest can setup IAMR (Instruction Authority Mask - * Register), enable key 0 and set it to 1. - * - * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11) - */ - mtspr(SPRN_AMOR, (3ul << 62)); -} - void __init radix__early_init_mmu(void) { unsigned long lpcr; +#ifdef CONFIG_PPC_64S_HASH_MMU #ifdef CONFIG_PPC_64K_PAGES /* PAGE_SIZE mappings */ mmu_virtual_psize = MMU_PAGE_64K; @@ -604,6 +582,7 @@ void __init radix__early_init_mmu(void) mmu_vmemmap_psize = MMU_PAGE_2M; } else mmu_vmemmap_psize = mmu_virtual_psize; +#endif #endif /* * initialize page table size @@ -644,7 +623,6 @@ void __init radix__early_init_mmu(void) lpcr = mfspr(SPRN_LPCR); mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR); radix_init_partition_table(); - radix_init_amor(); } else { radix_init_pseries(); } @@ -668,8 +646,6 @@ void radix__early_init_mmu_secondary(void) set_ptcr_when_no_uv(__pa(partition_tb) | (PATB_SIZE_SHIFT - 12)); - - radix_init_amor(); } radix__switch_mmu_context(NULL, &init_mm); @@ -920,6 +896,13 @@ void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long #endif #endif +#ifdef CONFIG_DEBUG_PAGEALLOC +void radix__kernel_map_pages(struct page *page, int numpages, int enable) +{ + pr_warn_once("DEBUG_PAGEALLOC not supported in radix mode\n"); +} +#endif + #ifdef CONFIG_TRANSPARENT_HUGEPAGE unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, diff --git a/arch/powerpc/mm/book3s64/slb.c b/arch/powerpc/mm/book3s64/slb.c index f0037bcc47..31f4cef3ad 100644 --- a/arch/powerpc/mm/book3s64/slb.c +++ b/arch/powerpc/mm/book3s64/slb.c @@ -868,19 +868,3 @@ DEFINE_INTERRUPT_HANDLER_RAW(do_slb_fault) return err; } } - -DEFINE_INTERRUPT_HANDLER(do_bad_slb_fault) -{ - int err = regs->result; - - if (err == -EFAULT) { - if (user_mode(regs)) - _exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar); - else - bad_page_fault(regs, SIGSEGV); - } else if (err == -EINVAL) { - unrecoverable_exception(regs); - } else { - BUG(); - } -} diff --git a/arch/powerpc/mm/copro_fault.c b/arch/powerpc/mm/copro_fault.c index 8acd001789..c1cb21a008 100644 --- a/arch/powerpc/mm/copro_fault.c +++ b/arch/powerpc/mm/copro_fault.c @@ -82,6 +82,7 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea, } EXPORT_SYMBOL_GPL(copro_handle_mm_fault); +#ifdef CONFIG_PPC_64S_HASH_MMU int copro_calculate_slb(struct mm_struct *mm, u64 ea, struct copro_slb *slb) { u64 vsid, vsidkey; @@ -146,3 +147,4 @@ void copro_flush_all_slbs(struct mm_struct *mm) cxl_slbia(mm); } EXPORT_SYMBOL_GPL(copro_flush_all_slbs); +#endif diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index a8d0ce85d3..eb8ecd7343 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -516,10 +517,8 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, * case. */ if (unlikely(fault & VM_FAULT_RETRY)) { - if (flags & FAULT_FLAG_ALLOW_RETRY) { - flags |= FAULT_FLAG_TRIED; - goto retry; - } + flags |= FAULT_FLAG_TRIED; + goto retry; } mmap_read_unlock(current->mm); @@ -620,4 +619,27 @@ DEFINE_INTERRUPT_HANDLER(do_bad_page_fault_segv) { bad_page_fault(regs, SIGSEGV); } + +/* + * In radix, segment interrupts indicate the EA is not addressable by the + * page table geometry, so they are always sent here. + * + * In hash, this is called if do_slb_fault returns error. Typically it is + * because the EA was outside the region allowed by software. + */ +DEFINE_INTERRUPT_HANDLER(do_bad_segment_interrupt) +{ + int err = regs->result; + + if (err == -EFAULT) { + if (user_mode(regs)) + _exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar); + else + bad_page_fault(regs, SIGSEGV); + } else if (err == -EINVAL) { + unrecoverable_exception(regs); + } else { + BUG(); + } +} #endif diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 9a75ba078e..ddead41e21 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -229,17 +229,22 @@ static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) m->hstate = hstate; return 1; } + +bool __init hugetlb_node_alloc_supported(void) +{ + return false; +} #endif -int __init alloc_bootmem_huge_page(struct hstate *h) +int __init alloc_bootmem_huge_page(struct hstate *h, int nid) { #ifdef CONFIG_PPC_BOOK3S_64 if (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled()) return pseries_alloc_bootmem_huge_page(h); #endif - return __alloc_bootmem_huge_page(h); + return __alloc_bootmem_huge_page(h, nid); } #ifndef CONFIG_PPC_BOOK3S_64 @@ -537,20 +542,26 @@ struct page *follow_huge_pd(struct vm_area_struct *vma, return page; } -#ifdef CONFIG_PPC_MM_SLICES +#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA +static inline int file_to_psize(struct file *file) +{ + struct hstate *hstate = hstate_file(file); + return shift_to_mmu_psize(huge_page_shift(hstate)); +} + unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - struct hstate *hstate = hstate_file(file); - int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate)); - #ifdef CONFIG_PPC_RADIX_MMU if (radix_enabled()) return radix__hugetlb_get_unmapped_area(file, addr, len, pgoff, flags); #endif - return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1); +#ifdef CONFIG_PPC_MM_SLICES + return slice_get_unmapped_area(addr, len, flags, file_to_psize(file), 1); +#endif + BUG(); } #endif diff --git a/arch/powerpc/mm/init-common.c b/arch/powerpc/mm/init-common.c index 3a82f89827..119ef491f7 100644 --- a/arch/powerpc/mm/init-common.c +++ b/arch/powerpc/mm/init-common.c @@ -20,6 +20,7 @@ #include #include #include +#include phys_addr_t memstart_addr __ro_after_init = (phys_addr_t)~0ull; EXPORT_SYMBOL_GPL(memstart_addr); @@ -33,6 +34,9 @@ bool disable_kuap = !IS_ENABLED(CONFIG_PPC_KUAP); static int __init parse_nosmep(char *p) { + if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + return 0; + disable_kuep = true; pr_warn("Disabling Kernel Userspace Execution Prevention\n"); return 0; @@ -47,6 +51,23 @@ static int __init parse_nosmap(char *p) } early_param("nosmap", parse_nosmap); +void __weak setup_kuep(bool disabled) +{ + if (!IS_ENABLED(CONFIG_PPC_KUEP) || disabled) + return; + + if (smp_processor_id() != boot_cpuid) + return; + + pr_info("Activating Kernel Userspace Execution Prevention\n"); +} + +void setup_kup(void) +{ + setup_kuap(disable_kuap); + setup_kuep(disable_kuep); +} + #define CTOR(shift) static void ctor_##shift(void *addr) \ { \ memset(addr, 0, sizeof(void *) << (shift)); \ diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c index 386be13602..35f46bf542 100644 --- a/arch/powerpc/mm/init_64.c +++ b/arch/powerpc/mm/init_64.c @@ -370,6 +370,9 @@ void register_page_bootmem_memmap(unsigned long section_nr, #endif /* CONFIG_SPARSEMEM_VMEMMAP */ #ifdef CONFIG_PPC_BOOK3S_64 +unsigned int mmu_lpid_bits; +unsigned int mmu_pid_bits; + static bool disable_radix = !IS_ENABLED(CONFIG_PPC_RADIX_MMU_DEFAULT); static int __init parse_disable_radix(char *p) @@ -437,11 +440,56 @@ static void __init early_check_vec5(void) } } +static int __init dt_scan_mmu_pid_width(unsigned long node, + const char *uname, int depth, + void *data) +{ + int size = 0; + const __be32 *prop; + const char *type = of_get_flat_dt_prop(node, "device_type", NULL); + + /* We are scanning "cpu" nodes only */ + if (type == NULL || strcmp(type, "cpu") != 0) + return 0; + + /* Find MMU LPID, PID register size */ + prop = of_get_flat_dt_prop(node, "ibm,mmu-lpid-bits", &size); + if (prop && size == 4) + mmu_lpid_bits = be32_to_cpup(prop); + + prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size); + if (prop && size == 4) + mmu_pid_bits = be32_to_cpup(prop); + + if (!mmu_pid_bits && !mmu_lpid_bits) + return 0; + + return 1; +} + void __init mmu_early_init_devtree(void) { + bool hvmode = !!(mfmsr() & MSR_HV); + /* Disable radix mode based on kernel command line. */ - if (disable_radix) - cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX; + if (disable_radix) { + if (IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) + cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX; + else + pr_warn("WARNING: Ignoring cmdline option disable_radix\n"); + } + + of_scan_flat_dt(dt_scan_mmu_pid_width, NULL); + if (hvmode && !mmu_lpid_bits) { + if (early_cpu_has_feature(CPU_FTR_ARCH_207S)) + mmu_lpid_bits = 12; /* POWER8-10 */ + else + mmu_lpid_bits = 10; /* POWER7 */ + } + if (!mmu_pid_bits) { + if (early_cpu_has_feature(CPU_FTR_ARCH_300)) + mmu_pid_bits = 20; /* POWER9-10 */ + } /* * Check /chosen/ibm,architecture-vec-5 if running as a guest. @@ -449,11 +497,12 @@ void __init mmu_early_init_devtree(void) * even though the ibm,architecture-vec-5 property created by * skiboot doesn't have the necessary bits set. */ - if (!(mfmsr() & MSR_HV)) + if (!hvmode) early_check_vec5(); if (early_radix_enabled()) { radix__early_init_devtree(); + /* * We have finalized the translation we are going to use by now. * Radix mode is not limited by RMA / VRMA addressing. @@ -463,5 +512,9 @@ void __init mmu_early_init_devtree(void) memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); } else hash__early_init_devtree(); + + if (!(cur_cpu_spec->mmu_features & MMU_FTR_HPTE_TABLE) && + !(cur_cpu_spec->mmu_features & MMU_FTR_TYPE_RADIX)) + panic("kernel does not support any MMU type offered by platform"); } #endif /* CONFIG_PPC_BOOK3S_64 */ diff --git a/arch/powerpc/mm/ioremap.c b/arch/powerpc/mm/ioremap.c index 57342154d2..4f12504fb4 100644 --- a/arch/powerpc/mm/ioremap.c +++ b/arch/powerpc/mm/ioremap.c @@ -98,23 +98,3 @@ void __iomem *do_ioremap(phys_addr_t pa, phys_addr_t offset, unsigned long size, return NULL; } - -#ifdef CONFIG_ZONE_DEVICE -/* - * Override the generic version in mm/memremap.c. - * - * With hash translation, the direct-map range is mapped with just one - * page size selected by htab_init_page_sizes(). Consult - * mmu_psize_defs[] to determine the minimum page size alignment. -*/ -unsigned long memremap_compat_align(void) -{ - unsigned int shift = mmu_psize_defs[mmu_linear_psize].shift; - - if (radix_enabled()) - return SUBSECTION_SIZE; - return max(SUBSECTION_SIZE, 1UL << shift); - -} -EXPORT_SYMBOL_GPL(memremap_compat_align); -#endif diff --git a/arch/powerpc/mm/maccess.c b/arch/powerpc/mm/maccess.c index aad7c47e00..ea821d0ffe 100644 --- a/arch/powerpc/mm/maccess.c +++ b/arch/powerpc/mm/maccess.c @@ -11,20 +11,3 @@ bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) { return is_kernel_addr((unsigned long)unsafe_src); } - -int copy_inst_from_kernel_nofault(struct ppc_inst *inst, u32 *src) -{ - unsigned int val, suffix; - int err; - - err = copy_from_kernel_nofault(&val, src, sizeof(val)); - if (err) - return err; - if (IS_ENABLED(CONFIG_PPC64) && get_op(val) == OP_PREFIX) { - err = copy_from_kernel_nofault(&suffix, src + 1, sizeof(suffix)); - *inst = ppc_inst_prefix(val, suffix); - } else { - *inst = ppc_inst(val); - } - return err; -} diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 05b9c3f314..8e301cd892 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -26,7 +26,6 @@ #include unsigned long long memory_limit; -bool init_mem_is_free; unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; EXPORT_SYMBOL(empty_zero_page); @@ -256,7 +255,7 @@ void __init mem_init(void) #endif high_memory = (void *) __va(max_low_pfn * PAGE_SIZE); - set_max_mapnr(max_pfn); + set_max_mapnr(max_low_pfn); kasan_late_init(); @@ -312,7 +311,6 @@ void free_initmem(void) { ppc_md.progress = ppc_printk_progress; mark_initmem_nx(); - init_mem_is_free = true; free_initmem_default(POISON_FREE_INITMEM); } diff --git a/arch/powerpc/mm/mmap.c b/arch/powerpc/mm/mmap.c index ae683fdc71..c475cf810a 100644 --- a/arch/powerpc/mm/mmap.c +++ b/arch/powerpc/mm/mmap.c @@ -80,6 +80,7 @@ static inline unsigned long mmap_base(unsigned long rnd, return PAGE_ALIGN(DEFAULT_MAP_WINDOW - gap - rnd); } +#ifdef HAVE_ARCH_UNMAPPED_AREA #ifdef CONFIG_PPC_RADIX_MMU /* * Same function as generic code used only for radix, because we don't need to overload @@ -181,11 +182,42 @@ radix__arch_get_unmapped_area_topdown(struct file *filp, */ return radix__arch_get_unmapped_area(filp, addr0, len, pgoff, flags); } +#endif + +unsigned long arch_get_unmapped_area(struct file *filp, + unsigned long addr, + unsigned long len, + unsigned long pgoff, + unsigned long flags) +{ +#ifdef CONFIG_PPC_MM_SLICES + return slice_get_unmapped_area(addr, len, flags, + mm_ctx_user_psize(¤t->mm->context), 0); +#else + BUG(); +#endif +} + +unsigned long arch_get_unmapped_area_topdown(struct file *filp, + const unsigned long addr0, + const unsigned long len, + const unsigned long pgoff, + const unsigned long flags) +{ +#ifdef CONFIG_PPC_MM_SLICES + return slice_get_unmapped_area(addr0, len, flags, + mm_ctx_user_psize(¤t->mm->context), 1); +#else + BUG(); +#endif +} +#endif /* HAVE_ARCH_UNMAPPED_AREA */ static void radix__arch_pick_mmap_layout(struct mm_struct *mm, unsigned long random_factor, struct rlimit *rlim_stack) { +#ifdef CONFIG_PPC_RADIX_MMU if (mmap_is_legacy(rlim_stack)) { mm->mmap_base = TASK_UNMAPPED_BASE; mm->get_unmapped_area = radix__arch_get_unmapped_area; @@ -193,13 +225,9 @@ static void radix__arch_pick_mmap_layout(struct mm_struct *mm, mm->mmap_base = mmap_base(random_factor, rlim_stack); mm->get_unmapped_area = radix__arch_get_unmapped_area_topdown; } -} -#else -/* dummy */ -extern void radix__arch_pick_mmap_layout(struct mm_struct *mm, - unsigned long random_factor, - struct rlimit *rlim_stack); #endif +} + /* * This function, called very early during the creation of a new * process VM image, sets up which VM layout function to use: diff --git a/arch/powerpc/mm/mmu_context.c b/arch/powerpc/mm/mmu_context.c index 74246536b8..1fb9c99f86 100644 --- a/arch/powerpc/mm/mmu_context.c +++ b/arch/powerpc/mm/mmu_context.c @@ -18,6 +18,12 @@ static inline void switch_mm_pgdir(struct task_struct *tsk, { /* 32-bit keeps track of the current PGDIR in the thread struct */ tsk->thread.pgdir = mm->pgd; +#ifdef CONFIG_PPC_BOOK3S_32 + tsk->thread.sr0 = mm->context.sr0; +#endif +#if defined(CONFIG_BOOKE_OR_40x) && defined(CONFIG_PPC_KUAP) + tsk->thread.pid = mm->context.id; +#endif } #elif defined(CONFIG_PPC_BOOK3E_64) static inline void switch_mm_pgdir(struct task_struct *tsk, @@ -25,6 +31,9 @@ static inline void switch_mm_pgdir(struct task_struct *tsk, { /* 64-bit Book3E keeps track of current PGD in the PACA */ get_paca()->pgd = mm->pgd; +#ifdef CONFIG_PPC_KUAP + tsk->thread.pid = mm->context.id; +#endif } #else static inline void switch_mm_pgdir(struct task_struct *tsk, @@ -81,7 +90,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, * context */ if (cpu_has_feature(CPU_FTR_ALTIVEC)) - asm volatile ("dssall"); + asm volatile (PPC_DSSALL); if (!new_on_cpu) membarrier_arch_switch_mm(prev, next, tsk); diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h index dd1cabc2ea..0dd4c18f83 100644 --- a/arch/powerpc/mm/mmu_decl.h +++ b/arch/powerpc/mm/mmu_decl.h @@ -126,7 +126,7 @@ unsigned long mmu_mapin_ram(unsigned long base, unsigned long top); #ifdef CONFIG_PPC_FSL_BOOK3E extern unsigned long map_mem_in_cams(unsigned long ram, int max_cam_idx, - bool dryrun); + bool dryrun, bool init); extern unsigned long calc_cam_sz(unsigned long ram, unsigned long virt, phys_addr_t phys); #ifdef CONFIG_PPC32 @@ -168,7 +168,7 @@ static inline phys_addr_t v_block_mapped(unsigned long va) { return 0; } static inline unsigned long p_block_mapped(phys_addr_t pa) { return 0; } #endif -#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_8xx) +#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_8xx) || defined(CONFIG_PPC_FSL_BOOK3E) void mmu_mark_initmem_nx(void); void mmu_mark_rodata_ro(void); #else diff --git a/arch/powerpc/mm/nohash/44x.c b/arch/powerpc/mm/nohash/44x.c index e079f26b26..1beae802bb 100644 --- a/arch/powerpc/mm/nohash/44x.c +++ b/arch/powerpc/mm/nohash/44x.c @@ -38,7 +38,7 @@ int icache_44x_need_flush; unsigned long tlb_47x_boltmap[1024/8]; -static void ppc44x_update_tlb_hwater(void) +static void __init ppc44x_update_tlb_hwater(void) { /* The TLB miss handlers hard codes the watermark in a cmpli * instruction to improve performances rather than loading it @@ -122,7 +122,7 @@ static void __init ppc47x_update_boltmap(void) /* * "Pins" a 256MB TLB entry in AS0 for kernel lowmem for 47x type MMU */ -static void ppc47x_pin_tlb(unsigned int virt, unsigned int phys) +static void __init ppc47x_pin_tlb(unsigned int virt, unsigned int phys) { unsigned int rA; int bolted; @@ -240,19 +240,3 @@ void __init mmu_init_secondary(int cpu) } } #endif /* CONFIG_SMP */ - -#ifdef CONFIG_PPC_KUEP -void setup_kuep(bool disabled) -{ - if (smp_processor_id() != boot_cpuid) - return; - - if (disabled) - patch_instruction_site(&patch__tlb_44x_kuep, ppc_inst(PPC_RAW_NOP())); - else - pr_info("Activating Kernel Userspace Execution Prevention\n"); - - if (IS_ENABLED(CONFIG_PPC_47x) && disabled) - patch_instruction_site(&patch__tlb_47x_kuep, ppc_inst(PPC_RAW_NOP())); -} -#endif diff --git a/arch/powerpc/mm/nohash/8xx.c b/arch/powerpc/mm/nohash/8xx.c index 0df9fe29dd..27f9186ae3 100644 --- a/arch/powerpc/mm/nohash/8xx.c +++ b/arch/powerpc/mm/nohash/8xx.c @@ -8,11 +8,7 @@ */ #include -#include #include -#include -#include -#include #include @@ -212,35 +208,6 @@ void __init setup_initial_memory_limit(phys_addr_t first_memblock_base, memblock_set_current_limit(min_t(u64, first_memblock_size, SZ_32M)); } -#ifdef CONFIG_PPC_KUEP -void __init setup_kuep(bool disabled) -{ - if (disabled) - return; - - pr_info("Activating Kernel Userspace Execution Prevention\n"); - - mtspr(SPRN_MI_AP, MI_APG_KUEP); -} -#endif - -#ifdef CONFIG_PPC_KUAP -struct static_key_false disable_kuap_key; -EXPORT_SYMBOL(disable_kuap_key); - -void __init setup_kuap(bool disabled) -{ - if (disabled) { - static_branch_enable(&disable_kuap_key); - return; - } - - pr_info("Activating Kernel Userspace Access Protection\n"); - - mtspr(SPRN_MD_AP, MD_APG_KUAP); -} -#endif - int pud_clear_huge(pud_t *pud) { return 0; diff --git a/arch/powerpc/mm/nohash/Makefile b/arch/powerpc/mm/nohash/Makefile index 0424f6ce5b..b467a25ee1 100644 --- a/arch/powerpc/mm/nohash/Makefile +++ b/arch/powerpc/mm/nohash/Makefile @@ -2,12 +2,12 @@ ccflags-$(CONFIG_PPC64) := $(NO_MINIMAL_TOC) -obj-y += mmu_context.o tlb.o tlb_low.o +obj-y += mmu_context.o tlb.o tlb_low.o kup.o obj-$(CONFIG_PPC_BOOK3E_64) += tlb_low_64e.o book3e_pgtable.o obj-$(CONFIG_40x) += 40x.o obj-$(CONFIG_44x) += 44x.o obj-$(CONFIG_PPC_8xx) += 8xx.o -obj-$(CONFIG_PPC_FSL_BOOK3E) += fsl_booke.o +obj-$(CONFIG_PPC_FSL_BOOK3E) += fsl_book3e.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr_booke.o ifdef CONFIG_HUGETLB_PAGE obj-$(CONFIG_PPC_FSL_BOOK3E) += book3e_hugetlbpage.o @@ -16,4 +16,4 @@ endif # Disable kcov instrumentation on sensitive code # This is necessary for booting with kcov enabled on book3e machines KCOV_INSTRUMENT_tlb.o := n -KCOV_INSTRUMENT_fsl_booke.o := n +KCOV_INSTRUMENT_fsl_book3e.o := n diff --git a/arch/powerpc/mm/nohash/book3e_pgtable.c b/arch/powerpc/mm/nohash/book3e_pgtable.c index 77884e2428..7d4368d055 100644 --- a/arch/powerpc/mm/nohash/book3e_pgtable.c +++ b/arch/powerpc/mm/nohash/book3e_pgtable.c @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -115,3 +116,17 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) smp_wmb(); return 0; } + +void __patch_exception(int exc, unsigned long addr) +{ + unsigned int *ibase = &interrupt_base_book3e; + + /* + * Our exceptions vectors start with a NOP and -then- a branch + * to deal with single stepping from userspace which stops on + * the second instruction. Thus we need to patch the second + * instruction of the exception, not the first one. + */ + + patch_branch(ibase + (exc / 4) + 1, addr, 0); +} diff --git a/arch/powerpc/mm/nohash/kaslr_booke.c b/arch/powerpc/mm/nohash/kaslr_booke.c index 4c74e8a548..96c38f9716 100644 --- a/arch/powerpc/mm/nohash/kaslr_booke.c +++ b/arch/powerpc/mm/nohash/kaslr_booke.c @@ -44,9 +44,7 @@ struct regions __initdata regions; static __init void kaslr_get_cmdline(void *fdt) { - int node = fdt_path_offset(fdt, "/chosen"); - - early_init_dt_scan_chosen(node, "chosen", 1, boot_command_line); + early_init_dt_scan_chosen(boot_command_line); } static unsigned long __init rotate_xor(unsigned long hash, const void *area, @@ -314,7 +312,7 @@ static unsigned long __init kaslr_choose_location(void *dt_ptr, phys_addr_t size pr_warn("KASLR: No safe seed for randomizing the kernel base.\n"); ram = min_t(phys_addr_t, __max_low_memory, size); - ram = map_mem_in_cams(ram, CONFIG_LOWMEM_CAM_NUM, true); + ram = map_mem_in_cams(ram, CONFIG_LOWMEM_CAM_NUM, true, true); linear_sz = min_t(unsigned long, ram, SZ_512M); /* If the linear size is smaller than 64M, do not randmize */ diff --git a/arch/powerpc/mm/nohash/mmu_context.c b/arch/powerpc/mm/nohash/mmu_context.c index 44b2b5e7ca..85b048f04c 100644 --- a/arch/powerpc/mm/nohash/mmu_context.c +++ b/arch/powerpc/mm/nohash/mmu_context.c @@ -33,6 +33,7 @@ #include #include #include +#include #include @@ -217,7 +218,7 @@ static void set_context(unsigned long id, pgd_t *pgd) /* sync */ mb(); - } else { + } else if (kuap_is_disabled()) { if (IS_ENABLED(CONFIG_40x)) mb(); /* sync */ @@ -305,6 +306,9 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next, if (IS_ENABLED(CONFIG_BDI_SWITCH)) abatron_pteptrs[1] = next->pgd; set_context(id, next->pgd); +#if defined(CONFIG_BOOKE_OR_40x) && defined(CONFIG_PPC_KUAP) + tsk->thread.pid = id; +#endif raw_spin_unlock(&context_lock); } diff --git a/arch/powerpc/mm/nohash/tlb.c b/arch/powerpc/mm/nohash/tlb.c index 5872f69141..fd2c77af5c 100644 --- a/arch/powerpc/mm/nohash/tlb.c +++ b/arch/powerpc/mm/nohash/tlb.c @@ -150,7 +150,6 @@ static inline int mmu_get_tsize(int psize) */ #ifdef CONFIG_PPC64 -int mmu_linear_psize; /* Page size used for the linear mapping */ int mmu_pte_psize; /* Page size used for PTE pages */ int mmu_vmemmap_psize; /* Page size used for the virtual mem map */ int book3e_htw_mode; /* HW tablewalk? Value is PPC_HTW_* */ @@ -185,6 +184,7 @@ EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx); * processor */ +#ifndef CONFIG_PPC_8xx /* * These are the base non-SMP variants of page and mm flushing */ @@ -218,6 +218,7 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) mmu_get_tsize(mmu_virtual_psize), 0); } EXPORT_SYMBOL(local_flush_tlb_page); +#endif /* * And here are the SMP non-local implementations @@ -431,7 +432,7 @@ void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address) } } -static void setup_page_sizes(void) +static void __init setup_page_sizes(void) { unsigned int tlb0cfg; unsigned int tlb0ps; @@ -569,7 +570,7 @@ static void setup_page_sizes(void) } } -static void setup_mmu_htw(void) +static void __init setup_mmu_htw(void) { /* * If we want to use HW tablewalk, enable it by patching the TLB miss @@ -643,7 +644,7 @@ static void early_init_this_mmu(void) if (map) linear_map_top = map_mem_in_cams(linear_map_top, - num_cams, false); + num_cams, false, true); } #endif @@ -655,14 +656,6 @@ static void early_init_this_mmu(void) static void __init early_init_mmu_global(void) { - /* XXX This will have to be decided at runtime, but right - * now our boot and TLB miss code hard wires it. Ideally - * we should find out a suitable page size and patch the - * TLB miss code (either that or use the PACA to store - * the value we want) - */ - mmu_linear_psize = MMU_PAGE_1G; - /* XXX This should be decided at runtime based on supported * page sizes in the TLB, but for now let's assume 16M is * always there and a good fit (which it probably is) @@ -764,7 +757,7 @@ void setup_initial_memory_limit(phys_addr_t first_memblock_base, num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4; linear_sz = map_mem_in_cams(first_memblock_size, num_cams, - true); + true, true); ppc64_rma_size = min_t(u64, linear_sz, 0x40000000); } else diff --git a/arch/powerpc/mm/nohash/tlb_low.S b/arch/powerpc/mm/nohash/tlb_low.S index 5add4a51e5..dd39074de9 100644 --- a/arch/powerpc/mm/nohash/tlb_low.S +++ b/arch/powerpc/mm/nohash/tlb_low.S @@ -369,7 +369,7 @@ _GLOBAL(_tlbivax_bcast) * extern void loadcam_entry(unsigned int index) * * Load TLBCAM[index] entry in to the L2 CAM MMU - * Must preserve r7, r8, r9, r10 and r11 + * Must preserve r7, r8, r9, r10, r11, r12 */ _GLOBAL(loadcam_entry) mflr r5 @@ -401,7 +401,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS) * * r3 = first entry to write * r4 = number of entries to write - * r5 = temporary tlb entry + * r5 = temporary tlb entry (0 means no switch to AS1) */ _GLOBAL(loadcam_multi) mflr r8 @@ -409,6 +409,8 @@ _GLOBAL(loadcam_multi) mfmsr r11 andi. r11,r11,MSR_IS bne 10f + mr. r12, r5 + beq 10f /* * Set up temporary TLB entry that is the same as what we're @@ -446,6 +448,8 @@ _GLOBAL(loadcam_multi) /* Don't return to AS=0 if we were in AS=1 at function start */ andi. r11,r11,MSR_IS bne 3f + cmpwi r12, 0 + beq 3f /* Return to AS=0 and clear the temporary entry */ mfmsr r6 diff --git a/arch/powerpc/mm/nohash/tlb_low_64e.S b/arch/powerpc/mm/nohash/tlb_low_64e.S index 9235e720e3..8b97c4acfe 100644 --- a/arch/powerpc/mm/nohash/tlb_low_64e.S +++ b/arch/powerpc/mm/nohash/tlb_low_64e.S @@ -128,6 +128,13 @@ END_BTB_FLUSH_SECTION bne tlb_miss_kernel_bolted +tlb_miss_user_bolted: +#ifdef CONFIG_PPC_KUAP + mfspr r10,SPRN_MAS1 + rlwinm. r10,r10,0,0x3fff0000 + beq- tlb_miss_fault_bolted /* KUAP fault */ +#endif + tlb_miss_common_bolted: /* * This is the guts of the TLB miss handler for bolted-linear. @@ -246,7 +253,7 @@ itlb_miss_fault_bolted: cmpldi cr0,r15,0 /* Check for user region */ oris r11,r11,_PAGE_ACCESSED@h - beq tlb_miss_common_bolted + beq tlb_miss_user_bolted b itlb_miss_kernel_bolted #ifdef CONFIG_PPC_FSL_BOOK3E @@ -676,6 +683,11 @@ finish_normal_tlb_miss: /* Check if required permissions are met */ andc. r15,r11,r14 bne- normal_tlb_miss_access_fault +#ifdef CONFIG_PPC_KUAP + mfspr r11,SPRN_MAS1 + rlwinm. r10,r11,0,0x3fff0000 + beq- normal_tlb_miss_access_fault /* KUAP fault */ +#endif /* Now we build the MAS: * @@ -689,15 +701,17 @@ finish_normal_tlb_miss: * * TODO: mix up code below for better scheduling */ - clrrdi r11,r16,12 /* Clear low crap in EA */ - rlwimi r11,r14,32-19,27,31 /* Insert WIMGE */ - mtspr SPRN_MAS2,r11 + clrrdi r10,r16,12 /* Clear low crap in EA */ + rlwimi r10,r14,32-19,27,31 /* Insert WIMGE */ + mtspr SPRN_MAS2,r10 /* Check page size, if not standard, update MAS1 */ - rldicl r11,r14,64-8,64-8 - cmpldi cr0,r11,BOOK3E_PAGESZ_4K + rldicl r10,r14,64-8,64-8 + cmpldi cr0,r10,BOOK3E_PAGESZ_4K beq- 1f +#ifndef CONFIG_PPC_KUAP mfspr r11,SPRN_MAS1 +#endif rlwimi r11,r14,31,21,24 rlwinm r11,r11,0,21,19 mtspr SPRN_MAS1,r11 @@ -786,7 +800,16 @@ virt_page_table_tlb_miss: mfspr r10,SPRN_MAS1 rlwinm r10,r10,0,16,1 /* Clear TID */ mtspr SPRN_MAS1,r10 +#ifdef CONFIG_PPC_KUAP + b 2f 1: + mfspr r10,SPRN_MAS1 + rlwinm. r10,r10,0,0x3fff0000 + beq- virt_page_table_tlb_miss_fault /* KUAP fault */ +2: +#else +1: +#endif BEGIN_MMU_FTR_SECTION /* Search if we already have a TLB entry for that virtual address, and * if we do, bail out. @@ -1027,6 +1050,11 @@ virt_page_table_tlb_miss_whacko_fault: * avoid too much complication, it will save/restore things for us */ htw_tlb_miss: +#ifdef CONFIG_PPC_KUAP + mfspr r10,SPRN_MAS1 + rlwinm. r10,r10,0,0x3fff0000 + beq- htw_tlb_miss_fault /* KUAP fault */ +#endif /* Search if we already have a TLB entry for that virtual address, and * if we do, bail out. * diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index 59d3cfcd78..9d5f710d2c 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -134,7 +134,7 @@ static int __init fake_numa_create_new_node(unsigned long end_pfn, return 0; } -static void reset_numa_cpu_lookup_table(void) +static void __init reset_numa_cpu_lookup_table(void) { unsigned int cpu; @@ -372,7 +372,7 @@ void update_numa_distance(struct device_node *node) * ibm,numa-lookup-index-table= {N, domainid1, domainid2, ..... domainidN} * ibm,numa-distance-table = { N, 1, 2, 4, 5, 1, 6, .... N elements} */ -static void initialize_form2_numa_distance_lookup_table(void) +static void __init initialize_form2_numa_distance_lookup_table(void) { int i, j; struct device_node *root; @@ -581,7 +581,7 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa) return 0; } -static int get_nid_and_numa_distance(struct drmem_lmb *lmb) +static int __init get_nid_and_numa_distance(struct drmem_lmb *lmb) { struct assoc_arrays aa = { .arrays = NULL }; int default_nid = NUMA_NO_NODE; diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index 9a93c1a5aa..6ec5a7dd79 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -81,9 +81,6 @@ static struct page *maybe_pte_to_page(pte_t pte) static pte_t set_pte_filter_hash(pte_t pte) { - if (radix_enabled()) - return pte; - pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) || cpu_has_feature(CPU_FTR_NOEXECUTE))) { @@ -112,6 +109,9 @@ static inline pte_t set_pte_filter(pte_t pte) { struct page *pg; + if (radix_enabled()) + return pte; + if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) return set_pte_filter_hash(pte); @@ -144,6 +144,9 @@ static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma, { struct page *pg; + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + return pte; + if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) return pte; @@ -280,7 +283,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_ { pmd_t *pmd = pmd_off(mm, addr); pte_basic_t val; - pte_basic_t *entry = &ptep->pte; + pte_basic_t *entry = (pte_basic_t *)ptep; int num, i; /* diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index fde1ed445c..906e4e4328 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -33,8 +33,6 @@ #include -extern char etext[], _stext[], _sinittext[], _einittext[]; - static u8 early_fixmap_pagetable[FIXMAP_PTE_SIZE] __page_aligned_data; notrace void __init early_ioremap_init(void) @@ -104,14 +102,13 @@ static void __init __mapin_ram_chunk(unsigned long offset, unsigned long top) { unsigned long v, s; phys_addr_t p; - int ktext; + bool ktext; s = offset; v = PAGE_OFFSET + s; p = memstart_addr + s; for (; s < top; s += PAGE_SIZE) { - ktext = ((char *)v >= _stext && (char *)v < etext) || - ((char *)v >= _sinittext && (char *)v < _einittext); + ktext = core_kernel_text(v); map_kernel_page(v, p, ktext ? PAGE_KERNEL_TEXT : PAGE_KERNEL); v += PAGE_SIZE; p += PAGE_SIZE; diff --git a/arch/powerpc/mm/ptdump/Makefile b/arch/powerpc/mm/ptdump/Makefile index 4050cbb55a..b533caaf09 100644 --- a/arch/powerpc/mm/ptdump/Makefile +++ b/arch/powerpc/mm/ptdump/Makefile @@ -10,5 +10,5 @@ obj-$(CONFIG_PPC_BOOK3S_64) += book3s64.o ifdef CONFIG_PTDUMP_DEBUGFS obj-$(CONFIG_PPC_BOOK3S_32) += bats.o segment_regs.o -obj-$(CONFIG_PPC_BOOK3S_64) += hashpagetable.o +obj-$(CONFIG_PPC_64S_HASH_MMU) += hashpagetable.o endif diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c index 82b45b1cb9..f42711f865 100644 --- a/arch/powerpc/mm/slice.c +++ b/arch/powerpc/mm/slice.c @@ -639,26 +639,6 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, } EXPORT_SYMBOL_GPL(slice_get_unmapped_area); -unsigned long arch_get_unmapped_area(struct file *filp, - unsigned long addr, - unsigned long len, - unsigned long pgoff, - unsigned long flags) -{ - return slice_get_unmapped_area(addr, len, flags, - mm_ctx_user_psize(¤t->mm->context), 0); -} - -unsigned long arch_get_unmapped_area_topdown(struct file *filp, - const unsigned long addr0, - const unsigned long len, - const unsigned long pgoff, - const unsigned long flags) -{ - return slice_get_unmapped_area(addr0, len, flags, - mm_ctx_user_psize(¤t->mm->context), 1); -} - unsigned int notrace get_slice_psize(struct mm_struct *mm, unsigned long addr) { unsigned char *psizes; diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 7e9b978b76..b20a2a83a6 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -31,7 +31,7 @@ pr_err_ratelimited("Branch offset 0x%lx (@%u) out of range\n", offset, ctx->idx); \ return -ERANGE; \ } \ - EMIT(PPC_INST_BRANCH | (offset & 0x03fffffc)); \ + EMIT(PPC_RAW_BRANCH(offset)); \ } while (0) /* blr; (unconditional 'branch' with link) to absolute address */ @@ -125,8 +125,7 @@ #define COND_LE (CR0_GT | COND_CMP_FALSE) #define SEEN_FUNC 0x20000000 /* might call external helpers */ -#define SEEN_STACK 0x40000000 /* uses BPF stack */ -#define SEEN_TAILCALL 0x80000000 /* uses tail calls */ +#define SEEN_TAILCALL 0x40000000 /* uses tail calls */ #define SEEN_VREG_MASK 0x1ff80000 /* Volatile registers r3-r12 */ #define SEEN_NVREG_MASK 0x0003ffff /* Non volatile registers r14-r31 */ @@ -151,8 +150,15 @@ struct codegen_context { unsigned int idx; unsigned int stack_size; int b2p[ARRAY_SIZE(b2p)]; + unsigned int exentry_idx; }; +#ifdef CONFIG_PPC32 +#define BPF_FIXUP_LEN 3 /* Three instructions => 12 bytes */ +#else +#define BPF_FIXUP_LEN 2 /* Two instructions => 8 bytes */ +#endif + static inline void bpf_flush_icache(void *start, void *end) { smp_wmb(); /* smp write barrier */ @@ -176,11 +182,14 @@ static inline void bpf_clear_seen_register(struct codegen_context *ctx, int i) void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func); int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, - u32 *addrs, bool extra_pass); + u32 *addrs, int pass); void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx); void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx); void bpf_jit_realloc_regs(struct codegen_context *ctx); +int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx, + int insn_idx, int jmp_off, int dst_reg); + #endif #endif diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 8acf8a611a..56dd1f4e3e 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -118,6 +118,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) struct bpf_prog *tmp_fp; bool bpf_blinded = false; bool extra_pass = false; + u32 extable_len; + u32 fixup_len; if (!fp->jit_requested) return org_fp; @@ -148,7 +150,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) image = jit_data->image; bpf_hdr = jit_data->header; proglen = jit_data->proglen; - alloclen = proglen + FUNCTION_DESCR_SIZE; extra_pass = true; goto skip_init_ctx; } @@ -166,7 +167,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) cgctx.stack_size = round_up(fp->aux->stack_depth, 16); /* Scouting faux-generate pass 0 */ - if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { + if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) { /* We hit something illegal or unsupported. */ fp = org_fp; goto out_addrs; @@ -179,7 +180,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) */ if (cgctx.seen & SEEN_TAILCALL) { cgctx.idx = 0; - if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { + if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) { fp = org_fp; goto out_addrs; } @@ -194,8 +195,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) bpf_jit_build_prologue(0, &cgctx); bpf_jit_build_epilogue(0, &cgctx); + fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; + extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); + proglen = cgctx.idx * 4; - alloclen = proglen + FUNCTION_DESCR_SIZE; + alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns); if (!bpf_hdr) { @@ -203,6 +207,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) goto out_addrs; } + if (extable_len) + fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len; + skip_init_ctx: code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); @@ -227,7 +234,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) /* Now build the prologue, body code & epilogue for real. */ cgctx.idx = 0; bpf_jit_build_prologue(code_base, &cgctx); - if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass)) { + if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) { bpf_jit_binary_free(bpf_hdr); fp = org_fp; goto out_addrs; @@ -255,7 +262,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) fp->bpf_func = (void *)image; fp->jited = 1; - fp->jited_len = alloclen; + fp->jited_len = proglen + FUNCTION_DESCR_SIZE; bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE)); if (!fp->is_func || extra_pass) { @@ -279,3 +286,52 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) return fp; } + +/* + * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling + * this function, as this only applies to BPF_PROBE_MEM, for now. + */ +int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx, + int insn_idx, int jmp_off, int dst_reg) +{ + off_t offset; + unsigned long pc; + struct exception_table_entry *ex; + u32 *fixup; + + /* Populate extable entries only in the last pass */ + if (pass != 2) + return 0; + + if (!fp->aux->extable || + WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) + return -EINVAL; + + pc = (unsigned long)&image[insn_idx]; + + fixup = (void *)fp->aux->extable - + (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + + (ctx->exentry_idx * BPF_FIXUP_LEN * 4); + + fixup[0] = PPC_RAW_LI(dst_reg, 0); + if (IS_ENABLED(CONFIG_PPC32)) + fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ + + fixup[BPF_FIXUP_LEN - 1] = + PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); + + ex = &fp->aux->extable[ctx->exentry_idx]; + + offset = pc - (long)&ex->insn; + if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) + return -ERANGE; + ex->insn = offset; + + offset = (long)fixup - (long)&ex->fixup; + if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) + return -ERANGE; + ex->fixup = offset; + + ctx->exentry_idx++; + return 0; +} diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c index bce5eda851..cf8dd8aea3 100644 --- a/arch/powerpc/net/bpf_jit_comp32.c +++ b/arch/powerpc/net/bpf_jit_comp32.c @@ -224,13 +224,13 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o PPC_BCC(COND_GE, out); /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) * goto out; */ EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT)); /* tail_call_cnt++; */ EMIT(PPC_RAW_ADDIC(_R0, _R0, 1)); - PPC_BCC(COND_GT, out); + PPC_BCC(COND_GE, out); /* prog = array->ptrs[index]; */ EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29)); @@ -271,7 +271,7 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o /* Assemble the body code between the prologue & epilogue */ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, - u32 *addrs, bool extra_pass) + u32 *addrs, int pass) { const struct bpf_insn *insn = fp->insnsi; int flen = fp->len; @@ -287,6 +287,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * u32 src_reg = bpf_to_ppc(ctx, insn[i].src_reg); u32 src_reg_h = src_reg - 1; u32 tmp_reg = bpf_to_ppc(ctx, TMP_REG); + u32 size = BPF_SIZE(code); s16 off = insn[i].off; s32 imm = insn[i].imm; bool func_addr_fixed; @@ -817,23 +818,91 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * * BPF_LDX */ case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ - EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); - if (!fp->aux->verifier_zext) - EMIT(PPC_RAW_LI(dst_reg_h, 0)); - break; + case BPF_LDX | BPF_PROBE_MEM | BPF_B: case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ - EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); - if (!fp->aux->verifier_zext) - EMIT(PPC_RAW_LI(dst_reg_h, 0)); - break; + case BPF_LDX | BPF_PROBE_MEM | BPF_H: case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ - EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); - if (!fp->aux->verifier_zext) - EMIT(PPC_RAW_LI(dst_reg_h, 0)); - break; + case BPF_LDX | BPF_PROBE_MEM | BPF_W: case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ - EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off)); - EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4)); + case BPF_LDX | BPF_PROBE_MEM | BPF_DW: + /* + * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid + * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM + * load only if addr is kernel address (see is_kernel_addr()), otherwise + * set dst_reg=0 and move on. + */ + if (BPF_MODE(code) == BPF_PROBE_MEM) { + PPC_LI32(_R0, TASK_SIZE - off); + EMIT(PPC_RAW_CMPLW(src_reg, _R0)); + PPC_BCC(COND_GT, (ctx->idx + 5) * 4); + EMIT(PPC_RAW_LI(dst_reg, 0)); + /* + * For BPF_DW case, "li reg_h,0" would be needed when + * !fp->aux->verifier_zext. Emit NOP otherwise. + * + * Note that "li reg_h,0" is emitted for BPF_B/H/W case, + * if necessary. So, jump there insted of emitting an + * additional "li reg_h,0" instruction. + */ + if (size == BPF_DW && !fp->aux->verifier_zext) + EMIT(PPC_RAW_LI(dst_reg_h, 0)); + else + EMIT(PPC_RAW_NOP()); + /* + * Need to jump two instructions instead of one for BPF_DW case + * as there are two load instructions for dst_reg_h & dst_reg + * respectively. + */ + if (size == BPF_DW) + PPC_JMP((ctx->idx + 3) * 4); + else + PPC_JMP((ctx->idx + 2) * 4); + } + + switch (size) { + case BPF_B: + EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); + break; + case BPF_H: + EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); + break; + case BPF_W: + EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); + break; + case BPF_DW: + EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off)); + EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4)); + break; + } + + if (size != BPF_DW && !fp->aux->verifier_zext) + EMIT(PPC_RAW_LI(dst_reg_h, 0)); + + if (BPF_MODE(code) == BPF_PROBE_MEM) { + int insn_idx = ctx->idx - 1; + int jmp_off = 4; + + /* + * In case of BPF_DW, two lwz instructions are emitted, one + * for higher 32-bit and another for lower 32-bit. So, set + * ex->insn to the first of the two and jump over both + * instructions in fixup. + * + * Similarly, with !verifier_zext, two instructions are + * emitted for BPF_B/H/W case. So, set ex->insn to the + * instruction that could fault and skip over both + * instructions. + */ + if (size == BPF_DW || !fp->aux->verifier_zext) { + insn_idx -= 1; + jmp_off += 4; + } + + ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx, + jmp_off, dst_reg); + if (ret) + return ret; + } break; /* @@ -871,7 +940,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * case BPF_JMP | BPF_CALL: ctx->seen |= SEEN_FUNC; - ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass, + ret = bpf_jit_get_func_addr(fp, &insn[i], false, &func_addr, &func_addr_fixed); if (ret < 0) return ret; diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 57e1b66803..e1e8c93430 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -228,12 +228,12 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o PPC_BCC(COND_GE, out); /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) * goto out; */ PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); EMIT(PPC_RAW_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT)); - PPC_BCC(COND_GT, out); + PPC_BCC(COND_GE, out); /* * tail_call_cnt++; @@ -297,7 +297,7 @@ asm ( /* Assemble the body code between the prologue & epilogue */ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, - u32 *addrs, bool extra_pass) + u32 *addrs, int pass) { enum stf_barrier_type stf_barrier = stf_barrier_type_get(); const struct bpf_insn *insn = fp->insnsi; @@ -311,6 +311,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * u32 code = insn[i].code; u32 dst_reg = b2p[insn[i].dst_reg]; u32 src_reg = b2p[insn[i].src_reg]; + u32 size = BPF_SIZE(code); s16 off = insn[i].off; s32 imm = insn[i].imm; bool func_addr_fixed; @@ -783,25 +784,66 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * */ /* dst = *(u8 *)(ul) (src + off) */ case BPF_LDX | BPF_MEM | BPF_B: - EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); - if (insn_is_zext(&insn[i + 1])) - addrs[++i] = ctx->idx * 4; - break; + case BPF_LDX | BPF_PROBE_MEM | BPF_B: /* dst = *(u16 *)(ul) (src + off) */ case BPF_LDX | BPF_MEM | BPF_H: - EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); - if (insn_is_zext(&insn[i + 1])) - addrs[++i] = ctx->idx * 4; - break; + case BPF_LDX | BPF_PROBE_MEM | BPF_H: /* dst = *(u32 *)(ul) (src + off) */ case BPF_LDX | BPF_MEM | BPF_W: - EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); - if (insn_is_zext(&insn[i + 1])) - addrs[++i] = ctx->idx * 4; - break; + case BPF_LDX | BPF_PROBE_MEM | BPF_W: /* dst = *(u64 *)(ul) (src + off) */ case BPF_LDX | BPF_MEM | BPF_DW: - PPC_BPF_LL(dst_reg, src_reg, off); + case BPF_LDX | BPF_PROBE_MEM | BPF_DW: + /* + * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid + * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM + * load only if addr is kernel address (see is_kernel_addr()), otherwise + * set dst_reg=0 and move on. + */ + if (BPF_MODE(code) == BPF_PROBE_MEM) { + EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], src_reg, off)); + if (IS_ENABLED(CONFIG_PPC_BOOK3E_64)) + PPC_LI64(b2p[TMP_REG_2], 0x8000000000000000ul); + else /* BOOK3S_64 */ + PPC_LI64(b2p[TMP_REG_2], PAGE_OFFSET); + EMIT(PPC_RAW_CMPLD(b2p[TMP_REG_1], b2p[TMP_REG_2])); + PPC_BCC(COND_GT, (ctx->idx + 4) * 4); + EMIT(PPC_RAW_LI(dst_reg, 0)); + /* + * Check if 'off' is word aligned because PPC_BPF_LL() + * (BPF_DW case) generates two instructions if 'off' is not + * word-aligned and one instruction otherwise. + */ + if (BPF_SIZE(code) == BPF_DW && (off & 3)) + PPC_JMP((ctx->idx + 3) * 4); + else + PPC_JMP((ctx->idx + 2) * 4); + } + + switch (size) { + case BPF_B: + EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); + break; + case BPF_H: + EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); + break; + case BPF_W: + EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); + break; + case BPF_DW: + PPC_BPF_LL(dst_reg, src_reg, off); + break; + } + + if (size != BPF_DW && insn_is_zext(&insn[i + 1])) + addrs[++i] = ctx->idx * 4; + + if (BPF_MODE(code) == BPF_PROBE_MEM) { + ret = bpf_add_extable_entry(fp, image, pass, ctx, ctx->idx - 1, + 4, dst_reg); + if (ret) + return ret; + } break; /* @@ -840,7 +882,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * case BPF_JMP | BPF_CALL: ctx->seen |= SEEN_FUNC; - ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass, + ret = bpf_jit_get_func_addr(fp, &insn[i], false, &func_addr, &func_addr_fixed); if (ret < 0) return ret; diff --git a/arch/powerpc/perf/8xx-pmu.c b/arch/powerpc/perf/8xx-pmu.c index f970d1510d..4738c4dbf5 100644 --- a/arch/powerpc/perf/8xx-pmu.c +++ b/arch/powerpc/perf/8xx-pmu.c @@ -153,7 +153,7 @@ static void mpc8xx_pmu_read(struct perf_event *event) static void mpc8xx_pmu_del(struct perf_event *event, int flags) { - struct ppc_inst insn = ppc_inst(PPC_RAW_MFSPR(10, SPRN_SPRG_SCRATCH2)); + ppc_inst_t insn = ppc_inst(PPC_RAW_MFSPR(10, SPRN_SPRG_SCRATCH2)); mpc8xx_pmu_read(event); diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index e78de70509..b5b42cf0a7 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #ifdef CONFIG_PPC64 @@ -775,6 +776,34 @@ static void pmao_restore_workaround(bool ebb) mtspr(SPRN_PMC6, pmcs[5]); } +/* + * If the perf subsystem wants performance monitor interrupts as soon as + * possible (e.g., to sample the instruction address and stack chain), + * this should return true. The IRQ masking code can then enable MSR[EE] + * in some places (e.g., interrupt handlers) that allows PMI interrupts + * through to improve accuracy of profiles, at the cost of some performance. + * + * The PMU counters can be enabled by other means (e.g., sysfs raw SPR + * access), but in that case there is no need for prompt PMI handling. + * + * This currently returns true if any perf counter is being used. It + * could possibly return false if only events are being counted rather than + * samples being taken, but for now this is good enough. + */ +bool power_pmu_wants_prompt_pmi(void) +{ + struct cpu_hw_events *cpuhw; + + /* + * This could simply test local_paca->pmcregs_in_use if that were not + * under ifdef KVM. + */ + if (!ppmu) + return false; + + cpuhw = this_cpu_ptr(&cpu_hw_events); + return cpuhw->n_events; +} #endif /* CONFIG_PPC64 */ static void perf_event_interrupt(struct pt_regs *regs); @@ -2459,7 +2488,7 @@ static int power_pmu_prepare_cpu(unsigned int cpu) return 0; } -int register_power_pmu(struct power_pmu *pmu) +int __init register_power_pmu(struct power_pmu *pmu) { if (ppmu) return -EBUSY; /* something's already registered */ @@ -2486,8 +2515,24 @@ int register_power_pmu(struct power_pmu *pmu) } #ifdef CONFIG_PPC64 +static bool pmu_override = false; +static unsigned long pmu_override_val; +static void do_pmu_override(void *data) +{ + ppc_set_pmu_inuse(1); + if (pmu_override_val) + mtspr(SPRN_MMCR1, pmu_override_val); + mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC); +} + static int __init init_ppc64_pmu(void) { + if (cpu_has_feature(CPU_FTR_HVMODE) && pmu_override) { + pr_warn("disabling perf due to pmu_override= command line option.\n"); + on_each_cpu(do_pmu_override, NULL, 1); + return 0; + } + /* run through all the pmu drivers one at a time */ if (!init_power5_pmu()) return 0; @@ -2509,4 +2554,23 @@ static int __init init_ppc64_pmu(void) return init_generic_compat_pmu(); } early_initcall(init_ppc64_pmu); + +static int __init pmu_setup(char *str) +{ + unsigned long val; + + if (!early_cpu_has_feature(CPU_FTR_HVMODE)) + return 0; + + pmu_override = true; + + if (kstrtoul(str, 0, &val)) + val = 0; + + pmu_override_val = val; + + return 1; +} +__setup("pmu_override=", pmu_setup); + #endif diff --git a/arch/powerpc/perf/generic-compat-pmu.c b/arch/powerpc/perf/generic-compat-pmu.c index 695975227e..b6e25f7510 100644 --- a/arch/powerpc/perf/generic-compat-pmu.c +++ b/arch/powerpc/perf/generic-compat-pmu.c @@ -307,7 +307,7 @@ static struct power_pmu generic_compat_pmu = { .attr_groups = generic_compat_pmu_attr_groups, }; -int init_generic_compat_pmu(void) +int __init init_generic_compat_pmu(void) { int rc = 0; diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c index 1816f560a4..1e8aa934e3 100644 --- a/arch/powerpc/perf/hv-24x7.c +++ b/arch/powerpc/perf/hv-24x7.c @@ -756,7 +756,7 @@ static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event, } if (calc_ev_end > ev_end) { - pr_warn("event %zu exceeds it's own length: event=%pK, end=%pK, offset=%zu, calc_ev_end=%pK\n", + pr_warn("event %zu exceeds its own length: event=%pK, end=%pK, offset=%zu, calc_ev_end=%pK\n", event_idx, event, ev_end, offset, calc_ev_end); return -1; } diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c index f92bf5f6b7..4037ea6525 100644 --- a/arch/powerpc/perf/isa207-common.c +++ b/arch/powerpc/perf/isa207-common.c @@ -220,36 +220,82 @@ static inline u64 isa207_find_source(u64 idx, u32 sub_idx) /* Nothing to do */ break; case 1: - ret = PH(LVL, L1); + ret = PH(LVL, L1) | LEVEL(L1) | P(SNOOP, HIT); break; case 2: - ret = PH(LVL, L2); + ret = PH(LVL, L2) | LEVEL(L2) | P(SNOOP, HIT); break; case 3: - ret = PH(LVL, L3); + ret = PH(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT); break; case 4: - if (sub_idx <= 1) - ret = PH(LVL, LOC_RAM); - else if (sub_idx > 1 && sub_idx <= 2) - ret = PH(LVL, REM_RAM1); - else - ret = PH(LVL, REM_RAM2); - ret |= P(SNOOP, HIT); + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + ret = P(SNOOP, HIT); + + if (sub_idx == 1) + ret |= PH(LVL, LOC_RAM) | LEVEL(RAM); + else if (sub_idx == 2 || sub_idx == 3) + ret |= P(LVL, HIT) | LEVEL(PMEM); + else if (sub_idx == 4) + ret |= PH(LVL, REM_RAM1) | REM | LEVEL(RAM) | P(HOPS, 2); + else if (sub_idx == 5 || sub_idx == 7) + ret |= P(LVL, HIT) | LEVEL(PMEM) | REM; + else if (sub_idx == 6) + ret |= PH(LVL, REM_RAM2) | REM | LEVEL(RAM) | P(HOPS, 3); + } else { + if (sub_idx <= 1) + ret = PH(LVL, LOC_RAM); + else if (sub_idx > 1 && sub_idx <= 2) + ret = PH(LVL, REM_RAM1); + else + ret = PH(LVL, REM_RAM2); + ret |= P(SNOOP, HIT); + } break; case 5: - ret = PH(LVL, REM_CCE1); - if ((sub_idx == 0) || (sub_idx == 2) || (sub_idx == 4)) - ret |= P(SNOOP, HIT); - else if ((sub_idx == 1) || (sub_idx == 3) || (sub_idx == 5)) - ret |= P(SNOOP, HITM); + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + ret = REM | P(HOPS, 0); + + if (sub_idx == 0 || sub_idx == 4) + ret |= PH(LVL, L2) | LEVEL(L2) | P(SNOOP, HIT); + else if (sub_idx == 1 || sub_idx == 5) + ret |= PH(LVL, L2) | LEVEL(L2) | P(SNOOP, HITM); + else if (sub_idx == 2 || sub_idx == 6) + ret |= PH(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT); + else if (sub_idx == 3 || sub_idx == 7) + ret |= PH(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM); + } else { + if (sub_idx == 0) + ret = PH(LVL, L2) | LEVEL(L2) | REM | P(SNOOP, HIT) | P(HOPS, 0); + else if (sub_idx == 1) + ret = PH(LVL, L2) | LEVEL(L2) | REM | P(SNOOP, HITM) | P(HOPS, 0); + else if (sub_idx == 2 || sub_idx == 4) + ret = PH(LVL, L3) | LEVEL(L3) | REM | P(SNOOP, HIT) | P(HOPS, 0); + else if (sub_idx == 3 || sub_idx == 5) + ret = PH(LVL, L3) | LEVEL(L3) | REM | P(SNOOP, HITM) | P(HOPS, 0); + } break; case 6: - ret = PH(LVL, REM_CCE2); - if ((sub_idx == 0) || (sub_idx == 2)) - ret |= P(SNOOP, HIT); - else if ((sub_idx == 1) || (sub_idx == 3)) - ret |= P(SNOOP, HITM); + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + if (sub_idx == 0) + ret = PH(LVL, REM_CCE1) | LEVEL(ANY_CACHE) | REM | + P(SNOOP, HIT) | P(HOPS, 2); + else if (sub_idx == 1) + ret = PH(LVL, REM_CCE1) | LEVEL(ANY_CACHE) | REM | + P(SNOOP, HITM) | P(HOPS, 2); + else if (sub_idx == 2) + ret = PH(LVL, REM_CCE2) | LEVEL(ANY_CACHE) | REM | + P(SNOOP, HIT) | P(HOPS, 3); + else if (sub_idx == 3) + ret = PH(LVL, REM_CCE2) | LEVEL(ANY_CACHE) | REM | + P(SNOOP, HITM) | P(HOPS, 3); + } else { + ret = PH(LVL, REM_CCE2); + if (sub_idx == 0 || sub_idx == 2) + ret |= P(SNOOP, HIT); + else if (sub_idx == 1 || sub_idx == 3) + ret |= P(SNOOP, HITM); + } break; case 7: ret = PM(LVL, L1); diff --git a/arch/powerpc/perf/isa207-common.h b/arch/powerpc/perf/isa207-common.h index 4a2cbc3dc0..ff12260398 100644 --- a/arch/powerpc/perf/isa207-common.h +++ b/arch/powerpc/perf/isa207-common.h @@ -273,6 +273,8 @@ #define P(a, b) PERF_MEM_S(a, b) #define PH(a, b) (P(LVL, HIT) | P(a, b)) #define PM(a, b) (P(LVL, MISS) | P(a, b)) +#define LEVEL(x) P(LVLNUM, x) +#define REM P(REMOTE, REMOTE) int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp, u64 event_config1); int isa207_compute_mmcr(u64 event[], int n_ev, diff --git a/arch/powerpc/perf/perf_regs.c b/arch/powerpc/perf/perf_regs.c index b931eed482..51d31b65e4 100644 --- a/arch/powerpc/perf/perf_regs.c +++ b/arch/powerpc/perf/perf_regs.c @@ -90,7 +90,11 @@ static u64 get_ext_regs_value(int idx) return mfspr(SPRN_SIER2); case PERF_REG_POWERPC_SIER3: return mfspr(SPRN_SIER3); + case PERF_REG_POWERPC_SDAR: + return mfspr(SPRN_SDAR); #endif + case PERF_REG_POWERPC_SIAR: + return mfspr(SPRN_SIAR); default: return 0; } } diff --git a/arch/powerpc/perf/power10-pmu.c b/arch/powerpc/perf/power10-pmu.c index 9dd75f3858..0975ad0b42 100644 --- a/arch/powerpc/perf/power10-pmu.c +++ b/arch/powerpc/perf/power10-pmu.c @@ -592,7 +592,7 @@ static struct power_pmu power10_pmu = { .check_attr_config = power10_check_attr_config, }; -int init_power10_pmu(void) +int __init init_power10_pmu(void) { unsigned int pvr; int rc; diff --git a/arch/powerpc/perf/power5+-pmu.c b/arch/powerpc/perf/power5+-pmu.c index 1873226799..753b4740ef 100644 --- a/arch/powerpc/perf/power5+-pmu.c +++ b/arch/powerpc/perf/power5+-pmu.c @@ -677,7 +677,7 @@ static struct power_pmu power5p_pmu = { .cache_events = &power5p_cache_events, }; -int init_power5p_pmu(void) +int __init init_power5p_pmu(void) { if (!cur_cpu_spec->oprofile_cpu_type || (strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power5+") diff --git a/arch/powerpc/perf/power5-pmu.c b/arch/powerpc/perf/power5-pmu.c index cb611c1e7a..1f83c4cba0 100644 --- a/arch/powerpc/perf/power5-pmu.c +++ b/arch/powerpc/perf/power5-pmu.c @@ -618,7 +618,7 @@ static struct power_pmu power5_pmu = { .flags = PPMU_HAS_SSLOT, }; -int init_power5_pmu(void) +int __init init_power5_pmu(void) { if (!cur_cpu_spec->oprofile_cpu_type || strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power5")) diff --git a/arch/powerpc/perf/power6-pmu.c b/arch/powerpc/perf/power6-pmu.c index 69ef382164..aec746f868 100644 --- a/arch/powerpc/perf/power6-pmu.c +++ b/arch/powerpc/perf/power6-pmu.c @@ -539,7 +539,7 @@ static struct power_pmu power6_pmu = { .cache_events = &power6_cache_events, }; -int init_power6_pmu(void) +int __init init_power6_pmu(void) { if (!cur_cpu_spec->oprofile_cpu_type || strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power6")) diff --git a/arch/powerpc/perf/power7-pmu.c b/arch/powerpc/perf/power7-pmu.c index 894c17f9a7..99b5ba314e 100644 --- a/arch/powerpc/perf/power7-pmu.c +++ b/arch/powerpc/perf/power7-pmu.c @@ -445,7 +445,7 @@ static struct power_pmu power7_pmu = { .cache_events = &power7_cache_events, }; -int init_power7_pmu(void) +int __init init_power7_pmu(void) { if (!cur_cpu_spec->oprofile_cpu_type || strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power7")) diff --git a/arch/powerpc/perf/power8-pmu.c b/arch/powerpc/perf/power8-pmu.c index 5282e8415d..f21194b560 100644 --- a/arch/powerpc/perf/power8-pmu.c +++ b/arch/powerpc/perf/power8-pmu.c @@ -378,7 +378,7 @@ static struct power_pmu power8_pmu = { .bhrb_nr = 32, }; -int init_power8_pmu(void) +int __init init_power8_pmu(void) { int rc; diff --git a/arch/powerpc/perf/power9-pmu.c b/arch/powerpc/perf/power9-pmu.c index ff3382140d..4b7c17e361 100644 --- a/arch/powerpc/perf/power9-pmu.c +++ b/arch/powerpc/perf/power9-pmu.c @@ -452,7 +452,7 @@ static struct power_pmu power9_pmu = { .check_attr_config = power9_check_attr_config, }; -int init_power9_pmu(void) +int __init init_power9_pmu(void) { int rc = 0; unsigned int pvr = mfspr(SPRN_PVR); diff --git a/arch/powerpc/perf/ppc970-pmu.c b/arch/powerpc/perf/ppc970-pmu.c index 1f82637852..09802482ba 100644 --- a/arch/powerpc/perf/ppc970-pmu.c +++ b/arch/powerpc/perf/ppc970-pmu.c @@ -489,7 +489,7 @@ static struct power_pmu ppc970_pmu = { .flags = PPMU_NO_SIPR | PPMU_NO_CONT_SAMPLING, }; -int init_ppc970_pmu(void) +int __init init_ppc970_pmu(void) { if (!cur_cpu_spec->oprofile_cpu_type || (strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/970") diff --git a/arch/riscv/Kbuild b/arch/riscv/Kbuild index 4614c01ba5..fb3397223d 100644 --- a/arch/riscv/Kbuild +++ b/arch/riscv/Kbuild @@ -2,3 +2,6 @@ obj-y += kernel/ mm/ net/ obj-$(CONFIG_BUILTIN_DTB) += boot/dts/ + +# for cleaning +subdir- += boot diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 20bc4b8c13..5adcbd9b5e 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -14,6 +14,7 @@ config RISCV def_bool y select ARCH_CLOCKSOURCE_INIT select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION + select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_VIRTUAL if MMU @@ -62,7 +63,7 @@ config RISCV select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL if MMU && 64BIT - select HANDLE_DOMAIN_IRQ + select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL select HAVE_ARCH_JUMP_LABEL_RELATIVE if !XIP_KERNEL @@ -75,6 +76,7 @@ config RISCV select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE if 64BIT && MMU + select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select HAVE_ARCH_THREAD_STRUCT_WHITELIST select HAVE_ARCH_VMAP_STACK if MMU && 64BIT select HAVE_ASM_MODVERSIONS @@ -83,7 +85,6 @@ config RISCV select HAVE_DMA_CONTIGUOUS if MMU select HAVE_EBPF_JIT if MMU select HAVE_FUNCTION_ERROR_INJECTION - select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_GCC_PLUGINS select HAVE_GENERIC_VDSO if MMU && 64BIT select HAVE_IRQ_TIME_ACCOUNTING @@ -146,26 +147,16 @@ config MMU Select if you want MMU-based virtualised addressing space support by paged memory management. If unsure, say 'Y'. -config VA_BITS - int - default 32 if 32BIT - default 39 if 64BIT - -config PA_BITS - int - default 34 if 32BIT - default 56 if 64BIT - config PAGE_OFFSET hex default 0xC0000000 if 32BIT default 0x80000000 if 64BIT && !MMU - default 0xffffffe000000000 if 64BIT + default 0xffffaf8000000000 if 64BIT config KASAN_SHADOW_OFFSET hex depends on KASAN_GENERIC - default 0xdfffffc800000000 if 64BIT + default 0xdfffffff00000000 if 64BIT default 0xffffffff if 32BIT config ARCH_FLATMEM_ENABLE @@ -211,7 +202,7 @@ config FIX_EARLYCON_MEM config PGTABLE_LEVELS int - default 3 if 64BIT + default 4 if 64BIT default 2 config LOCKDEP_SUPPORT @@ -315,6 +306,8 @@ config NUMA select GENERIC_ARCH_NUMA select OF_NUMA select ARCH_SUPPORTS_NUMA_BALANCING + select USE_PERCPU_NUMA_NODE_ID + select NEED_PER_CPU_EMBED_FIRST_CHUNK help Enable NUMA (Non-Uniform Memory Access) support. @@ -330,14 +323,6 @@ config NODES_SHIFT Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables. -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y - depends on NUMA - config RISCV_ISA_C bool "Emit compressed instructions when building Linux" default y @@ -378,12 +363,25 @@ source "kernel/Kconfig.hz" config RISCV_SBI_V01 bool "SBI v0.1 support" - default y depends on RISCV_SBI help This config allows kernel to use SBI v0.1 APIs. This will be deprecated in future once legacy M-mode software are no longer in use. +config RISCV_BOOT_SPINWAIT + bool "Spinwait booting method" + depends on SMP + default y + help + This enables support for booting Linux via spinwait method. In the + spinwait method, all cores randomly jump to Linux. One of the cores + gets chosen via lottery and all other keep spinning on a percpu + variable. This method cannot support CPU hotplug and sparse hartid + scheme. It should be only enabled for M-mode Linux or platforms relying + on older firmware without SBI HSM extension. All other platforms should + rely on ordered booting via SBI HSM extension which gets chosen + dynamically at runtime if the firmware supports it. + config KEXEC bool "Kexec system call" select KEXEC_CORE @@ -548,3 +546,5 @@ menu "Power management options" source "kernel/power/Kconfig" endmenu + +source "arch/riscv/kvm/Kconfig" diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 46a534f047..c112ab2a90 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -19,6 +19,14 @@ config SOC_SIFIVE help This enables support for SiFive SoC platform hardware. +config SOC_STARFIVE + bool "StarFive SoCs" + select PINCTRL + select RESET_CONTROLLER + select SIFIVE_PLIC + help + This enables support for StarFive SoC platform hardware. + config SOC_VIRT bool "QEMU Virt Machine" select CLINT_TIMER if RISCV_M_MODE diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index e03f45f771..7d81102cff 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -1,7 +1,5 @@ # This file is included by the global makefile so that you can add your own -# architecture-specific flags and dependencies. Remember to do have actions -# for "archclean" and "archdep" for cleaning up and making dependencies for -# this architecture +# architecture-specific flags and dependencies. # # This file is subject to the terms and conditions of the GNU General Public # License. See the file "COPYING" in the main directory of this archive @@ -106,6 +104,7 @@ endif head-y := arch/riscv/kernel/head.o core-$(CONFIG_RISCV_ERRATA_ALTERNATIVE) += arch/riscv/errata/ +core-$(CONFIG_KVM) += arch/riscv/kvm/ libs-y += arch/riscv/lib/ libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a @@ -146,5 +145,12 @@ install zinstall: $(CONFIG_SHELL) $(srctree)/$(boot)/install.sh $(KERNELRELEASE) \ $(boot)/$(install-image) System.map "$(INSTALL_PATH)" -archclean: - $(Q)$(MAKE) $(clean)=$(boot) +PHONY += rv32_randconfig +rv32_randconfig: + $(Q)$(MAKE) KCONFIG_ALLCONFIG=$(srctree)/arch/riscv/configs/32-bit.config \ + -f $(srctree)/Makefile randconfig + +PHONY += rv64_randconfig +rv64_randconfig: + $(Q)$(MAKE) KCONFIG_ALLCONFIG=$(srctree)/arch/riscv/configs/64-bit.config \ + -f $(srctree)/Makefile randconfig diff --git a/arch/riscv/boot/dts/Makefile b/arch/riscv/boot/dts/Makefile index fe996b8831..ff174996cd 100644 --- a/arch/riscv/boot/dts/Makefile +++ b/arch/riscv/boot/dts/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 subdir-y += sifive +subdir-y += starfive subdir-$(CONFIG_SOC_CANAAN_K210_DTB_BUILTIN) += canaan subdir-y += microchip diff --git a/arch/riscv/boot/dts/canaan/Makefile b/arch/riscv/boot/dts/canaan/Makefile index 9ee7156c0c..c61b08ac85 100644 --- a/arch/riscv/boot/dts/canaan/Makefile +++ b/arch/riscv/boot/dts/canaan/Makefile @@ -1,5 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -ifneq ($(CONFIG_SOC_CANAAN_K210_DTB_SOURCE),"") -dtb-y += $(strip $(shell echo $(CONFIG_SOC_CANAAN_K210_DTB_SOURCE))).dtb +dtb-$(CONFIG_SOC_CANAAN_K210_DTB_BUILTIN) += $(addsuffix .dtb, $(CONFIG_SOC_CANAAN_K210_DTB_SOURCE)) obj-$(CONFIG_SOC_CANAAN_K210_DTB_BUILTIN) += $(addsuffix .o, $(dtb-y)) -endif diff --git a/arch/riscv/boot/dts/canaan/k210.dtsi b/arch/riscv/boot/dts/canaan/k210.dtsi index 780416d489..44d3385147 100644 --- a/arch/riscv/boot/dts/canaan/k210.dtsi +++ b/arch/riscv/boot/dts/canaan/k210.dtsi @@ -103,8 +103,8 @@ rom0: nvmem@1000 { clint0: timer@2000000 { compatible = "canaan,k210-clint", "sifive,clint0"; reg = <0x2000000 0xC000>; - interrupts-extended = <&cpu0_intc 3 &cpu0_intc 7 - &cpu1_intc 3 &cpu1_intc 7>; + interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>, + <&cpu1_intc 3>, <&cpu1_intc 7>; }; plic0: interrupt-controller@c000000 { @@ -131,10 +131,11 @@ gpio0: gpio-controller@38001000 { compatible = "canaan,k210-gpiohs", "sifive,gpio0"; reg = <0x38001000 0x1000>; interrupt-controller; - interrupts = <34 35 36 37 38 39 40 41 - 42 43 44 45 46 47 48 49 - 50 51 52 53 54 55 56 57 - 58 59 60 61 62 63 64 65>; + interrupts = <34>, <35>, <36>, <37>, <38>, <39>, <40>, + <41>, <42>, <43>, <44>, <45>, <46>, <47>, + <48>, <49>, <50>, <51>, <52>, <53>, <54>, + <55>, <56>, <57>, <58>, <59>, <60>, <61>, + <62>, <63>, <64>, <65>; gpio-controller; ngpios = <32>; }; @@ -142,7 +143,7 @@ gpio0: gpio-controller@38001000 { dmac0: dma-controller@50000000 { compatible = "snps,axi-dma-1.01a"; reg = <0x50000000 0x1000>; - interrupts = <27 28 29 30 31 32>; + interrupts = <27>, <28>, <29>, <30>, <31>, <32>; #dma-cells = <1>; clocks = <&sysclk K210_CLK_DMA>, <&sysclk K210_CLK_DMA>; clock-names = "core-clk", "cfgr-clk"; @@ -317,7 +318,7 @@ fpioa: pinmux@502b0000 { timer0: timer@502d0000 { compatible = "snps,dw-apb-timer"; reg = <0x502D0000 0x100>; - interrupts = <14 15>; + interrupts = <14>, <15>; clocks = <&sysclk K210_CLK_TIMER0>, <&sysclk K210_CLK_APB0>; clock-names = "timer", "pclk"; @@ -327,7 +328,7 @@ timer0: timer@502d0000 { timer1: timer@502e0000 { compatible = "snps,dw-apb-timer"; reg = <0x502E0000 0x100>; - interrupts = <16 17>; + interrupts = <16>, <17>; clocks = <&sysclk K210_CLK_TIMER1>, <&sysclk K210_CLK_APB0>; clock-names = "timer", "pclk"; @@ -337,7 +338,7 @@ timer1: timer@502e0000 { timer2: timer@502f0000 { compatible = "snps,dw-apb-timer"; reg = <0x502F0000 0x100>; - interrupts = <18 19>; + interrupts = <18>, <19>; clocks = <&sysclk K210_CLK_TIMER2>, <&sysclk K210_CLK_APB0>; clock-names = "timer", "pclk"; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts index 0bcaf35045..984872f3d3 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts @@ -199,7 +199,7 @@ slot@0 { }; &spi3 { - spi-flash@0 { + flash@0 { compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts index ac8a03f586..7ba99b4da3 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts @@ -201,7 +201,7 @@ slot@0 { }; &spi3 { - spi-flash@0 { + flash@0 { compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts index 623998194b..be9b12c9b3 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts @@ -209,7 +209,7 @@ slot@0 { }; &spi3 { - spi-flash@0 { + flash@0 { compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts index cf605ba0d6..031c0c28f8 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts @@ -174,7 +174,7 @@ slot@0 { }; &spi3 { - spi-flash@0 { + flash@0 { compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; diff --git a/arch/riscv/boot/dts/microchip/microchip-mpfs-icicle-kit.dts b/arch/riscv/boot/dts/microchip/microchip-mpfs-icicle-kit.dts index cce5eca31f..0c748ae1b0 100644 --- a/arch/riscv/boot/dts/microchip/microchip-mpfs-icicle-kit.dts +++ b/arch/riscv/boot/dts/microchip/microchip-mpfs-icicle-kit.dts @@ -9,8 +9,6 @@ #define RTCCLK_FREQ 1000000 / { - #address-cells = <2>; - #size-cells = <2>; model = "Microchip PolarFire-SoC Icicle Kit"; compatible = "microchip,mpfs-icicle-kit", "microchip,mpfs"; @@ -35,9 +33,10 @@ memory@80000000 { reg = <0x0 0x80000000 0x0 0x40000000>; clocks = <&clkcfg 26>; }; +}; - soc { - }; +&refclk { + clock-frequency = <600000000>; }; &serial0 { diff --git a/arch/riscv/boot/dts/microchip/microchip-mpfs.dtsi b/arch/riscv/boot/dts/microchip/microchip-mpfs.dtsi index 4ef4bcb748..869aaf0d5c 100644 --- a/arch/riscv/boot/dts/microchip/microchip-mpfs.dtsi +++ b/arch/riscv/boot/dts/microchip/microchip-mpfs.dtsi @@ -139,6 +139,11 @@ cpu4_intc: interrupt-controller { }; }; + refclk: msspllclk { + compatible = "fixed-clock"; + #clock-cells = <0>; + }; + soc { #address-cells = <2>; #size-cells = <2>; @@ -153,62 +158,48 @@ cache-controller@2010000 { cache-size = <2097152>; cache-unified; interrupt-parent = <&plic>; - interrupts = <1 2 3>; + interrupts = <1>, <2>, <3>; reg = <0x0 0x2010000 0x0 0x1000>; }; clint@2000000 { - compatible = "sifive,clint0"; + compatible = "sifive,fu540-c000-clint", "sifive,clint0"; reg = <0x0 0x2000000 0x0 0xC000>; - interrupts-extended = <&cpu0_intc 3 &cpu0_intc 7 - &cpu1_intc 3 &cpu1_intc 7 - &cpu2_intc 3 &cpu2_intc 7 - &cpu3_intc 3 &cpu3_intc 7 - &cpu4_intc 3 &cpu4_intc 7>; + interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>, + <&cpu1_intc 3>, <&cpu1_intc 7>, + <&cpu2_intc 3>, <&cpu2_intc 7>, + <&cpu3_intc 3>, <&cpu3_intc 7>, + <&cpu4_intc 3>, <&cpu4_intc 7>; }; plic: interrupt-controller@c000000 { - #interrupt-cells = <1>; - compatible = "sifive,plic-1.0.0"; + compatible = "sifive,fu540-c000-plic", "sifive,plic-1.0.0"; reg = <0x0 0xc000000 0x0 0x4000000>; - riscv,ndev = <186>; + #address-cells = <0>; + #interrupt-cells = <1>; interrupt-controller; - interrupts-extended = <&cpu0_intc 11 - &cpu1_intc 11 &cpu1_intc 9 - &cpu2_intc 11 &cpu2_intc 9 - &cpu3_intc 11 &cpu3_intc 9 - &cpu4_intc 11 &cpu4_intc 9>; + interrupts-extended = <&cpu0_intc 11>, + <&cpu1_intc 11>, <&cpu1_intc 9>, + <&cpu2_intc 11>, <&cpu2_intc 9>, + <&cpu3_intc 11>, <&cpu3_intc 9>, + <&cpu4_intc 11>, <&cpu4_intc 9>; + riscv,ndev = <186>; }; dma@3000000 { compatible = "sifive,fu540-c000-pdma"; reg = <0x0 0x3000000 0x0 0x8000>; interrupt-parent = <&plic>; - interrupts = <23 24 25 26 27 28 29 30>; + interrupts = <23>, <24>, <25>, <26>, <27>, <28>, <29>, + <30>; #dma-cells = <1>; }; - refclk: refclk { - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <600000000>; - clock-output-names = "msspllclk"; - }; - clkcfg: clkcfg@20002000 { compatible = "microchip,mpfs-clkcfg"; reg = <0x0 0x20002000 0x0 0x1000>; - reg-names = "mss_sysreg"; clocks = <&refclk>; #clock-cells = <1>; - clock-output-names = "cpu", "axi", "ahb", "envm", /* 0-3 */ - "mac0", "mac1", "mmc", "timer", /* 4-7 */ - "mmuart0", "mmuart1", "mmuart2", "mmuart3", /* 8-11 */ - "mmuart4", "spi0", "spi1", "i2c0", /* 12-15 */ - "i2c1", "can0", "can1", "usb", /* 16-19 */ - "rsvd", "rtc", "qspi", "gpio0", /* 20-23 */ - "gpio1", "gpio2", "ddrc", "fic0", /* 24-27 */ - "fic1", "fic2", "fic3", "athena", "cfm"; /* 28-32 */ }; serial0: serial@20000000 { @@ -261,11 +252,10 @@ serial3: serial@20104000 { /* Common node entry for emmc/sd */ mmc: mmc@20008000 { - compatible = "cdns,sd4hc"; + compatible = "microchip,mpfs-sd4hc", "cdns,sd4hc"; reg = <0x0 0x20008000 0x0 0x1000>; interrupt-parent = <&plic>; - interrupts = <88 89>; - pinctrl-names = "default"; + interrupts = <88>, <89>; clocks = <&clkcfg 6>; max-frequency = <200000000>; status = "disabled"; @@ -275,7 +265,7 @@ emac0: ethernet@20110000 { compatible = "cdns,macb"; reg = <0x0 0x20110000 0x0 0x2000>; interrupt-parent = <&plic>; - interrupts = <64 65 66 67>; + interrupts = <64>, <65>, <66>, <67>; local-mac-address = [00 00 00 00 00 00]; clocks = <&clkcfg 4>, <&clkcfg 2>; clock-names = "pclk", "hclk"; @@ -288,7 +278,7 @@ emac1: ethernet@20112000 { compatible = "cdns,macb"; reg = <0x0 0x20112000 0x0 0x2000>; interrupt-parent = <&plic>; - interrupts = <70 71 72 73>; + interrupts = <70>, <71>, <72>, <73>; local-mac-address = [00 00 00 00 00 00]; clocks = <&clkcfg 5>, <&clkcfg 2>; status = "disabled"; diff --git a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi index 7db8610534..3eef52b1a5 100644 --- a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi +++ b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi @@ -137,20 +137,21 @@ cpu4_intc: interrupt-controller { soc { #address-cells = <2>; #size-cells = <2>; - compatible = "sifive,fu540-c000", "sifive,fu540", "simple-bus"; + compatible = "simple-bus"; ranges; plic0: interrupt-controller@c000000 { - #interrupt-cells = <1>; - compatible = "sifive,plic-1.0.0"; + compatible = "sifive,fu540-c000-plic", "sifive,plic-1.0.0"; reg = <0x0 0xc000000 0x0 0x4000000>; - riscv,ndev = <53>; + #address-cells = <0>; + #interrupt-cells = <1>; interrupt-controller; - interrupts-extended = < - &cpu0_intc 0xffffffff - &cpu1_intc 0xffffffff &cpu1_intc 9 - &cpu2_intc 0xffffffff &cpu2_intc 9 - &cpu3_intc 0xffffffff &cpu3_intc 9 - &cpu4_intc 0xffffffff &cpu4_intc 9>; + interrupts-extended = + <&cpu0_intc 0xffffffff>, + <&cpu1_intc 0xffffffff>, <&cpu1_intc 9>, + <&cpu2_intc 0xffffffff>, <&cpu2_intc 9>, + <&cpu3_intc 0xffffffff>, <&cpu3_intc 9>, + <&cpu4_intc 0xffffffff>, <&cpu4_intc 9>; + riscv,ndev = <53>; }; prci: clock-controller@10000000 { compatible = "sifive,fu540-c000-prci"; @@ -170,7 +171,8 @@ dma: dma@3000000 { compatible = "sifive,fu540-c000-pdma"; reg = <0x0 0x3000000 0x0 0x8000>; interrupt-parent = <&plic0>; - interrupts = <23 24 25 26 27 28 29 30>; + interrupts = <23>, <24>, <25>, <26>, <27>, <28>, <29>, + <30>; #dma-cells = <1>; }; uart1: serial@10011000 { @@ -195,8 +197,8 @@ i2c0: i2c@10030000 { }; qspi0: spi@10040000 { compatible = "sifive,fu540-c000-spi", "sifive,spi0"; - reg = <0x0 0x10040000 0x0 0x1000 - 0x0 0x20000000 0x0 0x10000000>; + reg = <0x0 0x10040000 0x0 0x1000>, + <0x0 0x20000000 0x0 0x10000000>; interrupt-parent = <&plic0>; interrupts = <51>; clocks = <&prci PRCI_CLK_TLCLK>; @@ -206,8 +208,8 @@ qspi0: spi@10040000 { }; qspi1: spi@10041000 { compatible = "sifive,fu540-c000-spi", "sifive,spi0"; - reg = <0x0 0x10041000 0x0 0x1000 - 0x0 0x30000000 0x0 0x10000000>; + reg = <0x0 0x10041000 0x0 0x1000>, + <0x0 0x30000000 0x0 0x10000000>; interrupt-parent = <&plic0>; interrupts = <52>; clocks = <&prci PRCI_CLK_TLCLK>; @@ -229,8 +231,8 @@ eth0: ethernet@10090000 { compatible = "sifive,fu540-c000-gem"; interrupt-parent = <&plic0>; interrupts = <53>; - reg = <0x0 0x10090000 0x0 0x2000 - 0x0 0x100a0000 0x0 0x1000>; + reg = <0x0 0x10090000 0x0 0x2000>, + <0x0 0x100a0000 0x0 0x1000>; local-mac-address = [00 00 00 00 00 00]; clock-names = "pclk", "hclk"; clocks = <&prci PRCI_CLK_GEMGXLPLL>, @@ -243,7 +245,7 @@ pwm0: pwm@10020000 { compatible = "sifive,fu540-c000-pwm", "sifive,pwm0"; reg = <0x0 0x10020000 0x0 0x1000>; interrupt-parent = <&plic0>; - interrupts = <42 43 44 45>; + interrupts = <42>, <43>, <44>, <45>; clocks = <&prci PRCI_CLK_TLCLK>; #pwm-cells = <3>; status = "disabled"; @@ -252,7 +254,7 @@ pwm1: pwm@10021000 { compatible = "sifive,fu540-c000-pwm", "sifive,pwm0"; reg = <0x0 0x10021000 0x0 0x1000>; interrupt-parent = <&plic0>; - interrupts = <46 47 48 49>; + interrupts = <46>, <47>, <48>, <49>; clocks = <&prci PRCI_CLK_TLCLK>; #pwm-cells = <3>; status = "disabled"; @@ -265,7 +267,7 @@ l2cache: cache-controller@2010000 { cache-size = <2097152>; cache-unified; interrupt-parent = <&plic0>; - interrupts = <1 2 3>; + interrupts = <1>, <2>, <3>; reg = <0x0 0x2010000 0x0 0x1000>; }; gpio: gpio@10060000 { diff --git a/arch/riscv/boot/dts/sifive/fu740-c000.dtsi b/arch/riscv/boot/dts/sifive/fu740-c000.dtsi index abbb960f90..8464b0e3c8 100644 --- a/arch/riscv/boot/dts/sifive/fu740-c000.dtsi +++ b/arch/riscv/boot/dts/sifive/fu740-c000.dtsi @@ -147,12 +147,12 @@ plic0: interrupt-controller@c000000 { reg = <0x0 0xc000000 0x0 0x4000000>; riscv,ndev = <69>; interrupt-controller; - interrupts-extended = < - &cpu0_intc 0xffffffff - &cpu1_intc 0xffffffff &cpu1_intc 9 - &cpu2_intc 0xffffffff &cpu2_intc 9 - &cpu3_intc 0xffffffff &cpu3_intc 9 - &cpu4_intc 0xffffffff &cpu4_intc 9>; + interrupts-extended = + <&cpu0_intc 0xffffffff>, + <&cpu1_intc 0xffffffff>, <&cpu1_intc 9>, + <&cpu2_intc 0xffffffff>, <&cpu2_intc 9>, + <&cpu3_intc 0xffffffff>, <&cpu3_intc 9>, + <&cpu4_intc 0xffffffff>, <&cpu4_intc 9>; }; prci: clock-controller@10000000 { compatible = "sifive,fu740-c000-prci"; @@ -273,7 +273,7 @@ ccache: cache-controller@2010000 { cache-size = <2097152>; cache-unified; interrupt-parent = <&plic0>; - interrupts = <19 21 22 20>; + interrupts = <19>, <21>, <22>, <20>; reg = <0x0 0x2010000 0x0 0x1000>; }; gpio: gpio@10060000 { diff --git a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts index 22f971e971..ced0d4e479 100644 --- a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts +++ b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts @@ -8,10 +8,9 @@ #define RTCCLK_FREQ 1000000 / { - #address-cells = <2>; - #size-cells = <2>; model = "SiFive HiFive Unleashed A00"; - compatible = "sifive,hifive-unleashed-a00", "sifive,fu540-c000"; + compatible = "sifive,hifive-unleashed-a00", "sifive,fu540-c000", + "sifive,fu540"; chosen { stdout-path = "serial0"; @@ -26,9 +25,6 @@ memory@80000000 { reg = <0x0 0x80000000 0x2 0x00000000>; }; - soc { - }; - hfclk: hfclk { #clock-cells = <0>; compatible = "fixed-clock"; @@ -63,7 +59,7 @@ &i2c0 { &qspi0 { status = "okay"; flash@0 { - compatible = "issi,is25wp256", "jedec,spi-nor"; + compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; m25p,fast-read; diff --git a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts index b40990210f..c4ed9efdff 100644 --- a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts +++ b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts @@ -9,8 +9,6 @@ #define RTCCLK_FREQ 1000000 / { - #address-cells = <2>; - #size-cells = <2>; model = "SiFive HiFive Unmatched A00"; compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000", "sifive,fu740"; @@ -28,9 +26,6 @@ memory@80000000 { reg = <0x0 0x80000000 0x4 0x00000000>; }; - soc { - }; - hfclk: hfclk { #clock-cells = <0>; compatible = "fixed-clock"; @@ -44,6 +39,11 @@ rtcclk: rtcclk { clock-frequency = ; clock-output-names = "rtcclk"; }; + + gpio-poweroff { + compatible = "gpio-poweroff"; + gpios = <&gpio 2 GPIO_ACTIVE_LOW>; + }; }; &uart0 { @@ -60,10 +60,21 @@ &i2c0 { temperature-sensor@4c { compatible = "ti,tmp451"; reg = <0x4c>; + vcc-supply = <&vdd_bpro>; interrupt-parent = <&gpio>; interrupts = <6 IRQ_TYPE_LEVEL_LOW>; }; + eeprom@54 { + compatible = "microchip,24c02", "atmel,24c02"; + reg = <0x54>; + vcc-supply = <&vdd_bpro>; + label = "board-id"; + pagesize = <16>; + read-only; + size = <256>; + }; + pmic@58 { compatible = "dlg,da9063"; reg = <0x58>; @@ -71,48 +82,44 @@ pmic@58 { interrupts = <1 IRQ_TYPE_LEVEL_LOW>; interrupt-controller; - regulators { - vdd_bcore1: bcore1 { - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <900000>; - regulator-min-microamp = <5000000>; - regulator-max-microamp = <5000000>; - regulator-always-on; - }; + onkey { + compatible = "dlg,da9063-onkey"; + }; - vdd_bcore2: bcore2 { - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <900000>; - regulator-min-microamp = <5000000>; - regulator-max-microamp = <5000000>; + rtc { + compatible = "dlg,da9063-rtc"; + }; + + wdt { + compatible = "dlg,da9063-watchdog"; + }; + + regulators { + vdd_bcore: bcores-merged { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-min-microamp = <4800000>; + regulator-max-microamp = <4800000>; regulator-always-on; }; vdd_bpro: bpro { regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; - regulator-min-microamp = <2500000>; - regulator-max-microamp = <2500000>; + regulator-min-microamp = <2400000>; + regulator-max-microamp = <2400000>; regulator-always-on; }; vdd_bperi: bperi { - regulator-min-microvolt = <1050000>; - regulator-max-microvolt = <1050000>; + regulator-min-microvolt = <1060000>; + regulator-max-microvolt = <1060000>; regulator-min-microamp = <1500000>; regulator-max-microamp = <1500000>; regulator-always-on; }; - vdd_bmem: bmem { - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <1200000>; - regulator-min-microamp = <3000000>; - regulator-max-microamp = <3000000>; - regulator-always-on; - }; - - vdd_bio: bio { + vdd_bmem_bio: bmem-bio-merged { regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-min-microamp = <3000000>; @@ -123,86 +130,66 @@ vdd_bio: bio { vdd_ldo1: ldo1 { regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; - regulator-min-microamp = <100000>; - regulator-max-microamp = <100000>; regulator-always-on; }; vdd_ldo2: ldo2 { regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; - regulator-min-microamp = <200000>; - regulator-max-microamp = <200000>; regulator-always-on; }; vdd_ldo3: ldo3 { - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-min-microamp = <200000>; - regulator-max-microamp = <200000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; regulator-always-on; }; vdd_ldo4: ldo4 { - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-min-microamp = <200000>; - regulator-max-microamp = <200000>; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; regulator-always-on; }; vdd_ldo5: ldo5 { - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-min-microamp = <100000>; - regulator-max-microamp = <100000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; regulator-always-on; }; vdd_ldo6: ldo6 { - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - regulator-min-microamp = <200000>; - regulator-max-microamp = <200000>; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; regulator-always-on; }; vdd_ldo7: ldo7 { - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-min-microamp = <200000>; - regulator-max-microamp = <200000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; regulator-always-on; }; vdd_ldo8: ldo8 { - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-min-microamp = <200000>; - regulator-max-microamp = <200000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; regulator-always-on; }; vdd_ld09: ldo9 { regulator-min-microvolt = <1050000>; regulator-max-microvolt = <1050000>; - regulator-min-microamp = <200000>; - regulator-max-microamp = <200000>; + regulator-always-on; }; vdd_ldo10: ldo10 { regulator-min-microvolt = <1000000>; regulator-max-microvolt = <1000000>; - regulator-min-microamp = <300000>; - regulator-max-microamp = <300000>; + regulator-always-on; }; vdd_ldo11: ldo11 { regulator-min-microvolt = <2500000>; regulator-max-microvolt = <2500000>; - regulator-min-microamp = <300000>; - regulator-max-microamp = <300000>; regulator-always-on; }; }; @@ -212,7 +199,7 @@ vdd_ldo11: ldo11 { &qspi0 { status = "okay"; flash@0 { - compatible = "issi,is25wp256", "jedec,spi-nor"; + compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; m25p,fast-read; @@ -252,4 +239,8 @@ &pwm1 { &gpio { status = "okay"; + gpio-line-names = "J29.1", "PMICNTB", "PMICSHDN", "J8.1", "J8.3", + "PCIe_PWREN", "THERM", "UBRDG_RSTN", "PCIe_PERSTN", + "ULPI_RSTN", "J8.2", "UHUB_RSTN", "GEMGXL_RST", "J8.4", + "EN_VDD_SD", "SD_CD"; }; diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 4ebc80315f..f120fcc43d 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -2,6 +2,7 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y +CONFIG_BPF_SYSCALL=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_CGROUPS=y @@ -13,12 +14,14 @@ CONFIG_USER_NS=y CONFIG_CHECKPOINT_RESTORE=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y -CONFIG_BPF_SYSCALL=y +# CONFIG_SYSFS_SYSCALL is not set +CONFIG_SOC_MICROCHIP_POLARFIRE=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y -CONFIG_SOC_MICROCHIP_POLARFIRE=y CONFIG_SMP=y CONFIG_HOTPLUG_CPU=y +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=m CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y @@ -68,13 +71,14 @@ CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y CONFIG_SPI=y CONFIG_SPI_SIFIVE=y +# CONFIG_PTP_1588_CLOCK is not set CONFIG_GPIOLIB=y CONFIG_GPIO_SIFIVE=y -# CONFIG_PTP_1588_CLOCK is not set -CONFIG_POWER_RESET=y -CONFIG_DRM=y -CONFIG_DRM_RADEON=y -CONFIG_DRM_VIRTIO_GPU=y +CONFIG_DRM=m +CONFIG_DRM_RADEON=m +CONFIG_DRM_NOUVEAU=m +CONFIG_DRM_VIRTIO_GPU=m +CONFIG_FB=y CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_USB=y CONFIG_USB_XHCI_HCD=y @@ -85,10 +89,10 @@ CONFIG_USB_OHCI_HCD=y CONFIG_USB_OHCI_HCD_PLATFORM=y CONFIG_USB_STORAGE=y CONFIG_USB_UAS=y +CONFIG_MMC=y CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_PLTFM=y CONFIG_MMC_SDHCI_CADENCE=y -CONFIG_MMC=y CONFIG_MMC_SPI=y CONFIG_RTC_CLASS=y CONFIG_VIRTIO_PCI=y @@ -139,5 +143,3 @@ CONFIG_RCU_EQS_DEBUG=y # CONFIG_FTRACE is not set # CONFIG_RUNTIME_TESTING_MENU is not set CONFIG_MEMTEST=y -# CONFIG_SYSFS_SYSCALL is not set -CONFIG_EFI=y diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index 3b9f83221f..3f42ed87dd 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -73,7 +73,6 @@ CONFIG_LEDS_GPIO=y CONFIG_LEDS_USER=y # CONFIG_VIRTIO_MENU is not set # CONFIG_VHOST_MENU is not set -# CONFIG_SURFACE_PLATFORMS is not set # CONFIG_FILE_LOCKING is not set # CONFIG_DNOTIFY is not set # CONFIG_INOTIFY_USER is not set diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig index 15d1fd0a70..af64b95e88 100644 --- a/arch/riscv/configs/nommu_k210_sdcard_defconfig +++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig @@ -28,7 +28,6 @@ CONFIG_CMDLINE_FORCE=y # CONFIG_SECCOMP is not set # CONFIG_STACKPROTECTOR is not set # CONFIG_GCC_PLUGINS is not set -# CONFIG_BLK_DEV_BSG is not set # CONFIG_MQ_IOSCHED_DEADLINE is not set # CONFIG_MQ_IOSCHED_KYBER is not set CONFIG_BINFMT_FLAT=y @@ -70,7 +69,6 @@ CONFIG_LEDS_GPIO=y CONFIG_LEDS_USER=y # CONFIG_VIRTIO_MENU is not set # CONFIG_VHOST_MENU is not set -# CONFIG_SURFACE_PLATFORMS is not set CONFIG_EXT2_FS=y # CONFIG_FILE_LOCKING is not set # CONFIG_DNOTIFY is not set diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig index f224be6977..e1c9864b62 100644 --- a/arch/riscv/configs/nommu_virt_defconfig +++ b/arch/riscv/configs/nommu_virt_defconfig @@ -24,14 +24,12 @@ CONFIG_EXPERT=y # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set CONFIG_SLOB=y -# CONFIG_SLAB_MERGE_DEFAULT is not set # CONFIG_MMU is not set CONFIG_SOC_VIRT=y CONFIG_SMP=y CONFIG_CMDLINE="root=/dev/vda rw earlycon=uart8250,mmio,0x10000000,115200n8 console=ttyS0" CONFIG_CMDLINE_FORCE=y CONFIG_JUMP_LABEL=y -# CONFIG_BLK_DEV_BSG is not set CONFIG_PARTITION_ADVANCED=y # CONFIG_MSDOS_PARTITION is not set # CONFIG_EFI_PARTITION is not set diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 434ef5b645..8b56a7f1eb 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -2,6 +2,7 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y +CONFIG_BPF_SYSCALL=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_CGROUPS=y @@ -13,12 +14,14 @@ CONFIG_USER_NS=y CONFIG_CHECKPOINT_RESTORE=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y -CONFIG_BPF_SYSCALL=y +# CONFIG_SYSFS_SYSCALL is not set CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y CONFIG_SMP=y CONFIG_HOTPLUG_CPU=y +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=m CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y @@ -67,10 +70,10 @@ CONFIG_HW_RANDOM_VIRTIO=y CONFIG_SPI=y CONFIG_SPI_SIFIVE=y # CONFIG_PTP_1588_CLOCK is not set -CONFIG_POWER_RESET=y CONFIG_DRM=y CONFIG_DRM_RADEON=y CONFIG_DRM_VIRTIO_GPU=y +CONFIG_FB=y CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_USB=y CONFIG_USB_XHCI_HCD=y @@ -130,4 +133,3 @@ CONFIG_RCU_EQS_DEBUG=y # CONFIG_FTRACE is not set # CONFIG_RUNTIME_TESTING_MENU is not set CONFIG_MEMTEST=y -# CONFIG_SYSFS_SYSCALL is not set diff --git a/arch/riscv/errata/alternative.c b/arch/riscv/errata/alternative.c index 3b15885db7..e8b4a0fe48 100644 --- a/arch/riscv/errata/alternative.c +++ b/arch/riscv/errata/alternative.c @@ -22,7 +22,8 @@ static struct cpu_manufacturer_info_t { } cpu_mfr_info; static void (*vendor_patch_func)(struct alt_entry *begin, struct alt_entry *end, - unsigned long archid, unsigned long impid); + unsigned long archid, + unsigned long impid) __initdata; static inline void __init riscv_fill_cpu_mfr_info(void) { diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index 445ccc9730..57b86fd991 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 generic-y += early_ioremap.h -generic-y += extable.h generic-y += flat.h generic-y += kvm_para.h generic-y += user.h diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h index 396a3303c5..3540b69094 100644 --- a/arch/riscv/include/asm/bitops.h +++ b/arch/riscv/include/asm/bitops.h @@ -20,7 +20,6 @@ #include #include #include -#include #include #include diff --git a/arch/riscv/include/asm/cpu_ops.h b/arch/riscv/include/asm/cpu_ops.h index a8ec3c5c1b..134590f1b8 100644 --- a/arch/riscv/include/asm/cpu_ops.h +++ b/arch/riscv/include/asm/cpu_ops.h @@ -40,7 +40,5 @@ struct cpu_operations { extern const struct cpu_operations *cpu_ops[NR_CPUS]; void __init cpu_set_ops(int cpu); -void cpu_update_secondary_bootdata(unsigned int cpuid, - struct task_struct *tidle); #endif /* ifndef __ASM_CPU_OPS_H */ diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 87ac656968..ae711692ee 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -40,14 +40,13 @@ #ifndef CONFIG_64BIT #define SATP_PPN _AC(0x003FFFFF, UL) #define SATP_MODE_32 _AC(0x80000000, UL) -#define SATP_MODE SATP_MODE_32 #define SATP_ASID_BITS 9 #define SATP_ASID_SHIFT 22 #define SATP_ASID_MASK _AC(0x1FF, UL) #else #define SATP_PPN _AC(0x00000FFFFFFFFFFF, UL) #define SATP_MODE_39 _AC(0x8000000000000000, UL) -#define SATP_MODE SATP_MODE_39 +#define SATP_MODE_48 _AC(0x9000000000000000, UL) #define SATP_ASID_BITS 16 #define SATP_ASID_SHIFT 44 #define SATP_ASID_MASK _AC(0xFFFF, UL) @@ -58,22 +57,32 @@ /* Interrupt causes (minus the high bit) */ #define IRQ_S_SOFT 1 +#define IRQ_VS_SOFT 2 #define IRQ_M_SOFT 3 #define IRQ_S_TIMER 5 +#define IRQ_VS_TIMER 6 #define IRQ_M_TIMER 7 #define IRQ_S_EXT 9 +#define IRQ_VS_EXT 10 #define IRQ_M_EXT 11 /* Exception causes */ #define EXC_INST_MISALIGNED 0 #define EXC_INST_ACCESS 1 +#define EXC_INST_ILLEGAL 2 #define EXC_BREAKPOINT 3 #define EXC_LOAD_ACCESS 5 #define EXC_STORE_ACCESS 7 #define EXC_SYSCALL 8 +#define EXC_HYPERVISOR_SYSCALL 9 +#define EXC_SUPERVISOR_SYSCALL 10 #define EXC_INST_PAGE_FAULT 12 #define EXC_LOAD_PAGE_FAULT 13 #define EXC_STORE_PAGE_FAULT 15 +#define EXC_INST_GUEST_PAGE_FAULT 20 +#define EXC_LOAD_GUEST_PAGE_FAULT 21 +#define EXC_VIRTUAL_INST_FAULT 22 +#define EXC_STORE_GUEST_PAGE_FAULT 23 /* PMP configuration */ #define PMP_R 0x01 @@ -85,6 +94,58 @@ #define PMP_A_NAPOT 0x18 #define PMP_L 0x80 +/* HSTATUS flags */ +#ifdef CONFIG_64BIT +#define HSTATUS_VSXL _AC(0x300000000, UL) +#define HSTATUS_VSXL_SHIFT 32 +#endif +#define HSTATUS_VTSR _AC(0x00400000, UL) +#define HSTATUS_VTW _AC(0x00200000, UL) +#define HSTATUS_VTVM _AC(0x00100000, UL) +#define HSTATUS_VGEIN _AC(0x0003f000, UL) +#define HSTATUS_VGEIN_SHIFT 12 +#define HSTATUS_HU _AC(0x00000200, UL) +#define HSTATUS_SPVP _AC(0x00000100, UL) +#define HSTATUS_SPV _AC(0x00000080, UL) +#define HSTATUS_GVA _AC(0x00000040, UL) +#define HSTATUS_VSBE _AC(0x00000020, UL) + +/* HGATP flags */ +#define HGATP_MODE_OFF _AC(0, UL) +#define HGATP_MODE_SV32X4 _AC(1, UL) +#define HGATP_MODE_SV39X4 _AC(8, UL) +#define HGATP_MODE_SV48X4 _AC(9, UL) + +#define HGATP32_MODE_SHIFT 31 +#define HGATP32_VMID_SHIFT 22 +#define HGATP32_VMID_MASK _AC(0x1FC00000, UL) +#define HGATP32_PPN _AC(0x003FFFFF, UL) + +#define HGATP64_MODE_SHIFT 60 +#define HGATP64_VMID_SHIFT 44 +#define HGATP64_VMID_MASK _AC(0x03FFF00000000000, UL) +#define HGATP64_PPN _AC(0x00000FFFFFFFFFFF, UL) + +#define HGATP_PAGE_SHIFT 12 + +#ifdef CONFIG_64BIT +#define HGATP_PPN HGATP64_PPN +#define HGATP_VMID_SHIFT HGATP64_VMID_SHIFT +#define HGATP_VMID_MASK HGATP64_VMID_MASK +#define HGATP_MODE_SHIFT HGATP64_MODE_SHIFT +#else +#define HGATP_PPN HGATP32_PPN +#define HGATP_VMID_SHIFT HGATP32_VMID_SHIFT +#define HGATP_VMID_MASK HGATP32_VMID_MASK +#define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT +#endif + +/* VSIP & HVIP relation */ +#define VSIP_TO_HVIP_SHIFT (IRQ_VS_SOFT - IRQ_S_SOFT) +#define VSIP_VALID_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ + (_AC(1, UL) << IRQ_S_TIMER) | \ + (_AC(1, UL) << IRQ_S_EXT)) + /* symbolic CSR names: */ #define CSR_CYCLE 0xc00 #define CSR_TIME 0xc01 @@ -104,6 +165,31 @@ #define CSR_SIP 0x144 #define CSR_SATP 0x180 +#define CSR_VSSTATUS 0x200 +#define CSR_VSIE 0x204 +#define CSR_VSTVEC 0x205 +#define CSR_VSSCRATCH 0x240 +#define CSR_VSEPC 0x241 +#define CSR_VSCAUSE 0x242 +#define CSR_VSTVAL 0x243 +#define CSR_VSIP 0x244 +#define CSR_VSATP 0x280 + +#define CSR_HSTATUS 0x600 +#define CSR_HEDELEG 0x602 +#define CSR_HIDELEG 0x603 +#define CSR_HIE 0x604 +#define CSR_HTIMEDELTA 0x605 +#define CSR_HCOUNTEREN 0x606 +#define CSR_HGEIE 0x607 +#define CSR_HTIMEDELTAH 0x615 +#define CSR_HTVAL 0x643 +#define CSR_HIP 0x644 +#define CSR_HVIP 0x645 +#define CSR_HTINST 0x64a +#define CSR_HGATP 0x680 +#define CSR_HGEIP 0xe12 + #define CSR_MSTATUS 0x300 #define CSR_MISA 0x301 #define CSR_MIE 0x304 diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h index 54cbf07fb4..58a718573a 100644 --- a/arch/riscv/include/asm/fixmap.h +++ b/arch/riscv/include/asm/fixmap.h @@ -24,6 +24,7 @@ enum fixed_addresses { FIX_HOLE, FIX_PTE, FIX_PMD, + FIX_PUD, FIX_TEXT_POKE1, FIX_TEXT_POKE0, FIX_EARLYCON_MEM_BASE, diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index 1b00badb9f..fc8130f995 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -11,6 +11,7 @@ #include #include #include +#include /* We don't even really need the extable code, but for now keep it simple */ #ifndef CONFIG_MMU @@ -20,23 +21,14 @@ #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ { \ - uintptr_t tmp; \ __enable_user_access(); \ __asm__ __volatile__ ( \ "1: " insn " \n" \ "2: \n" \ - " .section .fixup,\"ax\" \n" \ - " .balign 4 \n" \ - "3: li %[r],%[e] \n" \ - " jump 2b,%[t] \n" \ - " .previous \n" \ - " .section __ex_table,\"a\" \n" \ - " .balign " RISCV_SZPTR " \n" \ - " " RISCV_PTR " 1b, 3b \n" \ - " .previous \n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %[r]) \ : [r] "+r" (ret), [ov] "=&r" (oldval), \ - [u] "+m" (*uaddr), [t] "=&r" (tmp) \ - : [op] "Jr" (oparg), [e] "i" (-EFAULT) \ + [u] "+m" (*uaddr) \ + : [op] "Jr" (oparg) \ : "memory"); \ __disable_user_access(); \ } @@ -98,18 +90,10 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, "2: sc.w.aqrl %[t],%z[nv],%[u] \n" " bnez %[t],1b \n" "3: \n" - " .section .fixup,\"ax\" \n" - " .balign 4 \n" - "4: li %[r],%[e] \n" - " jump 3b,%[t] \n" - " .previous \n" - " .section __ex_table,\"a\" \n" - " .balign " RISCV_SZPTR " \n" - " " RISCV_PTR " 1b, 4b \n" - " " RISCV_PTR " 2b, 4b \n" - " .previous \n" + _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %[r]) \ + _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %[r]) \ : [r] "+r" (ret), [v] "=&r" (val), [u] "+m" (*uaddr), [t] "=&r" (tmp) - : [ov] "Jr" (oldval), [nv] "Jr" (newval), [e] "i" (-EFAULT) + : [ov] "Jr" (oldval), [nv] "Jr" (newval) : "memory"); __disable_user_access(); diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h index b00f503ec1..0b85e363e7 100644 --- a/arch/riscv/include/asm/kasan.h +++ b/arch/riscv/include/asm/kasan.h @@ -27,13 +27,18 @@ */ #define KASAN_SHADOW_SCALE_SHIFT 3 -#define KASAN_SHADOW_SIZE (UL(1) << ((CONFIG_VA_BITS - 1) - KASAN_SHADOW_SCALE_SHIFT)) -#define KASAN_SHADOW_START KERN_VIRT_START -#define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) +#define KASAN_SHADOW_SIZE (UL(1) << ((VA_BITS - 1) - KASAN_SHADOW_SCALE_SHIFT)) +/* + * Depending on the size of the virtual address space, the region may not be + * aligned on PGDIR_SIZE, so force its alignment to ease its population. + */ +#define KASAN_SHADOW_START ((KASAN_SHADOW_END - KASAN_SHADOW_SIZE) & PGDIR_MASK) +#define KASAN_SHADOW_END MODULES_LOWEST_VADDR #define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL) void kasan_init(void); asmlinkage void kasan_early_init(void); +void kasan_swapper_init(void); #endif #endif diff --git a/arch/riscv/include/asm/kprobes.h b/arch/riscv/include/asm/kprobes.h index 9ea9b5ec31..217ef89f22 100644 --- a/arch/riscv/include/asm/kprobes.h +++ b/arch/riscv/include/asm/kprobes.h @@ -40,7 +40,7 @@ void arch_remove_kprobe(struct kprobe *p); int kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr); bool kprobe_breakpoint_handler(struct pt_regs *regs); bool kprobe_single_step_handler(struct pt_regs *regs); -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); void __kprobes *trampoline_probe_handler(struct pt_regs *regs); #endif /* CONFIG_KPROBES */ diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 109c97e991..004372f8da 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -31,9 +31,20 @@ * When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ +#ifdef CONFIG_64BIT +#ifdef CONFIG_MMU +#define PAGE_OFFSET kernel_map.page_offset +#else #define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) - -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#endif +/* + * By default, CONFIG_PAGE_OFFSET value corresponds to SV48 address space so + * define the PAGE_OFFSET value for SV39. + */ +#define PAGE_OFFSET_L3 _AC(0xffffffd800000000, UL) +#else +#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#endif /* CONFIG_64BIT */ #ifndef __ASSEMBLY__ @@ -86,6 +97,7 @@ extern unsigned long riscv_pfn_base; #endif /* CONFIG_MMU */ struct kernel_mapping { + unsigned long page_offset; unsigned long virt_addr; uintptr_t phys_addr; uintptr_t size; @@ -107,7 +119,7 @@ extern phys_addr_t phys_ram_base; ((x) >= kernel_map.virt_addr && (x) < (kernel_map.virt_addr + kernel_map.size)) #define is_linear_mapping(x) \ - ((x) >= PAGE_OFFSET && (!IS_ENABLED(CONFIG_64BIT) || (x) < kernel_map.virt_addr)) + ((x) >= PAGE_OFFSET && (!IS_ENABLED(CONFIG_64BIT) || (x) < PAGE_OFFSET + KERN_VIRT_SIZE)) #define linear_mapping_pa_to_va(x) ((void *)((unsigned long)(x) + kernel_map.va_pa_offset)) #define kernel_mapping_pa_to_va(y) ({ \ @@ -157,6 +169,8 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x); #define page_to_bus(page) (page_to_phys(page)) #define phys_to_page(paddr) (pfn_to_page(phys_to_pfn(paddr))) +#define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) + #ifdef CONFIG_FLATMEM #define pfn_valid(pfn) \ (((pfn) >= ARCH_PFN_OFFSET) && (((pfn) - ARCH_PFN_OFFSET) < max_mapnr)) diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 0af6933a71..11823004b8 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -11,6 +11,8 @@ #include #ifdef CONFIG_MMU +#define __HAVE_ARCH_PUD_ALLOC_ONE +#define __HAVE_ARCH_PUD_FREE #include static inline void pmd_populate_kernel(struct mm_struct *mm, @@ -36,6 +38,44 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) set_pud(pud, __pud((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); } + +static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) +{ + if (pgtable_l4_enabled) { + unsigned long pfn = virt_to_pfn(pud); + + set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); + } +} + +static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, + pud_t *pud) +{ + if (pgtable_l4_enabled) { + unsigned long pfn = virt_to_pfn(pud); + + set_p4d_safe(p4d, + __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); + } +} + +#define pud_alloc_one pud_alloc_one +static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + if (pgtable_l4_enabled) + return __pud_alloc_one(mm, addr); + + return NULL; +} + +#define pud_free pud_free +static inline void pud_free(struct mm_struct *mm, pud_t *pud) +{ + if (pgtable_l4_enabled) + __pud_free(mm, pud); +} + +#define __pud_free_tlb(tlb, pud, addr) pud_free((tlb)->mm, pud) #endif /* __PAGETABLE_PMD_FOLDED */ static inline pgd_t *pgd_alloc(struct mm_struct *mm) diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 228261aa96..bbbdd66e5e 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -8,16 +8,36 @@ #include -#define PGDIR_SHIFT 30 +extern bool pgtable_l4_enabled; + +#define PGDIR_SHIFT_L3 30 +#define PGDIR_SHIFT_L4 39 +#define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3) + +#define PGDIR_SHIFT (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3) /* Size of region mapped by a page global directory */ #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE - 1)) +/* pud is folded into pgd in case of 3-level page table */ +#define PUD_SHIFT 30 +#define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) +#define PUD_MASK (~(PUD_SIZE - 1)) + #define PMD_SHIFT 21 /* Size of region mapped by a page middle directory */ #define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE - 1)) +/* Page Upper Directory entry */ +typedef struct { + unsigned long pud; +} pud_t; + +#define pud_val(x) ((x).pud) +#define __pud(x) ((pud_t) { (x) }) +#define PTRS_PER_PUD (PAGE_SIZE / sizeof(pud_t)) + /* Page Middle Directory entry */ typedef struct { unsigned long pmd; @@ -59,6 +79,16 @@ static inline void pud_clear(pud_t *pudp) set_pud(pudp, __pud(0)); } +static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot) +{ + return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); +} + +static inline unsigned long _pud_pfn(pud_t pud) +{ + return pud_val(pud) >> _PAGE_PFN_SHIFT; +} + static inline pmd_t *pud_pgtable(pud_t pud) { return (pmd_t *)pfn_to_virt(pud_val(pud) >> _PAGE_PFN_SHIFT); @@ -69,6 +99,17 @@ static inline struct page *pud_page(pud_t pud) return pfn_to_page(pud_val(pud) >> _PAGE_PFN_SHIFT); } +#define mm_pud_folded mm_pud_folded +static inline bool mm_pud_folded(struct mm_struct *mm) +{ + if (pgtable_l4_enabled) + return false; + + return true; +} + +#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) + static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) { return __pmd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); @@ -84,4 +125,69 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) #define pmd_ERROR(e) \ pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) +#define pud_ERROR(e) \ + pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) + +static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) +{ + if (pgtable_l4_enabled) + *p4dp = p4d; + else + set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); +} + +static inline int p4d_none(p4d_t p4d) +{ + if (pgtable_l4_enabled) + return (p4d_val(p4d) == 0); + + return 0; +} + +static inline int p4d_present(p4d_t p4d) +{ + if (pgtable_l4_enabled) + return (p4d_val(p4d) & _PAGE_PRESENT); + + return 1; +} + +static inline int p4d_bad(p4d_t p4d) +{ + if (pgtable_l4_enabled) + return !p4d_present(p4d); + + return 0; +} + +static inline void p4d_clear(p4d_t *p4d) +{ + if (pgtable_l4_enabled) + set_p4d(p4d, __p4d(0)); +} + +static inline pud_t *p4d_pgtable(p4d_t p4d) +{ + if (pgtable_l4_enabled) + return (pud_t *)pfn_to_virt(p4d_val(p4d) >> _PAGE_PFN_SHIFT); + + return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) }); +} + +static inline struct page *p4d_page(p4d_t p4d) +{ + return pfn_to_page(p4d_val(p4d) >> _PAGE_PFN_SHIFT); +} + +#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) + +#define pud_offset pud_offset +static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) +{ + if (pgtable_l4_enabled) + return p4d_pgtable(*p4d) + pud_index(address); + + return (pud_t *)p4d; +} + #endif /* _ASM_RISCV_PGTABLE_64_H */ diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index 2ee4139129..a6b0c89824 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -31,7 +31,7 @@ * _PAGE_PROT_NONE is set on not-present pages (and ignored by the hardware) to * distinguish them from swapped out pages */ -#define _PAGE_PROT_NONE _PAGE_READ +#define _PAGE_PROT_NONE _PAGE_GLOBAL #define _PAGE_PFN_SHIFT 10 diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 39b550310e..e3549e50de 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -13,6 +13,7 @@ #ifndef CONFIG_MMU #define KERNEL_LINK_ADDR PAGE_OFFSET +#define KERN_VIRT_SIZE (UL(-1)) #else #define ADDRESS_SPACE_END (UL(-1)) @@ -24,8 +25,19 @@ #define KERNEL_LINK_ADDR PAGE_OFFSET #endif +/* Number of entries in the page global directory */ +#define PTRS_PER_PGD (PAGE_SIZE / sizeof(pgd_t)) +/* Number of entries in the page table */ +#define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t)) + +/* + * Half of the kernel address space (half of the entries of the page global + * directory) is for the direct mapping. + */ +#define KERN_VIRT_SIZE ((PTRS_PER_PGD / 2 * PGDIR_SIZE) / 2) + #define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1) -#define VMALLOC_END (PAGE_OFFSET - 1) +#define VMALLOC_END PAGE_OFFSET #define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE) #define BPF_JIT_REGION_SIZE (SZ_128M) @@ -39,8 +51,10 @@ /* Modules always live before the kernel */ #ifdef CONFIG_64BIT -#define MODULES_VADDR (PFN_ALIGN((unsigned long)&_end) - SZ_2G) -#define MODULES_END (PFN_ALIGN((unsigned long)&_start)) +/* This is used to define the end of the KASAN shadow region */ +#define MODULES_LOWEST_VADDR (KERNEL_LINK_ADDR - SZ_2G) +#define MODULES_VADDR (PFN_ALIGN((unsigned long)&_end) - SZ_2G) +#define MODULES_END (PFN_ALIGN((unsigned long)&_start)) #endif /* @@ -48,10 +62,16 @@ * struct pages to map half the virtual address space. Then * position vmemmap directly below the VMALLOC region. */ +#ifdef CONFIG_64BIT +#define VA_BITS (pgtable_l4_enabled ? 48 : 39) +#else +#define VA_BITS 32 +#endif + #define VMEMMAP_SHIFT \ - (CONFIG_VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT) + (VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT) #define VMEMMAP_SIZE BIT(VMEMMAP_SHIFT) -#define VMEMMAP_END (VMALLOC_START - 1) +#define VMEMMAP_END VMALLOC_START #define VMEMMAP_START (VMALLOC_START - VMEMMAP_SIZE) /* @@ -75,15 +95,15 @@ #endif #ifdef CONFIG_XIP_KERNEL -#define XIP_OFFSET SZ_8M +#define XIP_OFFSET SZ_32M +#define XIP_OFFSET_MASK (SZ_32M - 1) #else #define XIP_OFFSET 0 #endif #ifndef __ASSEMBLY__ -/* Page Upper Directory not used in RISC-V */ -#include +#include #include #include #include @@ -97,7 +117,8 @@ #ifdef CONFIG_XIP_KERNEL #define XIP_FIXUP(addr) ({ \ uintptr_t __a = (uintptr_t)(addr); \ - (__a >= CONFIG_XIP_PHYS_ADDR && __a < CONFIG_XIP_PHYS_ADDR + SZ_16M) ? \ + (__a >= CONFIG_XIP_PHYS_ADDR && \ + __a < CONFIG_XIP_PHYS_ADDR + XIP_OFFSET * 2) ? \ __a - CONFIG_XIP_PHYS_ADDR + CONFIG_PHYS_RAM_BASE - XIP_OFFSET :\ __a; \ }) @@ -105,19 +126,27 @@ #define XIP_FIXUP(addr) (addr) #endif /* CONFIG_XIP_KERNEL */ -#ifdef CONFIG_MMU -/* Number of entries in the page global directory */ -#define PTRS_PER_PGD (PAGE_SIZE / sizeof(pgd_t)) -/* Number of entries in the page table */ -#define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t)) +struct pt_alloc_ops { + pte_t *(*get_pte_virt)(phys_addr_t pa); + phys_addr_t (*alloc_pte)(uintptr_t va); +#ifndef __PAGETABLE_PMD_FOLDED + pmd_t *(*get_pmd_virt)(phys_addr_t pa); + phys_addr_t (*alloc_pmd)(uintptr_t va); + pud_t *(*get_pud_virt)(phys_addr_t pa); + phys_addr_t (*alloc_pud)(uintptr_t va); +#endif +}; +extern struct pt_alloc_ops pt_ops __initdata; + +#ifdef CONFIG_MMU /* Number of PGD entries that a user-mode program can use */ #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) /* Page protection bits */ #define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER) -#define PAGE_NONE __pgprot(_PAGE_PROT_NONE) +#define PAGE_NONE __pgprot(_PAGE_PROT_NONE | _PAGE_READ) #define PAGE_READ __pgprot(_PAGE_BASE | _PAGE_READ) #define PAGE_WRITE __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_WRITE) #define PAGE_EXEC __pgprot(_PAGE_BASE | _PAGE_EXEC) @@ -626,11 +655,12 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, * * Format of swap PTE: * bit 0: _PAGE_PRESENT (zero) - * bit 1: _PAGE_PROT_NONE (zero) - * bits 2 to 6: swap type - * bits 7 to XLEN-1: swap offset + * bit 1 to 3: _PAGE_LEAF (zero) + * bit 5: _PAGE_PROT_NONE (zero) + * bits 6 to 10: swap type + * bits 10 to XLEN-1: swap offset */ -#define __SWP_TYPE_SHIFT 2 +#define __SWP_TYPE_SHIFT 6 #define __SWP_TYPE_BITS 5 #define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1) #define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) @@ -646,12 +676,17 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) +#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) +#define __swp_entry_to_pmd(swp) __pmd((swp).val) +#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ + /* * In the RV64 Linux scheme, we give the user half of the virtual-address space * and give the kernel the other (upper) half. */ #ifdef CONFIG_64BIT -#define KERN_VIRT_START (-(BIT(CONFIG_VA_BITS)) + TASK_SIZE) +#define KERN_VIRT_START (-(BIT(VA_BITS)) + TASK_SIZE) #else #define KERN_VIRT_START FIXADDR_START #endif @@ -659,11 +694,22 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, /* * Task size is 0x4000000000 for RV64 or 0x9fc00000 for RV32. * Note that PGDIR_SIZE must evenly divide TASK_SIZE. + * Task size is: + * - 0x9fc00000 (~2.5GB) for RV32. + * - 0x4000000000 ( 256GB) for RV64 using SV39 mmu + * - 0x800000000000 ( 128TB) for RV64 using SV48 mmu + * + * Note that PGDIR_SIZE must evenly divide TASK_SIZE since "RISC-V + * Instruction Set Manual Volume II: Privileged Architecture" states that + * "load and store effective addresses, which are 64bits, must have bits + * 63–48 all equal to bit 47, or else a page-fault exception will occur." */ #ifdef CONFIG_64BIT -#define TASK_SIZE (PGDIR_SIZE * PTRS_PER_PGD / 2) +#define TASK_SIZE (PGDIR_SIZE * PTRS_PER_PGD / 2) +#define TASK_SIZE_MIN (PGDIR_SIZE_L3 * PTRS_PER_PGD / 2) #else -#define TASK_SIZE FIXADDR_START +#define TASK_SIZE FIXADDR_START +#define TASK_SIZE_MIN TASK_SIZE #endif #else /* CONFIG_MMU */ @@ -689,6 +735,8 @@ extern uintptr_t _dtb_early_pa; #define dtb_early_va _dtb_early_va #define dtb_early_pa _dtb_early_pa #endif /* CONFIG_XIP_KERNEL */ +extern u64 satp_mode; +extern bool pgtable_l4_enabled; void paging_init(void); void misc_mem_init(void); diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 46b492c78c..0749924d9e 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -66,7 +66,7 @@ static inline void release_thread(struct task_struct *dead_task) { } -extern unsigned long get_wchan(struct task_struct *p); +extern unsigned long __get_wchan(struct task_struct *p); static inline void wait_for_interrupt(void) diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 0d42693cb6..d1c37479d8 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -8,6 +8,7 @@ #define _ASM_RISCV_SBI_H #include +#include #ifdef CONFIG_RISCV_SBI enum sbi_ext_id { @@ -27,6 +28,15 @@ enum sbi_ext_id { SBI_EXT_IPI = 0x735049, SBI_EXT_RFENCE = 0x52464E43, SBI_EXT_HSM = 0x48534D, + SBI_EXT_SRST = 0x53525354, + + /* Experimentals extensions must lie within this range */ + SBI_EXT_EXPERIMENTAL_START = 0x08000000, + SBI_EXT_EXPERIMENTAL_END = 0x08FFFFFF, + + /* Vendor extensions must lie within this range */ + SBI_EXT_VENDOR_START = 0x09000000, + SBI_EXT_VENDOR_END = 0x09FFFFFF, }; enum sbi_ext_base_fid { @@ -70,6 +80,21 @@ enum sbi_hsm_hart_status { SBI_HSM_HART_STATUS_STOP_PENDING, }; +enum sbi_ext_srst_fid { + SBI_EXT_SRST_RESET = 0, +}; + +enum sbi_srst_reset_type { + SBI_SRST_RESET_TYPE_SHUTDOWN = 0, + SBI_SRST_RESET_TYPE_COLD_REBOOT, + SBI_SRST_RESET_TYPE_WARM_REBOOT, +}; + +enum sbi_srst_reset_reason { + SBI_SRST_RESET_REASON_NONE = 0, + SBI_SRST_RESET_REASON_SYS_FAILURE, +}; + #define SBI_SPEC_VERSION_DEFAULT 0x1 #define SBI_SPEC_VERSION_MAJOR_SHIFT 24 #define SBI_SPEC_VERSION_MAJOR_MASK 0x7f @@ -82,6 +107,7 @@ enum sbi_hsm_hart_status { #define SBI_ERR_INVALID_PARAM -3 #define SBI_ERR_DENIED -4 #define SBI_ERR_INVALID_ADDRESS -5 +#define SBI_ERR_ALREADY_AVAILABLE -6 extern unsigned long sbi_spec_version; struct sbiret { @@ -103,27 +129,27 @@ long sbi_get_mimpid(void); void sbi_set_timer(uint64_t stime_value); void sbi_shutdown(void); void sbi_clear_ipi(void); -int sbi_send_ipi(const unsigned long *hart_mask); -int sbi_remote_fence_i(const unsigned long *hart_mask); -int sbi_remote_sfence_vma(const unsigned long *hart_mask, +int sbi_send_ipi(const struct cpumask *cpu_mask); +int sbi_remote_fence_i(const struct cpumask *cpu_mask); +int sbi_remote_sfence_vma(const struct cpumask *cpu_mask, unsigned long start, unsigned long size); -int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, +int sbi_remote_sfence_vma_asid(const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long asid); -int sbi_remote_hfence_gvma(const unsigned long *hart_mask, +int sbi_remote_hfence_gvma(const struct cpumask *cpu_mask, unsigned long start, unsigned long size); -int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask, +int sbi_remote_hfence_gvma_vmid(const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long vmid); -int sbi_remote_hfence_vvma(const unsigned long *hart_mask, +int sbi_remote_hfence_vvma(const struct cpumask *cpu_mask, unsigned long start, unsigned long size); -int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask, +int sbi_remote_hfence_vvma_asid(const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long asid); @@ -148,9 +174,17 @@ static inline unsigned long sbi_minor_version(void) return sbi_spec_version & SBI_SPEC_VERSION_MINOR_MASK; } +/* Make SBI version */ +static inline unsigned long sbi_mk_version(unsigned long major, + unsigned long minor) +{ + return ((major & SBI_SPEC_VERSION_MAJOR_MASK) << + SBI_SPEC_VERSION_MAJOR_SHIFT) | minor; +} + int sbi_err_map_linux_errno(int err); #else /* CONFIG_RISCV_SBI */ -static inline int sbi_remote_fence_i(const unsigned long *hart_mask) { return -1; } +static inline int sbi_remote_fence_i(const struct cpumask *cpu_mask) { return -1; } static inline void sbi_init(void) {} #endif /* CONFIG_RISCV_SBI */ #endif /* _ASM_RISCV_SBI_H */ diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index 62d0e6e61d..23170c933d 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -62,8 +62,6 @@ asmlinkage void smp_callin(void); #if defined CONFIG_HOTPLUG_CPU int __cpu_disable(void); void __cpu_die(unsigned int cpu); -void cpu_stop(void); -#else #endif /* CONFIG_HOTPLUG_CPU */ #else @@ -94,8 +92,6 @@ static inline void riscv_clear_ipi(void) #endif /* CONFIG_SMP */ -void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out); - #if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP) bool cpu_has_hotplug(unsigned int cpu); #else diff --git a/arch/riscv/include/asm/sparsemem.h b/arch/riscv/include/asm/sparsemem.h index 45a7018a81..63acaecc33 100644 --- a/arch/riscv/include/asm/sparsemem.h +++ b/arch/riscv/include/asm/sparsemem.h @@ -4,7 +4,11 @@ #define _ASM_RISCV_SPARSEMEM_H #ifdef CONFIG_SPARSEMEM -#define MAX_PHYSMEM_BITS CONFIG_PA_BITS +#ifdef CONFIG_64BIT +#define MAX_PHYSMEM_BITS 56 +#else +#define MAX_PHYSMEM_BITS 34 +#endif /* CONFIG_64BIT */ #define SECTION_SIZE_BITS 27 #endif /* CONFIG_SPARSEMEM */ diff --git a/arch/riscv/include/asm/spinlock_types.h b/arch/riscv/include/asm/spinlock_types.h index f398e7638d..5a35a49505 100644 --- a/arch/riscv/include/asm/spinlock_types.h +++ b/arch/riscv/include/asm/spinlock_types.h @@ -6,7 +6,7 @@ #ifndef _ASM_RISCV_SPINLOCK_TYPES_H #define _ASM_RISCV_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/riscv/include/asm/syscall.h b/arch/riscv/include/asm/syscall.h index 34fbb3ea21..7ac6a0e275 100644 --- a/arch/riscv/include/asm/syscall.h +++ b/arch/riscv/include/asm/syscall.h @@ -64,15 +64,6 @@ static inline void syscall_get_arguments(struct task_struct *task, memcpy(args, ®s->a1, 5 * sizeof(args[0])); } -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - const unsigned long *args) -{ - regs->orig_a0 = args[0]; - args++; - memcpy(®s->a1, args, 5 * sizeof(regs->a1)); -} - static inline int syscall_get_arch(struct task_struct *task) { #ifdef CONFIG_64BIT diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index f314ff44c4..c701a5e57a 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -8,6 +8,7 @@ #ifndef _ASM_RISCV_UACCESS_H #define _ASM_RISCV_UACCESS_H +#include #include /* for TASK_SIZE */ /* @@ -80,25 +81,14 @@ static inline int __access_ok(unsigned long addr, unsigned long size) #define __get_user_asm(insn, x, ptr, err) \ do { \ - uintptr_t __tmp; \ __typeof__(x) __x; \ __asm__ __volatile__ ( \ "1:\n" \ - " " insn " %1, %3\n" \ + " " insn " %1, %2\n" \ "2:\n" \ - " .section .fixup,\"ax\"\n" \ - " .balign 4\n" \ - "3:\n" \ - " li %0, %4\n" \ - " li %1, 0\n" \ - " jump 2b, %2\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 3b\n" \ - " .previous" \ - : "+r" (err), "=&r" (__x), "=r" (__tmp) \ - : "m" (*(ptr)), "i" (-EFAULT)); \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %0, %1) \ + : "+r" (err), "=&r" (__x) \ + : "m" (*(ptr))); \ (x) = __x; \ } while (0) @@ -110,30 +100,18 @@ do { \ do { \ u32 __user *__ptr = (u32 __user *)(ptr); \ u32 __lo, __hi; \ - uintptr_t __tmp; \ __asm__ __volatile__ ( \ "1:\n" \ - " lw %1, %4\n" \ + " lw %1, %3\n" \ "2:\n" \ - " lw %2, %5\n" \ + " lw %2, %4\n" \ "3:\n" \ - " .section .fixup,\"ax\"\n" \ - " .balign 4\n" \ - "4:\n" \ - " li %0, %6\n" \ - " li %1, 0\n" \ - " li %2, 0\n" \ - " jump 3b, %3\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 4b\n" \ - " " RISCV_PTR " 2b, 4b\n" \ - " .previous" \ - : "+r" (err), "=&r" (__lo), "=r" (__hi), \ - "=r" (__tmp) \ - : "m" (__ptr[__LSW]), "m" (__ptr[__MSW]), \ - "i" (-EFAULT)); \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 3b, %0, %1) \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(2b, 3b, %0, %1) \ + : "+r" (err), "=&r" (__lo), "=r" (__hi) \ + : "m" (__ptr[__LSW]), "m" (__ptr[__MSW])); \ + if (err) \ + __hi = 0; \ (x) = (__typeof__(x))((__typeof__((x)-(x)))( \ (((u64)__hi << 32) | __lo))); \ } while (0) @@ -221,24 +199,14 @@ do { \ #define __put_user_asm(insn, x, ptr, err) \ do { \ - uintptr_t __tmp; \ __typeof__(*(ptr)) __x = x; \ __asm__ __volatile__ ( \ "1:\n" \ - " " insn " %z3, %2\n" \ + " " insn " %z2, %1\n" \ "2:\n" \ - " .section .fixup,\"ax\"\n" \ - " .balign 4\n" \ - "3:\n" \ - " li %0, %4\n" \ - " jump 2b, %1\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 3b\n" \ - " .previous" \ - : "+r" (err), "=r" (__tmp), "=m" (*(ptr)) \ - : "rJ" (__x), "i" (-EFAULT)); \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %0) \ + : "+r" (err), "=m" (*(ptr)) \ + : "rJ" (__x)); \ } while (0) #ifdef CONFIG_64BIT @@ -249,28 +217,18 @@ do { \ do { \ u32 __user *__ptr = (u32 __user *)(ptr); \ u64 __x = (__typeof__((x)-(x)))(x); \ - uintptr_t __tmp; \ __asm__ __volatile__ ( \ "1:\n" \ - " sw %z4, %2\n" \ + " sw %z3, %1\n" \ "2:\n" \ - " sw %z5, %3\n" \ + " sw %z4, %2\n" \ "3:\n" \ - " .section .fixup,\"ax\"\n" \ - " .balign 4\n" \ - "4:\n" \ - " li %0, %6\n" \ - " jump 3b, %1\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 4b\n" \ - " " RISCV_PTR " 2b, 4b\n" \ - " .previous" \ - : "+r" (err), "=r" (__tmp), \ + _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %0) \ + _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %0) \ + : "+r" (err), \ "=m" (__ptr[__LSW]), \ "=m" (__ptr[__MSW]) \ - : "rJ" (__x), "rJ" (__x >> 32), "i" (-EFAULT)); \ + : "rJ" (__x), "rJ" (__x >> 32)); \ } while (0) #endif /* CONFIG_64BIT */ @@ -388,81 +346,6 @@ unsigned long __must_check clear_user(void __user *to, unsigned long n) __clear_user(to, n) : n; } -/* - * Atomic compare-and-exchange, but with a fixup for userspace faults. Faults - * will set "err" to -EFAULT, while successful accesses return the previous - * value. - */ -#define __cmpxchg_user(ptr, old, new, err, size, lrb, scb) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - __typeof__(*(ptr)) __old = (old); \ - __typeof__(*(ptr)) __new = (new); \ - __typeof__(*(ptr)) __ret; \ - __typeof__(err) __err = 0; \ - register unsigned int __rc; \ - __enable_user_access(); \ - switch (size) { \ - case 4: \ - __asm__ __volatile__ ( \ - "0:\n" \ - " lr.w" #scb " %[ret], %[ptr]\n" \ - " bne %[ret], %z[old], 1f\n" \ - " sc.w" #lrb " %[rc], %z[new], %[ptr]\n" \ - " bnez %[rc], 0b\n" \ - "1:\n" \ - ".section .fixup,\"ax\"\n" \ - ".balign 4\n" \ - "2:\n" \ - " li %[err], %[efault]\n" \ - " jump 1b, %[rc]\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - ".balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 2b\n" \ - ".previous\n" \ - : [ret] "=&r" (__ret), \ - [rc] "=&r" (__rc), \ - [ptr] "+A" (*__ptr), \ - [err] "=&r" (__err) \ - : [old] "rJ" (__old), \ - [new] "rJ" (__new), \ - [efault] "i" (-EFAULT)); \ - break; \ - case 8: \ - __asm__ __volatile__ ( \ - "0:\n" \ - " lr.d" #scb " %[ret], %[ptr]\n" \ - " bne %[ret], %z[old], 1f\n" \ - " sc.d" #lrb " %[rc], %z[new], %[ptr]\n" \ - " bnez %[rc], 0b\n" \ - "1:\n" \ - ".section .fixup,\"ax\"\n" \ - ".balign 4\n" \ - "2:\n" \ - " li %[err], %[efault]\n" \ - " jump 1b, %[rc]\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - ".balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 2b\n" \ - ".previous\n" \ - : [ret] "=&r" (__ret), \ - [rc] "=&r" (__rc), \ - [ptr] "+A" (*__ptr), \ - [err] "=&r" (__err) \ - : [old] "rJ" (__old), \ - [new] "rJ" (__new), \ - [efault] "i" (-EFAULT)); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ - __disable_user_access(); \ - (err) = __err; \ - __ret; \ -}) - #define HAVE_GET_KERNEL_NOFAULT #define __get_kernel_nofault(dst, src, type, err_label) \ diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index 208e31bc5d..bc6f75f3a1 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -8,30 +8,19 @@ #ifndef _ASM_RISCV_VDSO_H #define _ASM_RISCV_VDSO_H - /* * All systems with an MMU have a VDSO, but systems without an MMU don't * support shared libraries and therefor don't have one. */ #ifdef CONFIG_MMU -#include -/* - * All systems with an MMU have a VDSO, but systems without an MMU don't - * support shared libraries and therefor don't have one. - */ -#ifdef CONFIG_MMU - -#define __VVAR_PAGES 1 +#define __VVAR_PAGES 2 #ifndef __ASSEMBLY__ #include #define VDSO_SYMBOL(base, name) \ (void __user *)((unsigned long)(base) + __vdso_##name##_offset) - -#endif /* CONFIG_MMU */ - #endif /* !__ASSEMBLY__ */ #endif /* CONFIG_MMU */ diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h index f839f16e0d..77d9c2f721 100644 --- a/arch/riscv/include/asm/vdso/gettimeofday.h +++ b/arch/riscv/include/asm/vdso/gettimeofday.h @@ -76,6 +76,13 @@ static __always_inline const struct vdso_data *__arch_get_vdso_data(void) return _vdso_data; } +#ifdef CONFIG_TIME_NS +static __always_inline +const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) +{ + return _timens_data; +} +#endif #endif /* !__ASSEMBLY__ */ #endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 16308ef1e5..ffc87e76b1 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -43,7 +43,8 @@ obj-$(CONFIG_FPU) += fpu.o obj-$(CONFIG_SMP) += smpboot.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_SMP) += cpu_ops.o -obj-$(CONFIG_SMP) += cpu_ops_spinwait.o + +obj-$(CONFIG_RISCV_BOOT_SPINWAIT) += cpu_ops_spinwait.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_MODULE_SECTIONS) += module-sections.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index 90f8ce64fa..df0519a64e 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -7,9 +7,12 @@ #define GENERATING_ASM_OFFSETS #include +#include #include +#include #include #include +#include void asm_offsets(void); @@ -33,7 +36,6 @@ void asm_offsets(void) OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp); - OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu); OFFSET(TASK_THREAD_F0, task_struct, thread.fstate.f[0]); OFFSET(TASK_THREAD_F1, task_struct, thread.fstate.f[1]); @@ -111,6 +113,160 @@ void asm_offsets(void) OFFSET(PT_BADADDR, pt_regs, badaddr); OFFSET(PT_CAUSE, pt_regs, cause); + OFFSET(KVM_ARCH_GUEST_ZERO, kvm_vcpu_arch, guest_context.zero); + OFFSET(KVM_ARCH_GUEST_RA, kvm_vcpu_arch, guest_context.ra); + OFFSET(KVM_ARCH_GUEST_SP, kvm_vcpu_arch, guest_context.sp); + OFFSET(KVM_ARCH_GUEST_GP, kvm_vcpu_arch, guest_context.gp); + OFFSET(KVM_ARCH_GUEST_TP, kvm_vcpu_arch, guest_context.tp); + OFFSET(KVM_ARCH_GUEST_T0, kvm_vcpu_arch, guest_context.t0); + OFFSET(KVM_ARCH_GUEST_T1, kvm_vcpu_arch, guest_context.t1); + OFFSET(KVM_ARCH_GUEST_T2, kvm_vcpu_arch, guest_context.t2); + OFFSET(KVM_ARCH_GUEST_S0, kvm_vcpu_arch, guest_context.s0); + OFFSET(KVM_ARCH_GUEST_S1, kvm_vcpu_arch, guest_context.s1); + OFFSET(KVM_ARCH_GUEST_A0, kvm_vcpu_arch, guest_context.a0); + OFFSET(KVM_ARCH_GUEST_A1, kvm_vcpu_arch, guest_context.a1); + OFFSET(KVM_ARCH_GUEST_A2, kvm_vcpu_arch, guest_context.a2); + OFFSET(KVM_ARCH_GUEST_A3, kvm_vcpu_arch, guest_context.a3); + OFFSET(KVM_ARCH_GUEST_A4, kvm_vcpu_arch, guest_context.a4); + OFFSET(KVM_ARCH_GUEST_A5, kvm_vcpu_arch, guest_context.a5); + OFFSET(KVM_ARCH_GUEST_A6, kvm_vcpu_arch, guest_context.a6); + OFFSET(KVM_ARCH_GUEST_A7, kvm_vcpu_arch, guest_context.a7); + OFFSET(KVM_ARCH_GUEST_S2, kvm_vcpu_arch, guest_context.s2); + OFFSET(KVM_ARCH_GUEST_S3, kvm_vcpu_arch, guest_context.s3); + OFFSET(KVM_ARCH_GUEST_S4, kvm_vcpu_arch, guest_context.s4); + OFFSET(KVM_ARCH_GUEST_S5, kvm_vcpu_arch, guest_context.s5); + OFFSET(KVM_ARCH_GUEST_S6, kvm_vcpu_arch, guest_context.s6); + OFFSET(KVM_ARCH_GUEST_S7, kvm_vcpu_arch, guest_context.s7); + OFFSET(KVM_ARCH_GUEST_S8, kvm_vcpu_arch, guest_context.s8); + OFFSET(KVM_ARCH_GUEST_S9, kvm_vcpu_arch, guest_context.s9); + OFFSET(KVM_ARCH_GUEST_S10, kvm_vcpu_arch, guest_context.s10); + OFFSET(KVM_ARCH_GUEST_S11, kvm_vcpu_arch, guest_context.s11); + OFFSET(KVM_ARCH_GUEST_T3, kvm_vcpu_arch, guest_context.t3); + OFFSET(KVM_ARCH_GUEST_T4, kvm_vcpu_arch, guest_context.t4); + OFFSET(KVM_ARCH_GUEST_T5, kvm_vcpu_arch, guest_context.t5); + OFFSET(KVM_ARCH_GUEST_T6, kvm_vcpu_arch, guest_context.t6); + OFFSET(KVM_ARCH_GUEST_SEPC, kvm_vcpu_arch, guest_context.sepc); + OFFSET(KVM_ARCH_GUEST_SSTATUS, kvm_vcpu_arch, guest_context.sstatus); + OFFSET(KVM_ARCH_GUEST_HSTATUS, kvm_vcpu_arch, guest_context.hstatus); + OFFSET(KVM_ARCH_GUEST_SCOUNTEREN, kvm_vcpu_arch, guest_csr.scounteren); + + OFFSET(KVM_ARCH_HOST_ZERO, kvm_vcpu_arch, host_context.zero); + OFFSET(KVM_ARCH_HOST_RA, kvm_vcpu_arch, host_context.ra); + OFFSET(KVM_ARCH_HOST_SP, kvm_vcpu_arch, host_context.sp); + OFFSET(KVM_ARCH_HOST_GP, kvm_vcpu_arch, host_context.gp); + OFFSET(KVM_ARCH_HOST_TP, kvm_vcpu_arch, host_context.tp); + OFFSET(KVM_ARCH_HOST_T0, kvm_vcpu_arch, host_context.t0); + OFFSET(KVM_ARCH_HOST_T1, kvm_vcpu_arch, host_context.t1); + OFFSET(KVM_ARCH_HOST_T2, kvm_vcpu_arch, host_context.t2); + OFFSET(KVM_ARCH_HOST_S0, kvm_vcpu_arch, host_context.s0); + OFFSET(KVM_ARCH_HOST_S1, kvm_vcpu_arch, host_context.s1); + OFFSET(KVM_ARCH_HOST_A0, kvm_vcpu_arch, host_context.a0); + OFFSET(KVM_ARCH_HOST_A1, kvm_vcpu_arch, host_context.a1); + OFFSET(KVM_ARCH_HOST_A2, kvm_vcpu_arch, host_context.a2); + OFFSET(KVM_ARCH_HOST_A3, kvm_vcpu_arch, host_context.a3); + OFFSET(KVM_ARCH_HOST_A4, kvm_vcpu_arch, host_context.a4); + OFFSET(KVM_ARCH_HOST_A5, kvm_vcpu_arch, host_context.a5); + OFFSET(KVM_ARCH_HOST_A6, kvm_vcpu_arch, host_context.a6); + OFFSET(KVM_ARCH_HOST_A7, kvm_vcpu_arch, host_context.a7); + OFFSET(KVM_ARCH_HOST_S2, kvm_vcpu_arch, host_context.s2); + OFFSET(KVM_ARCH_HOST_S3, kvm_vcpu_arch, host_context.s3); + OFFSET(KVM_ARCH_HOST_S4, kvm_vcpu_arch, host_context.s4); + OFFSET(KVM_ARCH_HOST_S5, kvm_vcpu_arch, host_context.s5); + OFFSET(KVM_ARCH_HOST_S6, kvm_vcpu_arch, host_context.s6); + OFFSET(KVM_ARCH_HOST_S7, kvm_vcpu_arch, host_context.s7); + OFFSET(KVM_ARCH_HOST_S8, kvm_vcpu_arch, host_context.s8); + OFFSET(KVM_ARCH_HOST_S9, kvm_vcpu_arch, host_context.s9); + OFFSET(KVM_ARCH_HOST_S10, kvm_vcpu_arch, host_context.s10); + OFFSET(KVM_ARCH_HOST_S11, kvm_vcpu_arch, host_context.s11); + OFFSET(KVM_ARCH_HOST_T3, kvm_vcpu_arch, host_context.t3); + OFFSET(KVM_ARCH_HOST_T4, kvm_vcpu_arch, host_context.t4); + OFFSET(KVM_ARCH_HOST_T5, kvm_vcpu_arch, host_context.t5); + OFFSET(KVM_ARCH_HOST_T6, kvm_vcpu_arch, host_context.t6); + OFFSET(KVM_ARCH_HOST_SEPC, kvm_vcpu_arch, host_context.sepc); + OFFSET(KVM_ARCH_HOST_SSTATUS, kvm_vcpu_arch, host_context.sstatus); + OFFSET(KVM_ARCH_HOST_HSTATUS, kvm_vcpu_arch, host_context.hstatus); + OFFSET(KVM_ARCH_HOST_SSCRATCH, kvm_vcpu_arch, host_sscratch); + OFFSET(KVM_ARCH_HOST_STVEC, kvm_vcpu_arch, host_stvec); + OFFSET(KVM_ARCH_HOST_SCOUNTEREN, kvm_vcpu_arch, host_scounteren); + + OFFSET(KVM_ARCH_TRAP_SEPC, kvm_cpu_trap, sepc); + OFFSET(KVM_ARCH_TRAP_SCAUSE, kvm_cpu_trap, scause); + OFFSET(KVM_ARCH_TRAP_STVAL, kvm_cpu_trap, stval); + OFFSET(KVM_ARCH_TRAP_HTVAL, kvm_cpu_trap, htval); + OFFSET(KVM_ARCH_TRAP_HTINST, kvm_cpu_trap, htinst); + + /* F extension */ + + OFFSET(KVM_ARCH_FP_F_F0, kvm_cpu_context, fp.f.f[0]); + OFFSET(KVM_ARCH_FP_F_F1, kvm_cpu_context, fp.f.f[1]); + OFFSET(KVM_ARCH_FP_F_F2, kvm_cpu_context, fp.f.f[2]); + OFFSET(KVM_ARCH_FP_F_F3, kvm_cpu_context, fp.f.f[3]); + OFFSET(KVM_ARCH_FP_F_F4, kvm_cpu_context, fp.f.f[4]); + OFFSET(KVM_ARCH_FP_F_F5, kvm_cpu_context, fp.f.f[5]); + OFFSET(KVM_ARCH_FP_F_F6, kvm_cpu_context, fp.f.f[6]); + OFFSET(KVM_ARCH_FP_F_F7, kvm_cpu_context, fp.f.f[7]); + OFFSET(KVM_ARCH_FP_F_F8, kvm_cpu_context, fp.f.f[8]); + OFFSET(KVM_ARCH_FP_F_F9, kvm_cpu_context, fp.f.f[9]); + OFFSET(KVM_ARCH_FP_F_F10, kvm_cpu_context, fp.f.f[10]); + OFFSET(KVM_ARCH_FP_F_F11, kvm_cpu_context, fp.f.f[11]); + OFFSET(KVM_ARCH_FP_F_F12, kvm_cpu_context, fp.f.f[12]); + OFFSET(KVM_ARCH_FP_F_F13, kvm_cpu_context, fp.f.f[13]); + OFFSET(KVM_ARCH_FP_F_F14, kvm_cpu_context, fp.f.f[14]); + OFFSET(KVM_ARCH_FP_F_F15, kvm_cpu_context, fp.f.f[15]); + OFFSET(KVM_ARCH_FP_F_F16, kvm_cpu_context, fp.f.f[16]); + OFFSET(KVM_ARCH_FP_F_F17, kvm_cpu_context, fp.f.f[17]); + OFFSET(KVM_ARCH_FP_F_F18, kvm_cpu_context, fp.f.f[18]); + OFFSET(KVM_ARCH_FP_F_F19, kvm_cpu_context, fp.f.f[19]); + OFFSET(KVM_ARCH_FP_F_F20, kvm_cpu_context, fp.f.f[20]); + OFFSET(KVM_ARCH_FP_F_F21, kvm_cpu_context, fp.f.f[21]); + OFFSET(KVM_ARCH_FP_F_F22, kvm_cpu_context, fp.f.f[22]); + OFFSET(KVM_ARCH_FP_F_F23, kvm_cpu_context, fp.f.f[23]); + OFFSET(KVM_ARCH_FP_F_F24, kvm_cpu_context, fp.f.f[24]); + OFFSET(KVM_ARCH_FP_F_F25, kvm_cpu_context, fp.f.f[25]); + OFFSET(KVM_ARCH_FP_F_F26, kvm_cpu_context, fp.f.f[26]); + OFFSET(KVM_ARCH_FP_F_F27, kvm_cpu_context, fp.f.f[27]); + OFFSET(KVM_ARCH_FP_F_F28, kvm_cpu_context, fp.f.f[28]); + OFFSET(KVM_ARCH_FP_F_F29, kvm_cpu_context, fp.f.f[29]); + OFFSET(KVM_ARCH_FP_F_F30, kvm_cpu_context, fp.f.f[30]); + OFFSET(KVM_ARCH_FP_F_F31, kvm_cpu_context, fp.f.f[31]); + OFFSET(KVM_ARCH_FP_F_FCSR, kvm_cpu_context, fp.f.fcsr); + + /* D extension */ + + OFFSET(KVM_ARCH_FP_D_F0, kvm_cpu_context, fp.d.f[0]); + OFFSET(KVM_ARCH_FP_D_F1, kvm_cpu_context, fp.d.f[1]); + OFFSET(KVM_ARCH_FP_D_F2, kvm_cpu_context, fp.d.f[2]); + OFFSET(KVM_ARCH_FP_D_F3, kvm_cpu_context, fp.d.f[3]); + OFFSET(KVM_ARCH_FP_D_F4, kvm_cpu_context, fp.d.f[4]); + OFFSET(KVM_ARCH_FP_D_F5, kvm_cpu_context, fp.d.f[5]); + OFFSET(KVM_ARCH_FP_D_F6, kvm_cpu_context, fp.d.f[6]); + OFFSET(KVM_ARCH_FP_D_F7, kvm_cpu_context, fp.d.f[7]); + OFFSET(KVM_ARCH_FP_D_F8, kvm_cpu_context, fp.d.f[8]); + OFFSET(KVM_ARCH_FP_D_F9, kvm_cpu_context, fp.d.f[9]); + OFFSET(KVM_ARCH_FP_D_F10, kvm_cpu_context, fp.d.f[10]); + OFFSET(KVM_ARCH_FP_D_F11, kvm_cpu_context, fp.d.f[11]); + OFFSET(KVM_ARCH_FP_D_F12, kvm_cpu_context, fp.d.f[12]); + OFFSET(KVM_ARCH_FP_D_F13, kvm_cpu_context, fp.d.f[13]); + OFFSET(KVM_ARCH_FP_D_F14, kvm_cpu_context, fp.d.f[14]); + OFFSET(KVM_ARCH_FP_D_F15, kvm_cpu_context, fp.d.f[15]); + OFFSET(KVM_ARCH_FP_D_F16, kvm_cpu_context, fp.d.f[16]); + OFFSET(KVM_ARCH_FP_D_F17, kvm_cpu_context, fp.d.f[17]); + OFFSET(KVM_ARCH_FP_D_F18, kvm_cpu_context, fp.d.f[18]); + OFFSET(KVM_ARCH_FP_D_F19, kvm_cpu_context, fp.d.f[19]); + OFFSET(KVM_ARCH_FP_D_F20, kvm_cpu_context, fp.d.f[20]); + OFFSET(KVM_ARCH_FP_D_F21, kvm_cpu_context, fp.d.f[21]); + OFFSET(KVM_ARCH_FP_D_F22, kvm_cpu_context, fp.d.f[22]); + OFFSET(KVM_ARCH_FP_D_F23, kvm_cpu_context, fp.d.f[23]); + OFFSET(KVM_ARCH_FP_D_F24, kvm_cpu_context, fp.d.f[24]); + OFFSET(KVM_ARCH_FP_D_F25, kvm_cpu_context, fp.d.f[25]); + OFFSET(KVM_ARCH_FP_D_F26, kvm_cpu_context, fp.d.f[26]); + OFFSET(KVM_ARCH_FP_D_F27, kvm_cpu_context, fp.d.f[27]); + OFFSET(KVM_ARCH_FP_D_F28, kvm_cpu_context, fp.d.f[28]); + OFFSET(KVM_ARCH_FP_D_F29, kvm_cpu_context, fp.d.f[29]); + OFFSET(KVM_ARCH_FP_D_F30, kvm_cpu_context, fp.d.f[30]); + OFFSET(KVM_ARCH_FP_D_F31, kvm_cpu_context, fp.d.f[31]); + OFFSET(KVM_ARCH_FP_D_FCSR, kvm_cpu_context, fp.d.fcsr); + /* * THREAD_{F,X}* might be larger than a S-type offset can handle, but * these are used in performance-sensitive assembly so we can't resort @@ -313,4 +469,6 @@ void asm_offsets(void) DEFINE(PT_SIZE_ON_STACK, ALIGN(sizeof(struct pt_regs), STACK_ALIGN)); OFFSET(KERNEL_MAP_VIRT_ADDR, kernel_mapping, virt_addr); + OFFSET(SBI_HART_BOOT_TASK_PTR_OFFSET, sbi_hart_boot_data, task_ptr); + OFFSET(SBI_HART_BOOT_STACK_PTR_OFFSET, sbi_hart_boot_data, stack_ptr); } diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index 66ddfba1cf..f7a832e3a1 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -15,12 +15,6 @@ #include #include -void cpu_stop(void); -void arch_cpu_idle_dead(void) -{ - cpu_stop(); -} - bool cpu_has_hotplug(unsigned int cpu) { if (cpu_ops[cpu]->cpu_stop) @@ -77,7 +71,7 @@ void __cpu_die(unsigned int cpu) /* * Called from the idle thread for the CPU which has been shutdown. */ -void cpu_stop(void) +void arch_cpu_idle_dead(void) { idle_task_exit(); diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 6d59e6906f..ad0a7e9f82 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -7,6 +7,7 @@ #include #include #include +#include /* * Returns the hart ID of the given device tree node, or -ENODEV if the node @@ -22,7 +23,8 @@ int riscv_of_processor_hartid(struct device_node *node) return -ENODEV; } - if (of_property_read_u32(node, "reg", &hart)) { + hart = of_get_cpu_hwid(node, 0); + if (hart == ~0U) { pr_warn("Found CPU without hart ID\n"); return -ENODEV; } @@ -70,18 +72,19 @@ static void print_isa(struct seq_file *f, const char *isa) seq_puts(f, "\n"); } -static void print_mmu(struct seq_file *f, const char *mmu_type) +static void print_mmu(struct seq_file *f) { -#if defined(CONFIG_32BIT) - if (strcmp(mmu_type, "riscv,sv32") != 0) - return; -#elif defined(CONFIG_64BIT) - if (strcmp(mmu_type, "riscv,sv39") != 0 && - strcmp(mmu_type, "riscv,sv48") != 0) - return; -#endif + char sv_type[16]; - seq_printf(f, "mmu\t\t: %s\n", mmu_type+6); +#if defined(CONFIG_32BIT) + strncpy(sv_type, "sv32", 5); +#elif defined(CONFIG_64BIT) + if (pgtable_l4_enabled) + strncpy(sv_type, "sv48", 5); + else + strncpy(sv_type, "sv39", 5); +#endif + seq_printf(f, "mmu\t\t: %s\n", sv_type); } static void *c_start(struct seq_file *m, loff_t *pos) @@ -106,14 +109,13 @@ static int c_show(struct seq_file *m, void *v) { unsigned long cpu_id = (unsigned long)v - 1; struct device_node *node = of_get_cpu_node(cpu_id, NULL); - const char *compat, *isa, *mmu; + const char *compat, *isa; seq_printf(m, "processor\t: %lu\n", cpu_id); seq_printf(m, "hart\t\t: %lu\n", cpuid_to_hartid_map(cpu_id)); if (!of_property_read_string(node, "riscv,isa", &isa)) print_isa(m, isa); - if (!of_property_read_string(node, "mmu-type", &mmu)) - print_mmu(m, mmu); + print_mmu(m); if (!of_property_read_string(node, "compatible", &compat) && strcmp(compat, "riscv")) seq_printf(m, "uarch\t\t: %s\n", compat); diff --git a/arch/riscv/kernel/cpu_ops.c b/arch/riscv/kernel/cpu_ops.c index 1985884fe8..170d07e577 100644 --- a/arch/riscv/kernel/cpu_ops.c +++ b/arch/riscv/kernel/cpu_ops.c @@ -8,37 +8,29 @@ #include #include #include -#include #include #include #include const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init; -void *__cpu_up_stack_pointer[NR_CPUS] __section(".data"); -void *__cpu_up_task_pointer[NR_CPUS] __section(".data"); - extern const struct cpu_operations cpu_ops_sbi; +#ifdef CONFIG_RISCV_BOOT_SPINWAIT extern const struct cpu_operations cpu_ops_spinwait; - -void cpu_update_secondary_bootdata(unsigned int cpuid, - struct task_struct *tidle) -{ - int hartid = cpuid_to_hartid_map(cpuid); - - /* Make sure tidle is updated */ - smp_mb(); - WRITE_ONCE(__cpu_up_stack_pointer[hartid], - task_stack_page(tidle) + THREAD_SIZE); - WRITE_ONCE(__cpu_up_task_pointer[hartid], tidle); -} +#else +const struct cpu_operations cpu_ops_spinwait = { + .name = "", + .cpu_prepare = NULL, + .cpu_start = NULL, +}; +#endif void __init cpu_set_ops(int cpuid) { #if IS_ENABLED(CONFIG_RISCV_SBI) if (sbi_probe_extension(SBI_EXT_HSM) > 0) { if (!cpuid) - pr_info("SBI v0.2 HSM extension detected\n"); + pr_info("SBI HSM extension detected\n"); cpu_ops[cpuid] = &cpu_ops_sbi; } else #endif diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c index 685fae72b7..dae29cbfe5 100644 --- a/arch/riscv/kernel/cpu_ops_sbi.c +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -7,13 +7,22 @@ #include #include +#include #include +#include #include #include extern char secondary_start_sbi[]; const struct cpu_operations cpu_ops_sbi; +/* + * Ordered booting via HSM brings one cpu at a time. However, cpu hotplug can + * be invoked from multiple threads in parallel. Define a per cpu data + * to handle that. + */ +DEFINE_PER_CPU(struct sbi_hart_boot_data, boot_data); + static int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr, unsigned long priv) { @@ -55,14 +64,19 @@ static int sbi_hsm_hart_get_status(unsigned long hartid) static int sbi_cpu_start(unsigned int cpuid, struct task_struct *tidle) { - int rc; unsigned long boot_addr = __pa_symbol(secondary_start_sbi); int hartid = cpuid_to_hartid_map(cpuid); + unsigned long hsm_data; + struct sbi_hart_boot_data *bdata = &per_cpu(boot_data, cpuid); - cpu_update_secondary_bootdata(cpuid, tidle); - rc = sbi_hsm_hart_start(hartid, boot_addr, 0); - - return rc; + /* Make sure tidle is updated */ + smp_mb(); + bdata->task_ptr = tidle; + bdata->stack_ptr = task_stack_page(tidle) + THREAD_SIZE; + /* Make sure boot data is updated */ + smp_mb(); + hsm_data = __pa(bdata); + return sbi_hsm_hart_start(hartid, boot_addr, hsm_data); } static int sbi_cpu_prepare(unsigned int cpuid) diff --git a/arch/riscv/kernel/cpu_ops_spinwait.c b/arch/riscv/kernel/cpu_ops_spinwait.c index b2c957bb68..346847f6c4 100644 --- a/arch/riscv/kernel/cpu_ops_spinwait.c +++ b/arch/riscv/kernel/cpu_ops_spinwait.c @@ -6,11 +6,36 @@ #include #include #include +#include #include #include #include const struct cpu_operations cpu_ops_spinwait; +void *__cpu_spinwait_stack_pointer[NR_CPUS] __section(".data"); +void *__cpu_spinwait_task_pointer[NR_CPUS] __section(".data"); + +static void cpu_update_secondary_bootdata(unsigned int cpuid, + struct task_struct *tidle) +{ + int hartid = cpuid_to_hartid_map(cpuid); + + /* + * The hartid must be less than NR_CPUS to avoid out-of-bound access + * errors for __cpu_spinwait_stack/task_pointer. That is not always possible + * for platforms with discontiguous hartid numbering scheme. That's why + * spinwait booting is not the recommended approach for any platforms + * booting Linux in S-mode and can be disabled in the future. + */ + if (hartid == INVALID_HARTID || hartid >= NR_CPUS) + return; + + /* Make sure tidle is updated */ + smp_mb(); + WRITE_ONCE(__cpu_spinwait_stack_pointer[hartid], + task_stack_page(tidle) + THREAD_SIZE); + WRITE_ONCE(__cpu_spinwait_task_pointer[hartid], tidle); +} static int spinwait_cpu_prepare(unsigned int cpuid) { @@ -28,7 +53,7 @@ static int spinwait_cpu_start(unsigned int cpuid, struct task_struct *tidle) * selects the first cpu to boot the kernel and causes the remainder * of the cpus to spin in a loop waiting for their stack pointer to be * setup by that main cpu. Writing to bootdata - * (i.e __cpu_up_stack_pointer) signals to the spinning cpus that they + * (i.e __cpu_spinwait_stack_pointer) signals to the spinning cpus that they * can continue the boot process. */ cpu_update_secondary_bootdata(cpuid, tidle); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 7e52ad5d61..d6a46ed0bf 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -130,8 +130,7 @@ skip_context_tracking: /* Handle interrupts */ move a0, sp /* pt_regs */ - la a1, handle_arch_irq - REG_L a1, (a1) + la a1, generic_handle_arch_irq jr a1 1: /* @@ -544,11 +543,6 @@ ENTRY(__switch_to) REG_L s9, TASK_THREAD_S9_RA(a4) REG_L s10, TASK_THREAD_S10_RA(a4) REG_L s11, TASK_THREAD_S11_RA(a4) - /* Swap the CPU entry around. */ - lw a3, TASK_TI_CPU(a0) - lw a4, TASK_TI_CPU(a1) - sw a3, TASK_TI_CPU(a1) - sw a4, TASK_TI_CPU(a0) /* The offset of thread_info in task_struct is zero. */ move tp, a1 ret diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 7f1e5203de..4716f4cdc0 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -154,11 +154,6 @@ int ftrace_update_ftrace_func(ftrace_func_t func) return ret; } - -int __init ftrace_dyn_arch_init(void) -{ - return 0; -} #endif #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 52c5ff9804..ec07f99186 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include "efi-header.S" @@ -20,10 +21,19 @@ REG_L t0, _xip_fixup add \reg, \reg, t0 .endm +.macro XIP_FIXUP_FLASH_OFFSET reg + la t0, __data_loc + REG_L t1, _xip_phys_offset + sub \reg, \reg, t1 + add \reg, \reg, t0 +.endm _xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET +_xip_phys_offset: .dword CONFIG_XIP_PHYS_ADDR + XIP_OFFSET #else .macro XIP_FIXUP_OFFSET reg .endm +.macro XIP_FIXUP_FLASH_OFFSET reg +.endm #endif /* CONFIG_XIP_KERNEL */ __HEAD @@ -95,7 +105,8 @@ relocate: /* Compute satp for kernel page tables, but don't load it yet */ srl a2, a0, PAGE_SHIFT - li a1, SATP_MODE + la a1, satp_mode + REG_L a1, 0(a1) or a2, a2, a1 /* @@ -125,7 +136,7 @@ relocate: /* * Switch to kernel page tables. A full fence is necessary in order to * avoid using the trampoline translations, which are only correct for - * the first superpage. Fetching the fence is guarnteed to work + * the first superpage. Fetching the fence is guaranteed to work * because that first superpage is translated the same way. */ csrw CSR_SATP, a2 @@ -157,18 +168,17 @@ secondary_start_sbi: la a3, .Lsecondary_park csrw CSR_TVEC, a3 - slli a3, a0, LGREG - la a4, __cpu_up_stack_pointer - XIP_FIXUP_OFFSET a4 - la a5, __cpu_up_task_pointer - XIP_FIXUP_OFFSET a5 - add a4, a3, a4 - add a5, a3, a5 - REG_L sp, (a4) - REG_L tp, (a5) + /* a0 contains the hartid & a1 contains boot data */ + li a2, SBI_HART_BOOT_TASK_PTR_OFFSET + XIP_FIXUP_OFFSET a2 + add a2, a2, a1 + REG_L tp, (a2) + li a3, SBI_HART_BOOT_STACK_PTR_OFFSET + XIP_FIXUP_OFFSET a3 + add a3, a3, a1 + REG_L sp, (a3) - .global secondary_start_common -secondary_start_common: +.Lsecondary_start_common: #ifdef CONFIG_MMU /* Enable virtual memory and relocate to virtual address */ @@ -248,13 +258,13 @@ pmp_done: li t0, SR_FS csrc CSR_STATUS, t0 -#ifdef CONFIG_SMP +#ifdef CONFIG_RISCV_BOOT_SPINWAIT li t0, CONFIG_NR_CPUS blt a0, t0, .Lgood_cores tail .Lsecondary_park .Lgood_cores: -#endif + /* The lottery system is only required for spinwait booting method */ #ifndef CONFIG_XIP_KERNEL /* Pick one hart to run the main boot sequence */ la a3, hart_lottery @@ -267,11 +277,16 @@ pmp_done: la a3, hart_lottery mv a2, a3 XIP_FIXUP_OFFSET a2 + XIP_FIXUP_FLASH_OFFSET a3 lw t1, (a3) amoswap.w t0, t1, (a2) /* first time here if hart_lottery in RAM is not set */ beq t0, t1, .Lsecondary_start +#endif /* CONFIG_XIP */ +#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ + +#ifdef CONFIG_XIP_KERNEL la sp, _end + THREAD_SIZE XIP_FIXUP_OFFSET sp mv s0, a0 @@ -305,6 +320,7 @@ clear_bss_done: XIP_FIXUP_OFFSET sp #ifdef CONFIG_BUILTIN_DTB la a0, __dtb_start + XIP_FIXUP_OFFSET a0 #else mv a0, s1 #endif /* CONFIG_BUILTIN_DTB */ @@ -318,7 +334,6 @@ clear_bss_done: call setup_trap_vector /* Restore C environment */ la tp, init_task - sw zero, TASK_TI_CPU(tp) la sp, init_thread_union + THREAD_SIZE #ifdef CONFIG_KASAN @@ -328,16 +343,16 @@ clear_bss_done: call soc_early_init tail start_kernel +#if CONFIG_RISCV_BOOT_SPINWAIT .Lsecondary_start: -#ifdef CONFIG_SMP /* Set trap vector to spin forever to help debug */ la a3, .Lsecondary_park csrw CSR_TVEC, a3 slli a3, a0, LGREG - la a1, __cpu_up_stack_pointer + la a1, __cpu_spinwait_stack_pointer XIP_FIXUP_OFFSET a1 - la a2, __cpu_up_task_pointer + la a2, __cpu_spinwait_task_pointer XIP_FIXUP_OFFSET a2 add a1, a3, a1 add a2, a3, a2 @@ -354,8 +369,8 @@ clear_bss_done: beqz tp, .Lwait_for_cpu_up fence - tail secondary_start_common -#endif + tail .Lsecondary_start_common +#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ END(_start_kernel) @@ -437,7 +452,3 @@ ENTRY(reset_regs) ret END(reset_regs) #endif /* CONFIG_RISCV_M_MODE */ - -__PAGE_ALIGNED_BSS - /* Empty zero page */ - .balign PAGE_SIZE diff --git a/arch/riscv/kernel/head.h b/arch/riscv/kernel/head.h index aabbc3ac3e..726731ada5 100644 --- a/arch/riscv/kernel/head.h +++ b/arch/riscv/kernel/head.h @@ -16,7 +16,9 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa); asmlinkage void __init __copy_data(void); #endif -extern void *__cpu_up_stack_pointer[]; -extern void *__cpu_up_task_pointer[]; +#ifdef CONFIG_RISCV_BOOT_SPINWAIT +extern void *__cpu_spinwait_stack_pointer[]; +extern void *__cpu_spinwait_task_pointer[]; +#endif #endif /* __ASM_HEAD_H */ diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index 8ecfc4c128..1fc075b8f7 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -56,13 +56,8 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); unsigned long fp = 0; - /* RISC-V does not support perf in guest mode. */ - if (guest_cbs && guest_cbs->is_in_guest()) - return; - fp = regs->s0; perf_callchain_store(entry, regs->epc); @@ -79,13 +74,5 @@ static bool fill_callchain(void *entry, unsigned long pc) void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - /* RISC-V does not support perf in guest mode. */ - if (guest_cbs && guest_cbs->is_in_guest()) { - pr_warn("RISC-V does not support perf in guest mode!"); - return; - } - walk_stackframe(NULL, regs, fill_callchain, entry); } diff --git a/arch/riscv/kernel/probes/ftrace.c b/arch/riscv/kernel/probes/ftrace.c index aab85a82f4..7142ec42e8 100644 --- a/arch/riscv/kernel/probes/ftrace.c +++ b/arch/riscv/kernel/probes/ftrace.c @@ -15,7 +15,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, if (bit < 0) return; - preempt_disable_notrace(); p = get_kprobe((kprobe_opcode_t *)ip); if (unlikely(!p) || kprobe_disabled(p)) goto out; @@ -52,7 +51,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, __this_cpu_write(current_kprobe, NULL); } out: - preempt_enable_notrace(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(kprobe_ftrace_handler); diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c index 00088dc6da..e6e950b7cf 100644 --- a/arch/riscv/kernel/probes/kprobes.c +++ b/arch/riscv/kernel/probes/kprobes.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ +#define pr_fmt(fmt) "kprobes: " fmt + #include #include #include @@ -50,11 +52,8 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) { unsigned long probe_addr = (unsigned long)p->addr; - if (probe_addr & 0x1) { - pr_warn("Address not aligned.\n"); - - return -EINVAL; - } + if (probe_addr & 0x1) + return -EILSEQ; /* copy instruction */ p->opcode = *p->addr; @@ -191,7 +190,7 @@ static int __kprobes reenter_kprobe(struct kprobe *p, break; case KPROBE_HIT_SS: case KPROBE_REENTER: - pr_warn("Unrecoverable kprobe detected.\n"); + pr_warn("Failed to recover from reentered kprobes.\n"); dump_kprobe(p); BUG(); break; @@ -348,7 +347,7 @@ int __init arch_populate_kprobe_blacklist(void) void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs) { - return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL); + return (void *)kretprobe_trampoline_handler(regs, NULL); } void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, @@ -356,7 +355,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, { ri->ret_addr = (kprobe_opcode_t *)regs->ra; ri->fp = NULL; - regs->ra = (unsigned long) &kretprobe_trampoline; + regs->ra = (unsigned long) &__kretprobe_trampoline; } int __kprobes arch_trampoline_kprobe(struct kprobe *p) diff --git a/arch/riscv/kernel/probes/kprobes_trampoline.S b/arch/riscv/kernel/probes/kprobes_trampoline.S index 6e85d021e2..7bdb09ded3 100644 --- a/arch/riscv/kernel/probes/kprobes_trampoline.S +++ b/arch/riscv/kernel/probes/kprobes_trampoline.S @@ -75,7 +75,7 @@ REG_L x31, PT_T6(sp) .endm -ENTRY(kretprobe_trampoline) +ENTRY(__kretprobe_trampoline) addi sp, sp, -(PT_SIZE_ON_STACK) save_all_base_regs @@ -90,4 +90,4 @@ ENTRY(kretprobe_trampoline) addi sp, sp, PT_SIZE_ON_STACK ret -ENDPROC(kretprobe_trampoline) +ENDPROC(__kretprobe_trampoline) diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index 9c0511119b..a892437301 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -42,12 +42,10 @@ static int riscv_gpr_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - int ret; struct pt_regs *regs; regs = task_pt_regs(target); - ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, regs, 0, -1); - return ret; + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, regs, 0, -1); } #ifdef CONFIG_FPU diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c index ee5878d968..9c842c4168 100644 --- a/arch/riscv/kernel/reset.c +++ b/arch/riscv/kernel/reset.c @@ -12,7 +12,7 @@ static void default_power_off(void) wait_for_interrupt(); } -void (*pm_power_off)(void) = default_power_off; +void (*pm_power_off)(void) = NULL; EXPORT_SYMBOL(pm_power_off); void machine_restart(char *cmd) @@ -23,10 +23,16 @@ void machine_restart(char *cmd) void machine_halt(void) { - pm_power_off(); + if (pm_power_off != NULL) + pm_power_off(); + else + default_power_off(); } void machine_power_off(void) { - pm_power_off(); + if (pm_power_off != NULL) + pm_power_off(); + else + default_power_off(); } diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c index 7402a417f3..775d3322b4 100644 --- a/arch/riscv/kernel/sbi.c +++ b/arch/riscv/kernel/sbi.c @@ -5,8 +5,10 @@ * Copyright (c) 2020 Western Digital Corporation or its affiliates. */ +#include #include #include +#include #include #include @@ -15,8 +17,8 @@ unsigned long sbi_spec_version __ro_after_init = SBI_SPEC_VERSION_DEFAULT; EXPORT_SYMBOL(sbi_spec_version); static void (*__sbi_set_timer)(uint64_t stime) __ro_after_init; -static int (*__sbi_send_ipi)(const unsigned long *hart_mask) __ro_after_init; -static int (*__sbi_rfence)(int fid, const unsigned long *hart_mask, +static int (*__sbi_send_ipi)(const struct cpumask *cpu_mask) __ro_after_init; +static int (*__sbi_rfence)(int fid, const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long arg4, unsigned long arg5) __ro_after_init; @@ -66,6 +68,30 @@ int sbi_err_map_linux_errno(int err) EXPORT_SYMBOL(sbi_err_map_linux_errno); #ifdef CONFIG_RISCV_SBI_V01 +static unsigned long __sbi_v01_cpumask_to_hartmask(const struct cpumask *cpu_mask) +{ + unsigned long cpuid, hartid; + unsigned long hmask = 0; + + /* + * There is no maximum hartid concept in RISC-V and NR_CPUS must not be + * associated with hartid. As SBI v0.1 is only kept for backward compatibility + * and will be removed in the future, there is no point in supporting hartid + * greater than BITS_PER_LONG (32 for RV32 and 64 for RV64). Ideally, SBI v0.2 + * should be used for platforms with hartid greater than BITS_PER_LONG. + */ + for_each_cpu(cpuid, cpu_mask) { + hartid = cpuid_to_hartid_map(cpuid); + if (hartid >= BITS_PER_LONG) { + pr_warn("Unable to send any request to hartid > BITS_PER_LONG for SBI v0.1\n"); + break; + } + hmask |= BIT(hartid); + } + + return hmask; +} + /** * sbi_console_putchar() - Writes given character to the console device. * @ch: The data to be written to the console. @@ -131,33 +157,44 @@ static void __sbi_set_timer_v01(uint64_t stime_value) #endif } -static int __sbi_send_ipi_v01(const unsigned long *hart_mask) +static int __sbi_send_ipi_v01(const struct cpumask *cpu_mask) { - sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)hart_mask, + unsigned long hart_mask; + + if (!cpu_mask || cpumask_empty(cpu_mask)) + cpu_mask = cpu_online_mask; + hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask); + + sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)(&hart_mask), 0, 0, 0, 0, 0); return 0; } -static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, +static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long arg4, unsigned long arg5) { int result = 0; + unsigned long hart_mask; + + if (!cpu_mask || cpumask_empty(cpu_mask)) + cpu_mask = cpu_online_mask; + hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask); /* v0.2 function IDs are equivalent to v0.1 extension IDs */ switch (fid) { case SBI_EXT_RFENCE_REMOTE_FENCE_I: sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0, - (unsigned long)hart_mask, 0, 0, 0, 0, 0); + (unsigned long)&hart_mask, 0, 0, 0, 0, 0); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0, - (unsigned long)hart_mask, start, size, + (unsigned long)&hart_mask, start, size, 0, 0, 0); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0, - (unsigned long)hart_mask, start, size, + (unsigned long)&hart_mask, start, size, arg4, 0, 0); break; default: @@ -179,7 +216,7 @@ static void __sbi_set_timer_v01(uint64_t stime_value) sbi_major_version(), sbi_minor_version()); } -static int __sbi_send_ipi_v01(const unsigned long *hart_mask) +static int __sbi_send_ipi_v01(const struct cpumask *cpu_mask) { pr_warn("IPI extension is not available in SBI v%lu.%lu\n", sbi_major_version(), sbi_minor_version()); @@ -187,7 +224,7 @@ static int __sbi_send_ipi_v01(const unsigned long *hart_mask) return 0; } -static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, +static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long arg4, unsigned long arg5) { @@ -211,37 +248,44 @@ static void __sbi_set_timer_v02(uint64_t stime_value) #endif } -static int __sbi_send_ipi_v02(const unsigned long *hart_mask) +static int __sbi_send_ipi_v02(const struct cpumask *cpu_mask) { - unsigned long hartid, hmask_val, hbase; - struct cpumask tmask; + unsigned long hartid, cpuid, hmask = 0, hbase = 0, htop = 0; struct sbiret ret = {0}; int result; - if (!hart_mask || !(*hart_mask)) { - riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); - hart_mask = cpumask_bits(&tmask); - } + if (!cpu_mask || cpumask_empty(cpu_mask)) + cpu_mask = cpu_online_mask; - hmask_val = 0; - hbase = 0; - for_each_set_bit(hartid, hart_mask, NR_CPUS) { - if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { - ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, - hmask_val, hbase, 0, 0, 0, 0); - if (ret.error) - goto ecall_failed; - hmask_val = 0; - hbase = 0; + for_each_cpu(cpuid, cpu_mask) { + hartid = cpuid_to_hartid_map(cpuid); + if (hmask) { + if (hartid + BITS_PER_LONG <= htop || + hbase + BITS_PER_LONG <= hartid) { + ret = sbi_ecall(SBI_EXT_IPI, + SBI_EXT_IPI_SEND_IPI, hmask, + hbase, 0, 0, 0, 0); + if (ret.error) + goto ecall_failed; + hmask = 0; + } else if (hartid < hbase) { + /* shift the mask to fit lower hartid */ + hmask <<= hbase - hartid; + hbase = hartid; + } } - if (!hmask_val) + if (!hmask) { hbase = hartid; - hmask_val |= 1UL << (hartid - hbase); + htop = hartid; + } else if (hartid > htop) { + htop = hartid; + } + hmask |= BIT(hartid - hbase); } - if (hmask_val) { + if (hmask) { ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, - hmask_val, hbase, 0, 0, 0, 0); + hmask, hbase, 0, 0, 0, 0); if (ret.error) goto ecall_failed; } @@ -251,11 +295,11 @@ static int __sbi_send_ipi_v02(const unsigned long *hart_mask) ecall_failed: result = sbi_err_map_linux_errno(ret.error); pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", - __func__, hbase, hmask_val, result); + __func__, hbase, hmask, result); return result; } -static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val, +static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask, unsigned long hbase, unsigned long start, unsigned long size, unsigned long arg4, unsigned long arg5) @@ -266,31 +310,31 @@ static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val, switch (fid) { case SBI_EXT_RFENCE_REMOTE_FENCE_I: - ret = sbi_ecall(ext, fid, hmask_val, hbase, 0, 0, 0, 0); + ret = sbi_ecall(ext, fid, hmask, hbase, 0, 0, 0, 0); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: - ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + ret = sbi_ecall(ext, fid, hmask, hbase, start, size, 0, 0); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: - ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + ret = sbi_ecall(ext, fid, hmask, hbase, start, size, arg4, 0); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: - ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + ret = sbi_ecall(ext, fid, hmask, hbase, start, size, 0, 0); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: - ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + ret = sbi_ecall(ext, fid, hmask, hbase, start, size, arg4, 0); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: - ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + ret = sbi_ecall(ext, fid, hmask, hbase, start, size, 0, 0); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: - ret = sbi_ecall(ext, fid, hmask_val, hbase, start, + ret = sbi_ecall(ext, fid, hmask, hbase, start, size, arg4, 0); break; default: @@ -302,43 +346,49 @@ static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val, if (ret.error) { result = sbi_err_map_linux_errno(ret.error); pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", - __func__, hbase, hmask_val, result); + __func__, hbase, hmask, result); } return result; } -static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask, +static int __sbi_rfence_v02(int fid, const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long arg4, unsigned long arg5) { - unsigned long hmask_val, hartid, hbase; - struct cpumask tmask; + unsigned long hartid, cpuid, hmask = 0, hbase = 0, htop = 0; int result; - if (!hart_mask || !(*hart_mask)) { - riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); - hart_mask = cpumask_bits(&tmask); - } + if (!cpu_mask || cpumask_empty(cpu_mask)) + cpu_mask = cpu_online_mask; - hmask_val = 0; - hbase = 0; - for_each_set_bit(hartid, hart_mask, NR_CPUS) { - if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { - result = __sbi_rfence_v02_call(fid, hmask_val, hbase, - start, size, arg4, arg5); - if (result) - return result; - hmask_val = 0; - hbase = 0; + for_each_cpu(cpuid, cpu_mask) { + hartid = cpuid_to_hartid_map(cpuid); + if (hmask) { + if (hartid + BITS_PER_LONG <= htop || + hbase + BITS_PER_LONG <= hartid) { + result = __sbi_rfence_v02_call(fid, hmask, + hbase, start, size, arg4, arg5); + if (result) + return result; + hmask = 0; + } else if (hartid < hbase) { + /* shift the mask to fit lower hartid */ + hmask <<= hbase - hartid; + hbase = hartid; + } } - if (!hmask_val) + if (!hmask) { hbase = hartid; - hmask_val |= 1UL << (hartid - hbase); + htop = hartid; + } else if (hartid > htop) { + htop = hartid; + } + hmask |= BIT(hartid - hbase); } - if (hmask_val) { - result = __sbi_rfence_v02_call(fid, hmask_val, hbase, + if (hmask) { + result = __sbi_rfence_v02_call(fid, hmask, hbase, start, size, arg4, arg5); if (result) return result; @@ -360,44 +410,44 @@ void sbi_set_timer(uint64_t stime_value) /** * sbi_send_ipi() - Send an IPI to any hart. - * @hart_mask: A cpu mask containing all the target harts. + * @cpu_mask: A cpu mask containing all the target harts. * * Return: 0 on success, appropriate linux error code otherwise. */ -int sbi_send_ipi(const unsigned long *hart_mask) +int sbi_send_ipi(const struct cpumask *cpu_mask) { - return __sbi_send_ipi(hart_mask); + return __sbi_send_ipi(cpu_mask); } EXPORT_SYMBOL(sbi_send_ipi); /** * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts. - * @hart_mask: A cpu mask containing all the target harts. + * @cpu_mask: A cpu mask containing all the target harts. * * Return: 0 on success, appropriate linux error code otherwise. */ -int sbi_remote_fence_i(const unsigned long *hart_mask) +int sbi_remote_fence_i(const struct cpumask *cpu_mask) { return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, - hart_mask, 0, 0, 0, 0); + cpu_mask, 0, 0, 0, 0); } EXPORT_SYMBOL(sbi_remote_fence_i); /** * sbi_remote_sfence_vma() - Execute SFENCE.VMA instructions on given remote * harts for the specified virtual address range. - * @hart_mask: A cpu mask containing all the target harts. + * @cpu_mask: A cpu mask containing all the target harts. * @start: Start of the virtual address * @size: Total size of the virtual address range. * * Return: 0 on success, appropriate linux error code otherwise. */ -int sbi_remote_sfence_vma(const unsigned long *hart_mask, +int sbi_remote_sfence_vma(const struct cpumask *cpu_mask, unsigned long start, unsigned long size) { return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, - hart_mask, start, size, 0, 0); + cpu_mask, start, size, 0, 0); } EXPORT_SYMBOL(sbi_remote_sfence_vma); @@ -405,38 +455,38 @@ EXPORT_SYMBOL(sbi_remote_sfence_vma); * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given * remote harts for a virtual address range belonging to a specific ASID. * - * @hart_mask: A cpu mask containing all the target harts. + * @cpu_mask: A cpu mask containing all the target harts. * @start: Start of the virtual address * @size: Total size of the virtual address range. * @asid: The value of address space identifier (ASID). * * Return: 0 on success, appropriate linux error code otherwise. */ -int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, +int sbi_remote_sfence_vma_asid(const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long asid) { return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, - hart_mask, start, size, asid, 0); + cpu_mask, start, size, asid, 0); } EXPORT_SYMBOL(sbi_remote_sfence_vma_asid); /** * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote * harts for the specified guest physical address range. - * @hart_mask: A cpu mask containing all the target harts. + * @cpu_mask: A cpu mask containing all the target harts. * @start: Start of the guest physical address * @size: Total size of the guest physical address range. * * Return: None */ -int sbi_remote_hfence_gvma(const unsigned long *hart_mask, +int sbi_remote_hfence_gvma(const struct cpumask *cpu_mask, unsigned long start, unsigned long size) { return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA, - hart_mask, start, size, 0, 0); + cpu_mask, start, size, 0, 0); } EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma); @@ -444,38 +494,38 @@ EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma); * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given * remote harts for a guest physical address range belonging to a specific VMID. * - * @hart_mask: A cpu mask containing all the target harts. + * @cpu_mask: A cpu mask containing all the target harts. * @start: Start of the guest physical address * @size: Total size of the guest physical address range. * @vmid: The value of guest ID (VMID). * * Return: 0 if success, Error otherwise. */ -int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask, +int sbi_remote_hfence_gvma_vmid(const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long vmid) { return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID, - hart_mask, start, size, vmid, 0); + cpu_mask, start, size, vmid, 0); } EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid); /** * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote * harts for the current guest virtual address range. - * @hart_mask: A cpu mask containing all the target harts. + * @cpu_mask: A cpu mask containing all the target harts. * @start: Start of the current guest virtual address * @size: Total size of the current guest virtual address range. * * Return: None */ -int sbi_remote_hfence_vvma(const unsigned long *hart_mask, +int sbi_remote_hfence_vvma(const struct cpumask *cpu_mask, unsigned long start, unsigned long size) { return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA, - hart_mask, start, size, 0, 0); + cpu_mask, start, size, 0, 0); } EXPORT_SYMBOL(sbi_remote_hfence_vvma); @@ -484,23 +534,49 @@ EXPORT_SYMBOL(sbi_remote_hfence_vvma); * remote harts for current guest virtual address range belonging to a specific * ASID. * - * @hart_mask: A cpu mask containing all the target harts. + * @cpu_mask: A cpu mask containing all the target harts. * @start: Start of the current guest virtual address * @size: Total size of the current guest virtual address range. * @asid: The value of address space identifier (ASID). * * Return: None */ -int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask, +int sbi_remote_hfence_vvma_asid(const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long asid) { return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID, - hart_mask, start, size, asid, 0); + cpu_mask, start, size, asid, 0); } EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid); +static void sbi_srst_reset(unsigned long type, unsigned long reason) +{ + sbi_ecall(SBI_EXT_SRST, SBI_EXT_SRST_RESET, type, reason, + 0, 0, 0, 0); + pr_warn("%s: type=0x%lx reason=0x%lx failed\n", + __func__, type, reason); +} + +static int sbi_srst_reboot(struct notifier_block *this, + unsigned long mode, void *cmd) +{ + sbi_srst_reset((mode == REBOOT_WARM || mode == REBOOT_SOFT) ? + SBI_SRST_RESET_TYPE_WARM_REBOOT : + SBI_SRST_RESET_TYPE_COLD_REBOOT, + SBI_SRST_RESET_REASON_NONE); + return NOTIFY_DONE; +} + +static struct notifier_block sbi_srst_reboot_nb; + +static void sbi_srst_power_off(void) +{ + sbi_srst_reset(SBI_SRST_RESET_TYPE_SHUTDOWN, + SBI_SRST_RESET_REASON_NONE); +} + /** * sbi_probe_extension() - Check if an SBI extension ID is supported or not. * @extid: The extension ID to be probed. @@ -564,11 +640,7 @@ long sbi_get_mimpid(void) static void sbi_send_cpumask_ipi(const struct cpumask *target) { - struct cpumask hartid_mask; - - riscv_cpuid_to_hartid_mask(target, &hartid_mask); - - sbi_send_ipi(cpumask_bits(&hartid_mask)); + sbi_send_ipi(target); } static const struct riscv_ipi_ops sbi_ipi_ops = { @@ -608,6 +680,14 @@ void __init sbi_init(void) } else { __sbi_rfence = __sbi_rfence_v01; } + if ((sbi_spec_version >= sbi_mk_version(0, 3)) && + (sbi_probe_extension(SBI_EXT_SRST) > 0)) { + pr_info("SBI SRST extension detected\n"); + pm_power_off = sbi_srst_power_off; + sbi_srst_reboot_nb.notifier_call = sbi_srst_reboot; + sbi_srst_reboot_nb.priority = 192; + register_restart_handler(&sbi_srst_reboot_nb); + } } else { __sbi_set_timer = __sbi_set_timer_v01; __sbi_send_ipi = __sbi_send_ipi_v01; diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 6c5caf5eb9..b42bfdc674 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -59,16 +59,6 @@ atomic_t hart_lottery __section(".sdata") unsigned long boot_cpu_hartid; static DEFINE_PER_CPU(struct cpu, cpu_devices); -void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out) -{ - int cpu; - - cpumask_clear(out); - for_each_cpu(cpu, in) - cpumask_set_cpu(cpuid_to_hartid_map(cpu), out); -} -EXPORT_SYMBOL_GPL(riscv_cpuid_to_hartid_mask); - /* * Place kernel memory regions on the resource tree so that * kexec-tools can retrieve them from /proc/iomem. While there @@ -240,13 +230,13 @@ static void __init init_resources(void) /* Clean-up any unused pre-allocated resources */ if (res_idx >= 0) - memblock_free(__pa(mem_res), (res_idx + 1) * sizeof(*mem_res)); + memblock_free(mem_res, (res_idx + 1) * sizeof(*mem_res)); return; error: /* Better an empty resource tree than an inconsistent one */ release_child_resources(&iomem_resource); - memblock_free(__pa(mem_res), mem_res_sz); + memblock_free(mem_res, mem_res_sz); } diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index d014729469..b5d30ea922 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -130,12 +130,9 @@ void arch_irq_work_raise(void) void handle_IPI(struct pt_regs *regs) { - struct pt_regs *old_regs = set_irq_regs(regs); unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; unsigned long *stats = ipi_data[smp_processor_id()].stats; - irq_enter(); - riscv_clear_ipi(); while (true) { @@ -146,7 +143,7 @@ void handle_IPI(struct pt_regs *regs) ops = xchg(pending_ipis, 0); if (ops == 0) - goto done; + return; if (ops & (1 << IPI_RESCHEDULE)) { stats[IPI_RESCHEDULE]++; @@ -179,10 +176,6 @@ void handle_IPI(struct pt_regs *regs) /* Order data access and bit testing. */ mb(); } - -done: - irq_exit(); - set_irq_regs(old_regs); } static const char * const ipi_names[] = { diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index bd82375db5..622f226454 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -96,7 +96,7 @@ void __init setup_smp(void) if (cpuid >= NR_CPUS) { pr_warn("Invalid cpuid [%d] for hartid [%d]\n", cpuid, hart); - break; + continue; } cpuid_to_hartid_map(cpuid) = hart; diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index c2601150b9..14d2b53ec3 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -129,25 +129,19 @@ static bool save_wchan(void *arg, unsigned long pc) return true; } -unsigned long get_wchan(struct task_struct *task) +unsigned long __get_wchan(struct task_struct *task) { unsigned long pc = 0; - if (likely(task && task != current && !task_is_running(task))) { - if (!try_get_task_stack(task)) - return 0; - walk_stackframe(task, NULL, save_wchan, &pc); - put_task_stack(task); - } + if (!try_get_task_stack(task)) + return 0; + walk_stackframe(task, NULL, save_wchan, &pc); + put_task_stack(task); return pc; } -#ifdef CONFIG_STACKTRACE - noinline void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, struct task_struct *task, struct pt_regs *regs) { walk_stackframe(task, regs, consume_entry, cookie); } - -#endif /* CONFIG_STACKTRACE */ diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 0daaa3e463..fe92e119e6 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -54,7 +54,7 @@ void die(struct pt_regs *regs, const char *str) if (panic_on_oops) panic("Fatal exception"); if (ret != NOTIFY_STOP) - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index b70956d804..a9436a6516 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -13,6 +13,7 @@ #include #include #include +#include #ifdef CONFIG_GENERIC_TIME_VSYSCALL #include @@ -25,14 +26,12 @@ extern char vdso_start[], vdso_end[]; enum vvar_pages { VVAR_DATA_PAGE_OFFSET, + VVAR_TIMENS_PAGE_OFFSET, VVAR_NR_PAGES, }; #define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT) -static unsigned int vdso_pages __ro_after_init; -static struct page **vdso_pagelist __ro_after_init; - /* * The vDSO data page. */ @@ -42,83 +41,228 @@ static union { } vdso_data_store __page_aligned_data; struct vdso_data *vdso_data = &vdso_data_store.data; -static int __init vdso_init(void) +struct __vdso_info { + const char *name; + const char *vdso_code_start; + const char *vdso_code_end; + unsigned long vdso_pages; + /* Data Mapping */ + struct vm_special_mapping *dm; + /* Code Mapping */ + struct vm_special_mapping *cm; +}; + +static struct __vdso_info vdso_info __ro_after_init = { + .name = "vdso", + .vdso_code_start = vdso_start, + .vdso_code_end = vdso_end, +}; + +static int vdso_mremap(const struct vm_special_mapping *sm, + struct vm_area_struct *new_vma) { - unsigned int i; - - vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; - vdso_pagelist = - kcalloc(vdso_pages + VVAR_NR_PAGES, sizeof(struct page *), GFP_KERNEL); - if (unlikely(vdso_pagelist == NULL)) { - pr_err("vdso: pagelist allocation failed\n"); - return -ENOMEM; - } - - for (i = 0; i < vdso_pages; i++) { - struct page *pg; - - pg = virt_to_page(vdso_start + (i << PAGE_SHIFT)); - vdso_pagelist[i] = pg; - } - vdso_pagelist[i] = virt_to_page(vdso_data); + current->mm->context.vdso = (void *)new_vma->vm_start; return 0; } + +static int __init __vdso_init(void) +{ + unsigned int i; + struct page **vdso_pagelist; + unsigned long pfn; + + if (memcmp(vdso_info.vdso_code_start, "\177ELF", 4)) { + pr_err("vDSO is not a valid ELF object!\n"); + return -EINVAL; + } + + vdso_info.vdso_pages = ( + vdso_info.vdso_code_end - + vdso_info.vdso_code_start) >> + PAGE_SHIFT; + + vdso_pagelist = kcalloc(vdso_info.vdso_pages, + sizeof(struct page *), + GFP_KERNEL); + if (vdso_pagelist == NULL) + return -ENOMEM; + + /* Grab the vDSO code pages. */ + pfn = sym_to_pfn(vdso_info.vdso_code_start); + + for (i = 0; i < vdso_info.vdso_pages; i++) + vdso_pagelist[i] = pfn_to_page(pfn + i); + + vdso_info.cm->pages = vdso_pagelist; + + return 0; +} + +#ifdef CONFIG_TIME_NS +struct vdso_data *arch_get_vdso_data(void *vvar_page) +{ + return (struct vdso_data *)(vvar_page); +} + +/* + * The vvar mapping contains data for a specific time namespace, so when a task + * changes namespace we must unmap its vvar data for the old namespace. + * Subsequent faults will map in data for the new namespace. + * + * For more details see timens_setup_vdso_data(). + */ +int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) +{ + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + + mmap_read_lock(mm); + + for (vma = mm->mmap; vma; vma = vma->vm_next) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, vdso_info.dm)) + zap_page_range(vma, vma->vm_start, size); + } + + mmap_read_unlock(mm); + return 0; +} + +static struct page *find_timens_vvar_page(struct vm_area_struct *vma) +{ + if (likely(vma->vm_mm == current->mm)) + return current->nsproxy->time_ns->vvar_page; + + /* + * VM_PFNMAP | VM_IO protect .fault() handler from being called + * through interfaces like /proc/$pid/mem or + * process_vm_{readv,writev}() as long as there's no .access() + * in special_mapping_vmops. + * For more details check_vma_flags() and __access_remote_vm() + */ + WARN(1, "vvar_page accessed remotely"); + + return NULL; +} +#else +static struct page *find_timens_vvar_page(struct vm_area_struct *vma) +{ + return NULL; +} +#endif + +static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, + struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct page *timens_page = find_timens_vvar_page(vma); + unsigned long pfn; + + switch (vmf->pgoff) { + case VVAR_DATA_PAGE_OFFSET: + if (timens_page) + pfn = page_to_pfn(timens_page); + else + pfn = sym_to_pfn(vdso_data); + break; +#ifdef CONFIG_TIME_NS + case VVAR_TIMENS_PAGE_OFFSET: + /* + * If a task belongs to a time namespace then a namespace + * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and + * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET + * offset. + * See also the comment near timens_setup_vdso_data(). + */ + if (!timens_page) + return VM_FAULT_SIGBUS; + pfn = sym_to_pfn(vdso_data); + break; +#endif /* CONFIG_TIME_NS */ + default: + return VM_FAULT_SIGBUS; + } + + return vmf_insert_pfn(vma, vmf->address, pfn); +} + +enum rv_vdso_map { + RV_VDSO_MAP_VVAR, + RV_VDSO_MAP_VDSO, +}; + +static struct vm_special_mapping rv_vdso_maps[] __ro_after_init = { + [RV_VDSO_MAP_VVAR] = { + .name = "[vvar]", + .fault = vvar_fault, + }, + [RV_VDSO_MAP_VDSO] = { + .name = "[vdso]", + .mremap = vdso_mremap, + }, +}; + +static int __init vdso_init(void) +{ + vdso_info.dm = &rv_vdso_maps[RV_VDSO_MAP_VVAR]; + vdso_info.cm = &rv_vdso_maps[RV_VDSO_MAP_VDSO]; + + return __vdso_init(); +} arch_initcall(vdso_init); -int arch_setup_additional_pages(struct linux_binprm *bprm, - int uses_interp) +static int __setup_additional_pages(struct mm_struct *mm, + struct linux_binprm *bprm, + int uses_interp) { - struct mm_struct *mm = current->mm; - unsigned long vdso_base, vdso_len; - int ret; + unsigned long vdso_base, vdso_text_len, vdso_mapping_len; + void *ret; BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES); - vdso_len = (vdso_pages + VVAR_NR_PAGES) << PAGE_SHIFT; + vdso_text_len = vdso_info.vdso_pages << PAGE_SHIFT; + /* Be sure to map the data page */ + vdso_mapping_len = vdso_text_len + VVAR_SIZE; + + vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); + if (IS_ERR_VALUE(vdso_base)) { + ret = ERR_PTR(vdso_base); + goto up_fail; + } + + ret = _install_special_mapping(mm, vdso_base, VVAR_SIZE, + (VM_READ | VM_MAYREAD | VM_PFNMAP), vdso_info.dm); + if (IS_ERR(ret)) + goto up_fail; + + vdso_base += VVAR_SIZE; + mm->context.vdso = (void *)vdso_base; + ret = + _install_special_mapping(mm, vdso_base, vdso_text_len, + (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), + vdso_info.cm); + + if (IS_ERR(ret)) + goto up_fail; + + return 0; + +up_fail: + mm->context.vdso = NULL; + return PTR_ERR(ret); +} + +int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) +{ + struct mm_struct *mm = current->mm; + int ret; if (mmap_write_lock_killable(mm)) return -EINTR; - vdso_base = get_unmapped_area(NULL, 0, vdso_len, 0, 0); - if (IS_ERR_VALUE(vdso_base)) { - ret = vdso_base; - goto end; - } - - mm->context.vdso = NULL; - ret = install_special_mapping(mm, vdso_base, VVAR_SIZE, - (VM_READ | VM_MAYREAD), &vdso_pagelist[vdso_pages]); - if (unlikely(ret)) - goto end; - - ret = - install_special_mapping(mm, vdso_base + VVAR_SIZE, - vdso_pages << PAGE_SHIFT, - (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), - vdso_pagelist); - - if (unlikely(ret)) - goto end; - - /* - * Put vDSO base into mm struct. We need to do this before calling - * install_special_mapping or the perf counter mmap tracking code - * will fail to recognise it as a vDSO (since arch_vma_name fails). - */ - mm->context.vdso = (void *)vdso_base + VVAR_SIZE; - -end: + ret = __setup_additional_pages(mm, bprm, uses_interp); mmap_write_unlock(mm); + return ret; } - -const char *arch_vma_name(struct vm_area_struct *vma) -{ - if (vma->vm_mm && (vma->vm_start == (long)vma->vm_mm->context.vdso)) - return "[vdso]"; - if (vma->vm_mm && (vma->vm_start == - (long)vma->vm_mm->context.vdso - VVAR_SIZE)) - return "[vdso_data]"; - return NULL; -} diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index e9111f700a..01d94aae5b 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -10,6 +10,9 @@ OUTPUT_ARCH(riscv) SECTIONS { PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE); +#ifdef CONFIG_TIME_NS + PROVIDE(_timens_data = _vdso_data + PAGE_SIZE); +#endif . = SIZEOF_HEADERS; .hash : { *(.hash) } :text diff --git a/arch/riscv/kernel/vmlinux-xip.lds.S b/arch/riscv/kernel/vmlinux-xip.lds.S index 9c9f35091e..75e0fa8a70 100644 --- a/arch/riscv/kernel/vmlinux-xip.lds.S +++ b/arch/riscv/kernel/vmlinux-xip.lds.S @@ -45,7 +45,6 @@ SECTIONS ENTRY_TEXT IRQENTRY_TEXT SOFTIRQENTRY_TEXT - *(.fixup) _etext = .; } RO_DATA(L1_CACHE_BYTES) @@ -64,8 +63,11 @@ SECTIONS /* * From this point, stuff is considered writable and will be copied to RAM */ - __data_loc = ALIGN(16); /* location in file */ - . = LOAD_OFFSET + XIP_OFFSET; /* location in memory */ + __data_loc = ALIGN(PAGE_SIZE); /* location in file */ + . = KERNEL_LINK_ADDR + XIP_OFFSET; /* location in memory */ + +#undef LOAD_OFFSET +#define LOAD_OFFSET (KERNEL_LINK_ADDR + XIP_OFFSET - (__data_loc & XIP_OFFSET_MASK)) _sdata = .; /* Start of data section */ _data = .; @@ -96,7 +98,6 @@ SECTIONS KEEP(*(__soc_builtin_dtb_table)) __soc_builtin_dtb_table_end = .; } - PERCPU_SECTION(L1_CACHE_BYTES) . = ALIGN(8); .alternative : { @@ -122,6 +123,8 @@ SECTIONS BSS_SECTION(PAGE_SIZE, PAGE_SIZE, 0) + PERCPU_SECTION(L1_CACHE_BYTES) + .rel.dyn : AT(ADDR(.rel.dyn) - LOAD_OFFSET) { *(.rel.dyn*) } diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 5104f3a871..4e6c88aa4d 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define RO_EXCEPTION_TABLE_ALIGN 16 +#define RO_EXCEPTION_TABLE_ALIGN 4 #ifdef CONFIG_XIP_KERNEL #include "vmlinux-xip.lds.S" @@ -48,7 +48,6 @@ SECTIONS ENTRY_TEXT IRQENTRY_TEXT SOFTIRQENTRY_TEXT - *(.fixup) _etext = .; } diff --git a/arch/riscv/lib/delay.c b/arch/riscv/lib/delay.c index f51c9a03bc..49d510ba75 100644 --- a/arch/riscv/lib/delay.c +++ b/arch/riscv/lib/delay.c @@ -4,10 +4,14 @@ */ #include +#include #include #include +#include #include +#include + /* * This is copies from arch/arm/include/asm/delay.h * diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 63bc691cff..8c475f4da3 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -1,15 +1,13 @@ #include #include #include +#include #include .macro fixup op reg addr lbl 100: \op \reg, \addr - .section __ex_table,"a" - .balign RISCV_SZPTR - RISCV_PTR 100b, \lbl - .previous + _asm_extable 100b, \lbl .endm ENTRY(__asm_copy_to_user) @@ -173,6 +171,13 @@ ENTRY(__asm_copy_from_user) csrc CSR_STATUS, t6 li a0, 0 ret + + /* Exception fixup code */ +10: + /* Disable access to user memory */ + csrs CSR_STATUS, t6 + mv a0, t5 + ret ENDPROC(__asm_copy_to_user) ENDPROC(__asm_copy_from_user) EXPORT_SYMBOL(__asm_copy_to_user) @@ -218,19 +223,12 @@ ENTRY(__clear_user) addi a0, a0, 1 bltu a0, a3, 5b j 3b -ENDPROC(__clear_user) -EXPORT_SYMBOL(__clear_user) - .section .fixup,"ax" - .balign 4 - /* Fixup code for __copy_user(10) and __clear_user(11) */ -10: - /* Disable access to user memory */ - csrs CSR_STATUS, t6 - mv a0, t5 - ret + /* Exception fixup code */ 11: + /* Disable access to user memory */ csrs CSR_STATUS, t6 mv a0, a1 ret - .previous +ENDPROC(__clear_user) +EXPORT_SYMBOL(__clear_user) diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index 89f81067e0..6cb7d96ad9 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -67,10 +67,7 @@ void flush_icache_mm(struct mm_struct *mm, bool local) */ smp_mb(); } else if (IS_ENABLED(CONFIG_RISCV_SBI)) { - cpumask_t hartid_mask; - - riscv_cpuid_to_hartid_mask(&others, &hartid_mask); - sbi_remote_fence_i(cpumask_bits(&hartid_mask)); + sbi_remote_fence_i(&others); } else { on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); } diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index ee3459cb67..7acbfbd145 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -192,7 +192,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned int cpu) switch_mm_fast: csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | ((cntx & asid_mask) << SATP_ASID_SHIFT) | - SATP_MODE); + satp_mode); if (need_flush_tlb) local_flush_tlb_all(); @@ -201,7 +201,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned int cpu) static void set_mm_noasid(struct mm_struct *mm) { /* Switch the page table and blindly nuke entire local TLB */ - csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | SATP_MODE); + csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | satp_mode); local_flush_tlb_all(); } @@ -233,8 +233,10 @@ static int __init asids_init(void) local_flush_tlb_all(); /* Pre-compute ASID details */ - num_asids = 1 << asid_bits; - asid_mask = num_asids - 1; + if (asid_bits) { + num_asids = 1 << asid_bits; + asid_mask = num_asids - 1; + } /* * Use ASID allocator only if number of HW ASIDs are @@ -255,7 +257,7 @@ static int __init asids_init(void) pr_info("ASID allocator using %lu bits (%lu entries)\n", asid_bits, num_asids); } else { - pr_info("ASID allocator disabled\n"); + pr_info("ASID allocator disabled (%lu bits)\n", asid_bits); } return 0; diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index 2fc7294221..35484d830f 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -7,18 +7,65 @@ */ +#include #include #include #include +#include +#include -int fixup_exception(struct pt_regs *regs) +static inline unsigned long +get_ex_fixup(const struct exception_table_entry *ex) { - const struct exception_table_entry *fixup; - - fixup = search_exception_tables(regs->epc); - if (fixup) { - regs->epc = fixup->fixup; - return 1; - } - return 0; + return ((unsigned long)&ex->fixup + ex->fixup); +} + +static bool ex_handler_fixup(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + regs->epc = get_ex_fixup(ex); + return true; +} + +static inline void regs_set_gpr(struct pt_regs *regs, unsigned int offset, + unsigned long val) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return; + + if (offset) + *(unsigned long *)((unsigned long)regs + offset) = val; +} + +static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + int reg_zero = FIELD_GET(EX_DATA_REG_ZERO, ex->data); + + regs_set_gpr(regs, reg_err * sizeof(unsigned long), -EFAULT); + regs_set_gpr(regs, reg_zero * sizeof(unsigned long), 0); + + regs->epc = get_ex_fixup(ex); + return true; +} + +bool fixup_exception(struct pt_regs *regs) +{ + const struct exception_table_entry *ex; + + ex = search_exception_tables(regs->epc); + if (!ex) + return false; + + switch (ex->type) { + case EX_TYPE_FIXUP: + return ex_handler_fixup(ex, regs); + case EX_TYPE_BPF: + return ex_handler_bpf(ex, regs); + case EX_TYPE_UACCESS_ERR_ZERO: + return ex_handler_uaccess_err_zero(ex, regs); + } + + BUG(); } diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index aa08dd2f8f..4e9efbe46d 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -31,7 +31,7 @@ static void die_kernel_fault(const char *msg, unsigned long addr, bust_spinlocks(0); die(regs, "Oops"); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } static inline void no_context(struct pt_regs *regs, unsigned long addr) @@ -235,7 +235,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) * only copy the information from the master page table, * nothing more. */ - if (unlikely((addr >= VMALLOC_START) && (addr <= VMALLOC_END))) { + if (unlikely((addr >= VMALLOC_START) && (addr < VMALLOC_END))) { vmalloc_fault(regs, code, addr); return; } @@ -330,7 +330,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) if (fault_signal_pending(fault, regs)) return; - if (unlikely((fault & VM_FAULT_RETRY) && (flags & FAULT_FLAG_ALLOW_RETRY))) { + if (unlikely(fault & VM_FAULT_RETRY)) { flags |= FAULT_FLAG_TRIED; /* diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 3de593b268..0d588032d6 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -37,13 +37,19 @@ EXPORT_SYMBOL(kernel_map); #define kernel_map (*(struct kernel_mapping *)XIP_FIXUP(&kernel_map)) #endif +#ifdef CONFIG_64BIT +u64 satp_mode = !IS_ENABLED(CONFIG_XIP_KERNEL) ? SATP_MODE_48 : SATP_MODE_39; +#else +u64 satp_mode = SATP_MODE_32; +#endif +EXPORT_SYMBOL(satp_mode); + +bool pgtable_l4_enabled = IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_XIP_KERNEL); +EXPORT_SYMBOL(pgtable_l4_enabled); + phys_addr_t phys_ram_base __ro_after_init; EXPORT_SYMBOL(phys_ram_base); -#ifdef CONFIG_XIP_KERNEL -extern char _xiprom[], _exiprom[]; -#endif - unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; EXPORT_SYMBOL(empty_zero_page); @@ -53,15 +59,6 @@ extern char _start[]; void *_dtb_early_va __initdata; uintptr_t _dtb_early_pa __initdata; -struct pt_alloc_ops { - pte_t *(*get_pte_virt)(phys_addr_t pa); - phys_addr_t (*alloc_pte)(uintptr_t va); -#ifndef __PAGETABLE_PMD_FOLDED - pmd_t *(*get_pmd_virt)(phys_addr_t pa); - phys_addr_t (*alloc_pmd)(uintptr_t va); -#endif -}; - static phys_addr_t dma32_phys_limit __initdata; static void __init zone_sizes_init(void) @@ -102,10 +99,14 @@ static void __init print_vm_layout(void) (unsigned long)VMALLOC_END); print_mlm("lowmem", (unsigned long)PAGE_OFFSET, (unsigned long)high_memory); -#ifdef CONFIG_64BIT - print_mlm("kernel", (unsigned long)KERNEL_LINK_ADDR, - (unsigned long)ADDRESS_SPACE_END); + if (IS_ENABLED(CONFIG_64BIT)) { +#ifdef CONFIG_KASAN + print_mlm("kasan", KASAN_SHADOW_START, KASAN_SHADOW_END); #endif + + print_mlm("kernel", (unsigned long)KERNEL_LINK_ADDR, + (unsigned long)ADDRESS_SPACE_END); + } } #else static void print_vm_layout(void) { } @@ -124,24 +125,13 @@ void __init mem_init(void) else swiotlb_force = SWIOTLB_NO_FORCE; #endif - high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); memblock_free_all(); print_vm_layout(); } -/* - * The default maximal physical memory size is -PAGE_OFFSET for 32-bit kernel, - * whereas for 64-bit kernel, the end of the virtual address space is occupied - * by the modules/BPF/kernel mappings which reduces the available size of the - * linear mapping. - * Limit the memory size via mem. - */ -#ifdef CONFIG_64BIT -static phys_addr_t memory_limit = -PAGE_OFFSET - SZ_4G; -#else -static phys_addr_t memory_limit = -PAGE_OFFSET; -#endif +/* Limit the memory size via mem. */ +static phys_addr_t memory_limit; static int __init early_mem(char *p) { @@ -162,35 +152,31 @@ early_param("mem", early_mem); static void __init setup_bootmem(void) { phys_addr_t vmlinux_end = __pa_symbol(&_end); - phys_addr_t vmlinux_start = __pa_symbol(&_start); - phys_addr_t __maybe_unused max_mapped_addr; - phys_addr_t phys_ram_end; + phys_addr_t max_mapped_addr; + phys_addr_t phys_ram_end, vmlinux_start; -#ifdef CONFIG_XIP_KERNEL - vmlinux_start = __pa_symbol(&_sdata); -#endif + if (IS_ENABLED(CONFIG_XIP_KERNEL)) + vmlinux_start = __pa_symbol(&_sdata); + else + vmlinux_start = __pa_symbol(&_start); memblock_enforce_memory_limit(memory_limit); - /* - * Reserve from the start of the kernel to the end of the kernel - */ -#if defined(CONFIG_64BIT) && defined(CONFIG_STRICT_KERNEL_RWX) /* * Make sure we align the reservation on PMD_SIZE since we will * map the kernel in the linear mapping as read-only: we do not want * any allocation to happen between _end and the next pmd aligned page. */ - vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK; -#endif + if (IS_ENABLED(CONFIG_64BIT) && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) + vmlinux_end = (vmlinux_end + PMD_SIZE - 1) & PMD_MASK; + /* + * Reserve from the start of the kernel to the end of the kernel + */ memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); - phys_ram_end = memblock_end_of_DRAM(); -#ifndef CONFIG_XIP_KERNEL - phys_ram_base = memblock_start_of_DRAM(); -#endif -#ifndef CONFIG_64BIT + if (!IS_ENABLED(CONFIG_XIP_KERNEL)) + phys_ram_base = memblock_start_of_DRAM(); /* * memblock allocator is not aware of the fact that last 4K bytes of * the addressable memory can not be mapped because of IS_ERR_VALUE @@ -200,13 +186,15 @@ static void __init setup_bootmem(void) * address space is occupied by the kernel mapping then this check must * be done as soon as the kernel mapping base address is determined. */ - max_mapped_addr = __pa(~(ulong)0); - if (max_mapped_addr == (phys_ram_end - 1)) - memblock_set_current_limit(max_mapped_addr - 4096); -#endif + if (!IS_ENABLED(CONFIG_64BIT)) { + max_mapped_addr = __pa(~(ulong)0); + if (max_mapped_addr == (phys_ram_end - 1)) + memblock_set_current_limit(max_mapped_addr - 4096); + } min_low_pfn = PFN_UP(phys_ram_base); max_low_pfn = max_pfn = PFN_DOWN(phys_ram_end); + high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); @@ -229,13 +217,7 @@ static void __init setup_bootmem(void) } #ifdef CONFIG_MMU -static struct pt_alloc_ops _pt_ops __initdata; - -#ifdef CONFIG_XIP_KERNEL -#define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&_pt_ops)) -#else -#define pt_ops _pt_ops -#endif +struct pt_alloc_ops pt_ops __initdata; unsigned long riscv_pfn_base __ro_after_init; EXPORT_SYMBOL(riscv_pfn_base); @@ -245,9 +227,12 @@ pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); +static pud_t __maybe_unused early_dtb_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE); static pmd_t __maybe_unused early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); #ifdef CONFIG_XIP_KERNEL +#define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&pt_ops)) +#define riscv_pfn_base (*(unsigned long *)XIP_FIXUP(&riscv_pfn_base)) #define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) #define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) #define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) @@ -333,6 +318,16 @@ static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); #define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd)) #endif /* CONFIG_XIP_KERNEL */ +static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss; +static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss; +static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE); + +#ifdef CONFIG_XIP_KERNEL +#define trampoline_pud ((pud_t *)XIP_FIXUP(trampoline_pud)) +#define fixmap_pud ((pud_t *)XIP_FIXUP(fixmap_pud)) +#define early_pud ((pud_t *)XIP_FIXUP(early_pud)) +#endif /* CONFIG_XIP_KERNEL */ + static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) { /* Before MMU is enabled */ @@ -352,7 +347,7 @@ static pmd_t *__init get_pmd_virt_late(phys_addr_t pa) static phys_addr_t __init alloc_pmd_early(uintptr_t va) { - BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT); + BUG_ON((va - kernel_map.virt_addr) >> PUD_SHIFT); return (uintptr_t)early_pmd; } @@ -367,7 +362,8 @@ static phys_addr_t __init alloc_pmd_late(uintptr_t va) unsigned long vaddr; vaddr = __get_free_page(GFP_KERNEL); - BUG_ON(!vaddr); + BUG_ON(!vaddr || !pgtable_pmd_page_ctor(virt_to_page(vaddr))); + return __pa(vaddr); } @@ -398,21 +394,97 @@ static void __init create_pmd_mapping(pmd_t *pmdp, create_pte_mapping(ptep, va, pa, sz, prot); } -#define pgd_next_t pmd_t -#define alloc_pgd_next(__va) pt_ops.alloc_pmd(__va) -#define get_pgd_next_virt(__pa) pt_ops.get_pmd_virt(__pa) +static pud_t *__init get_pud_virt_early(phys_addr_t pa) +{ + return (pud_t *)((uintptr_t)pa); +} + +static pud_t *__init get_pud_virt_fixmap(phys_addr_t pa) +{ + clear_fixmap(FIX_PUD); + return (pud_t *)set_fixmap_offset(FIX_PUD, pa); +} + +static pud_t *__init get_pud_virt_late(phys_addr_t pa) +{ + return (pud_t *)__va(pa); +} + +static phys_addr_t __init alloc_pud_early(uintptr_t va) +{ + /* Only one PUD is available for early mapping */ + BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT); + + return (uintptr_t)early_pud; +} + +static phys_addr_t __init alloc_pud_fixmap(uintptr_t va) +{ + return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); +} + +static phys_addr_t alloc_pud_late(uintptr_t va) +{ + unsigned long vaddr; + + vaddr = __get_free_page(GFP_KERNEL); + BUG_ON(!vaddr); + return __pa(vaddr); +} + +static void __init create_pud_mapping(pud_t *pudp, + uintptr_t va, phys_addr_t pa, + phys_addr_t sz, pgprot_t prot) +{ + pmd_t *nextp; + phys_addr_t next_phys; + uintptr_t pud_index = pud_index(va); + + if (sz == PUD_SIZE) { + if (pud_val(pudp[pud_index]) == 0) + pudp[pud_index] = pfn_pud(PFN_DOWN(pa), prot); + return; + } + + if (pud_val(pudp[pud_index]) == 0) { + next_phys = pt_ops.alloc_pmd(va); + pudp[pud_index] = pfn_pud(PFN_DOWN(next_phys), PAGE_TABLE); + nextp = pt_ops.get_pmd_virt(next_phys); + memset(nextp, 0, PAGE_SIZE); + } else { + next_phys = PFN_PHYS(_pud_pfn(pudp[pud_index])); + nextp = pt_ops.get_pmd_virt(next_phys); + } + + create_pmd_mapping(nextp, va, pa, sz, prot); +} + +#define pgd_next_t pud_t +#define alloc_pgd_next(__va) (pgtable_l4_enabled ? \ + pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va)) +#define get_pgd_next_virt(__pa) (pgtable_l4_enabled ? \ + pt_ops.get_pud_virt(__pa) : (pgd_next_t *)pt_ops.get_pmd_virt(__pa)) #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ - create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) -#define fixmap_pgd_next fixmap_pmd + (pgtable_l4_enabled ? \ + create_pud_mapping(__nextp, __va, __pa, __sz, __prot) : \ + create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot)) +#define fixmap_pgd_next (pgtable_l4_enabled ? \ + (uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd) +#define trampoline_pgd_next (pgtable_l4_enabled ? \ + (uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd) +#define early_dtb_pgd_next (pgtable_l4_enabled ? \ + (uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd) #else #define pgd_next_t pte_t #define alloc_pgd_next(__va) pt_ops.alloc_pte(__va) #define get_pgd_next_virt(__pa) pt_ops.get_pte_virt(__pa) #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ create_pte_mapping(__nextp, __va, __pa, __sz, __prot) -#define fixmap_pgd_next fixmap_pte +#define fixmap_pgd_next ((uintptr_t)fixmap_pte) +#define early_dtb_pgd_next ((uintptr_t)early_dtb_pmd) +#define create_pud_mapping(__pmdp, __va, __pa, __sz, __prot) #define create_pmd_mapping(__pmdp, __va, __pa, __sz, __prot) -#endif +#endif /* __PAGETABLE_PMD_FOLDED */ void __init create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, @@ -452,13 +524,14 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) #ifdef CONFIG_XIP_KERNEL #define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base)) +extern char _xiprom[], _exiprom[], __data_loc; + /* called from head.S with MMU off */ asmlinkage void __init __copy_data(void) { - void *from = (void *)(&_sdata); - void *end = (void *)(&_end); + void *from = (void *)(&__data_loc); void *to = (void *)CONFIG_PHYS_RAM_BASE; - size_t sz = (size_t)(end - from + 1); + size_t sz = (size_t)((uintptr_t)(&_end) - (uintptr_t)(&_sdata)); memcpy(to, from, sz); } @@ -501,6 +574,57 @@ static __init pgprot_t pgprot_from_va(uintptr_t va) } #endif /* CONFIG_STRICT_KERNEL_RWX */ +#ifdef CONFIG_64BIT +static void __init disable_pgtable_l4(void) +{ + pgtable_l4_enabled = false; + kernel_map.page_offset = PAGE_OFFSET_L3; + satp_mode = SATP_MODE_39; +} + +/* + * There is a simple way to determine if 4-level is supported by the + * underlying hardware: establish 1:1 mapping in 4-level page table mode + * then read SATP to see if the configuration was taken into account + * meaning sv48 is supported. + */ +static __init void set_satp_mode(void) +{ + u64 identity_satp, hw_satp; + uintptr_t set_satp_mode_pmd; + + set_satp_mode_pmd = ((unsigned long)set_satp_mode) & PMD_MASK; + create_pgd_mapping(early_pg_dir, + set_satp_mode_pmd, (uintptr_t)early_pud, + PGDIR_SIZE, PAGE_TABLE); + create_pud_mapping(early_pud, + set_satp_mode_pmd, (uintptr_t)early_pmd, + PUD_SIZE, PAGE_TABLE); + /* Handle the case where set_satp_mode straddles 2 PMDs */ + create_pmd_mapping(early_pmd, + set_satp_mode_pmd, set_satp_mode_pmd, + PMD_SIZE, PAGE_KERNEL_EXEC); + create_pmd_mapping(early_pmd, + set_satp_mode_pmd + PMD_SIZE, + set_satp_mode_pmd + PMD_SIZE, + PMD_SIZE, PAGE_KERNEL_EXEC); + + identity_satp = PFN_DOWN((uintptr_t)&early_pg_dir) | satp_mode; + + local_flush_tlb_all(); + csr_write(CSR_SATP, identity_satp); + hw_satp = csr_swap(CSR_SATP, 0ULL); + local_flush_tlb_all(); + + if (hw_satp != identity_satp) + disable_pgtable_l4(); + + memset(early_pg_dir, 0, PAGE_SIZE); + memset(early_pud, 0, PAGE_SIZE); + memset(early_pmd, 0, PAGE_SIZE); +} +#endif + /* * setup_vm() is called from head.S with MMU-off. * @@ -565,10 +689,15 @@ static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa) uintptr_t pa = dtb_pa & ~(PMD_SIZE - 1); create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, - IS_ENABLED(CONFIG_64BIT) ? (uintptr_t)early_dtb_pmd : pa, + IS_ENABLED(CONFIG_64BIT) ? early_dtb_pgd_next : pa, PGDIR_SIZE, IS_ENABLED(CONFIG_64BIT) ? PAGE_TABLE : PAGE_KERNEL); + if (pgtable_l4_enabled) { + create_pud_mapping(early_dtb_pud, DTB_EARLY_BASE_VA, + (uintptr_t)early_dtb_pmd, PUD_SIZE, PAGE_TABLE); + } + if (IS_ENABLED(CONFIG_64BIT)) { create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, pa, PMD_SIZE, PAGE_KERNEL); @@ -590,11 +719,64 @@ static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa) dtb_early_pa = dtb_pa; } +/* + * MMU is not enabled, the page tables are allocated directly using + * early_pmd/pud/p4d and the address returned is the physical one. + */ +void __init pt_ops_set_early(void) +{ + pt_ops.alloc_pte = alloc_pte_early; + pt_ops.get_pte_virt = get_pte_virt_early; +#ifndef __PAGETABLE_PMD_FOLDED + pt_ops.alloc_pmd = alloc_pmd_early; + pt_ops.get_pmd_virt = get_pmd_virt_early; + pt_ops.alloc_pud = alloc_pud_early; + pt_ops.get_pud_virt = get_pud_virt_early; +#endif +} + +/* + * MMU is enabled but page table setup is not complete yet. + * fixmap page table alloc functions must be used as a means to temporarily + * map the allocated physical pages since the linear mapping does not exist yet. + * + * Note that this is called with MMU disabled, hence kernel_mapping_pa_to_va, + * but it will be used as described above. + */ +void __init pt_ops_set_fixmap(void) +{ + pt_ops.alloc_pte = kernel_mapping_pa_to_va((uintptr_t)alloc_pte_fixmap); + pt_ops.get_pte_virt = kernel_mapping_pa_to_va((uintptr_t)get_pte_virt_fixmap); +#ifndef __PAGETABLE_PMD_FOLDED + pt_ops.alloc_pmd = kernel_mapping_pa_to_va((uintptr_t)alloc_pmd_fixmap); + pt_ops.get_pmd_virt = kernel_mapping_pa_to_va((uintptr_t)get_pmd_virt_fixmap); + pt_ops.alloc_pud = kernel_mapping_pa_to_va((uintptr_t)alloc_pud_fixmap); + pt_ops.get_pud_virt = kernel_mapping_pa_to_va((uintptr_t)get_pud_virt_fixmap); +#endif +} + +/* + * MMU is enabled and page table setup is complete, so from now, we can use + * generic page allocation functions to setup page table. + */ +void __init pt_ops_set_late(void) +{ + pt_ops.alloc_pte = alloc_pte_late; + pt_ops.get_pte_virt = get_pte_virt_late; +#ifndef __PAGETABLE_PMD_FOLDED + pt_ops.alloc_pmd = alloc_pmd_late; + pt_ops.get_pmd_virt = get_pmd_virt_late; + pt_ops.alloc_pud = alloc_pud_late; + pt_ops.get_pud_virt = get_pud_virt_late; +#endif +} + asmlinkage void __init setup_vm(uintptr_t dtb_pa) { pmd_t __maybe_unused fix_bmap_spmd, fix_bmap_epmd; kernel_map.virt_addr = KERNEL_LINK_ADDR; + kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); #ifdef CONFIG_XIP_KERNEL kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; @@ -609,11 +791,24 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) kernel_map.phys_addr = (uintptr_t)(&_start); kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr; #endif + +#if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) + set_satp_mode(); +#endif + kernel_map.va_pa_offset = PAGE_OFFSET - kernel_map.phys_addr; kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr; riscv_pfn_base = PFN_DOWN(kernel_map.phys_addr); + /* + * The default maximal physical memory size is KERN_VIRT_SIZE for 32-bit + * kernel, whereas for 64-bit kernel, the end of the virtual address + * space is occupied by the modules/BPF/kernel mappings which reduces + * the available size of the linear mapping. + */ + memory_limit = KERN_VIRT_SIZE - (IS_ENABLED(CONFIG_64BIT) ? SZ_4G : 0); + /* Sanity check alignment and size */ BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); BUG_ON((kernel_map.phys_addr % PMD_SIZE) != 0); @@ -626,23 +821,25 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) BUG_ON((kernel_map.virt_addr + kernel_map.size) > ADDRESS_SPACE_END - SZ_4K); #endif - pt_ops.alloc_pte = alloc_pte_early; - pt_ops.get_pte_virt = get_pte_virt_early; -#ifndef __PAGETABLE_PMD_FOLDED - pt_ops.alloc_pmd = alloc_pmd_early; - pt_ops.get_pmd_virt = get_pmd_virt_early; -#endif + pt_ops_set_early(); + /* Setup early PGD for fixmap */ create_pgd_mapping(early_pg_dir, FIXADDR_START, - (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); + fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); #ifndef __PAGETABLE_PMD_FOLDED - /* Setup fixmap PMD */ + /* Setup fixmap PUD and PMD */ + if (pgtable_l4_enabled) + create_pud_mapping(fixmap_pud, FIXADDR_START, + (uintptr_t)fixmap_pmd, PUD_SIZE, PAGE_TABLE); create_pmd_mapping(fixmap_pmd, FIXADDR_START, (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); /* Setup trampoline PGD and PMD */ create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, - (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); + trampoline_pgd_next, PGDIR_SIZE, PAGE_TABLE); + if (pgtable_l4_enabled) + create_pud_mapping(trampoline_pud, kernel_map.virt_addr, + (uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE); #ifdef CONFIG_XIP_KERNEL create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, kernel_map.xiprom, PMD_SIZE, PAGE_KERNEL_EXEC); @@ -670,7 +867,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap * range can not span multiple pmds. */ - BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) + BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); #ifndef __PAGETABLE_PMD_FOLDED @@ -695,6 +892,8 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); } #endif + + pt_ops_set_fixmap(); } static void __init setup_vm_final(void) @@ -703,16 +902,6 @@ static void __init setup_vm_final(void) phys_addr_t pa, start, end; u64 i; - /** - * MMU is enabled at this point. But page table setup is not complete yet. - * fixmap page table alloc functions should be used at this point - */ - pt_ops.alloc_pte = alloc_pte_fixmap; - pt_ops.get_pte_virt = get_pte_virt_fixmap; -#ifndef __PAGETABLE_PMD_FOLDED - pt_ops.alloc_pmd = alloc_pmd_fixmap; - pt_ops.get_pmd_virt = get_pmd_virt_fixmap; -#endif /* Setup swapper PGD for fixmap */ create_pgd_mapping(swapper_pg_dir, FIXADDR_START, __pa_symbol(fixmap_pgd_next), @@ -737,26 +926,24 @@ static void __init setup_vm_final(void) } } -#ifdef CONFIG_64BIT /* Map the kernel */ - create_kernel_page_table(swapper_pg_dir, false); + if (IS_ENABLED(CONFIG_64BIT)) + create_kernel_page_table(swapper_pg_dir, false); + +#ifdef CONFIG_KASAN + kasan_swapper_init(); #endif /* Clear fixmap PTE and PMD mappings */ clear_fixmap(FIX_PTE); clear_fixmap(FIX_PMD); + clear_fixmap(FIX_PUD); /* Move to swapper page table */ - csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); + csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | satp_mode); local_flush_tlb_all(); - /* generic page allocation functions must be used to setup page table */ - pt_ops.alloc_pte = alloc_pte_late; - pt_ops.get_pte_virt = get_pte_virt_late; -#ifndef __PAGETABLE_PMD_FOLDED - pt_ops.alloc_pmd = alloc_pmd_late; - pt_ops.get_pmd_virt = get_pmd_virt_late; -#endif + pt_ops_set_late(); } #else asmlinkage void __init setup_vm(uintptr_t dtb_pa) @@ -792,12 +979,10 @@ static void __init reserve_crashkernel(void) * since it doesn't make much sense and we have limited memory * resources. */ -#ifdef CONFIG_CRASH_DUMP if (is_kdump_kernel()) { pr_info("crashkernel: ignoring reservation request\n"); return; } -#endif ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), &crash_size, &crash_base); diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index e26e367a3d..cd1a145257 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -11,44 +11,27 @@ #include #include +/* + * Kasan shadow region must lie at a fixed address across sv39, sv48 and sv57 + * which is right before the kernel. + * + * For sv39, the region is aligned on PGDIR_SIZE so we only need to populate + * the page global directory with kasan_early_shadow_pmd. + * + * For sv48 and sv57, the region is not aligned on PGDIR_SIZE so the mapping + * must be divided as follows: + * - the first PGD entry, although incomplete, is populated with + * kasan_early_shadow_pud/p4d + * - the PGD entries in the middle are populated with kasan_early_shadow_pud/p4d + * - the last PGD entry is shared with the kernel mapping so populated at the + * lower levels pud/p4d + * + * In addition, when shallow populating a kasan region (for example vmalloc), + * this region may also not be aligned on PGDIR size, so we must go down to the + * pud level too. + */ + extern pgd_t early_pg_dir[PTRS_PER_PGD]; -asmlinkage void __init kasan_early_init(void) -{ - uintptr_t i; - pgd_t *pgd = early_pg_dir + pgd_index(KASAN_SHADOW_START); - - BUILD_BUG_ON(KASAN_SHADOW_OFFSET != - KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT))); - - for (i = 0; i < PTRS_PER_PTE; ++i) - set_pte(kasan_early_shadow_pte + i, - pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL)); - - for (i = 0; i < PTRS_PER_PMD; ++i) - set_pmd(kasan_early_shadow_pmd + i, - pfn_pmd(PFN_DOWN - (__pa((uintptr_t) kasan_early_shadow_pte)), - __pgprot(_PAGE_TABLE))); - - for (i = KASAN_SHADOW_START; i < KASAN_SHADOW_END; - i += PGDIR_SIZE, ++pgd) - set_pgd(pgd, - pfn_pgd(PFN_DOWN - (__pa(((uintptr_t) kasan_early_shadow_pmd))), - __pgprot(_PAGE_TABLE))); - - /* init for swapper_pg_dir */ - pgd = pgd_offset_k(KASAN_SHADOW_START); - - for (i = KASAN_SHADOW_START; i < KASAN_SHADOW_END; - i += PGDIR_SIZE, ++pgd) - set_pgd(pgd, - pfn_pgd(PFN_DOWN - (__pa(((uintptr_t) kasan_early_shadow_pmd))), - __pgprot(_PAGE_TABLE))); - - local_flush_tlb_all(); -} static void __init kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned long end) { @@ -72,15 +55,19 @@ static void __init kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned set_pmd(pmd, pfn_pmd(PFN_DOWN(__pa(base_pte)), PAGE_TABLE)); } -static void __init kasan_populate_pmd(pgd_t *pgd, unsigned long vaddr, unsigned long end) +static void __init kasan_populate_pmd(pud_t *pud, unsigned long vaddr, unsigned long end) { phys_addr_t phys_addr; pmd_t *pmdp, *base_pmd; unsigned long next; - base_pmd = (pmd_t *)pgd_page_vaddr(*pgd); - if (base_pmd == lm_alias(kasan_early_shadow_pmd)) + if (pud_none(*pud)) { base_pmd = memblock_alloc(PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); + } else { + base_pmd = (pmd_t *)pud_pgtable(*pud); + if (base_pmd == lm_alias(kasan_early_shadow_pmd)) + base_pmd = memblock_alloc(PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); + } pmdp = base_pmd + pmd_index(vaddr); @@ -104,59 +91,209 @@ static void __init kasan_populate_pmd(pgd_t *pgd, unsigned long vaddr, unsigned * it entirely, memblock could allocate a page at a physical address * where KASAN is not populated yet and then we'd get a page fault. */ - set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE)); + set_pud(pud, pfn_pud(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE)); } -static void __init kasan_populate_pgd(unsigned long vaddr, unsigned long end) +static void __init kasan_populate_pud(pgd_t *pgd, + unsigned long vaddr, unsigned long end, + bool early) +{ + phys_addr_t phys_addr; + pud_t *pudp, *base_pud; + unsigned long next; + + if (early) { + /* + * We can't use pgd_page_vaddr here as it would return a linear + * mapping address but it is not mapped yet, but when populating + * early_pg_dir, we need the physical address and when populating + * swapper_pg_dir, we need the kernel virtual address so use + * pt_ops facility. + */ + base_pud = pt_ops.get_pud_virt(pfn_to_phys(_pgd_pfn(*pgd))); + } else { + base_pud = (pud_t *)pgd_page_vaddr(*pgd); + if (base_pud == lm_alias(kasan_early_shadow_pud)) { + base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE); + memcpy(base_pud, (void *)kasan_early_shadow_pud, + sizeof(pud_t) * PTRS_PER_PUD); + } + } + + pudp = base_pud + pud_index(vaddr); + + do { + next = pud_addr_end(vaddr, end); + + if (pud_none(*pudp) && IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) { + if (early) { + phys_addr = __pa(((uintptr_t)kasan_early_shadow_pmd)); + set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_TABLE)); + continue; + } else { + phys_addr = memblock_phys_alloc(PUD_SIZE, PUD_SIZE); + if (phys_addr) { + set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL)); + continue; + } + } + } + + kasan_populate_pmd(pudp, vaddr, next); + } while (pudp++, vaddr = next, vaddr != end); + + /* + * Wait for the whole PGD to be populated before setting the PGD in + * the page table, otherwise, if we did set the PGD before populating + * it entirely, memblock could allocate a page at a physical address + * where KASAN is not populated yet and then we'd get a page fault. + */ + if (!early) + set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pud)), PAGE_TABLE)); +} + +#define kasan_early_shadow_pgd_next (pgtable_l4_enabled ? \ + (uintptr_t)kasan_early_shadow_pud : \ + (uintptr_t)kasan_early_shadow_pmd) +#define kasan_populate_pgd_next(pgdp, vaddr, next, early) \ + (pgtable_l4_enabled ? \ + kasan_populate_pud(pgdp, vaddr, next, early) : \ + kasan_populate_pmd((pud_t *)pgdp, vaddr, next)) + +static void __init kasan_populate_pgd(pgd_t *pgdp, + unsigned long vaddr, unsigned long end, + bool early) { phys_addr_t phys_addr; - pgd_t *pgdp = pgd_offset_k(vaddr); unsigned long next; do { next = pgd_addr_end(vaddr, end); - /* - * pgdp can't be none since kasan_early_init initialized all KASAN - * shadow region with kasan_early_shadow_pmd: if this is stillthe case, - * that means we can try to allocate a hugepage as a replacement. - */ - if (pgd_page_vaddr(*pgdp) == (unsigned long)lm_alias(kasan_early_shadow_pmd) && - IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) { - phys_addr = memblock_phys_alloc(PGDIR_SIZE, PGDIR_SIZE); - if (phys_addr) { - set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_KERNEL)); + if (IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) { + if (early) { + phys_addr = __pa((uintptr_t)kasan_early_shadow_pgd_next); + set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_TABLE)); continue; + } else if (pgd_page_vaddr(*pgdp) == + (unsigned long)lm_alias(kasan_early_shadow_pgd_next)) { + /* + * pgdp can't be none since kasan_early_init + * initialized all KASAN shadow region with + * kasan_early_shadow_pud: if this is still the + * case, that means we can try to allocate a + * hugepage as a replacement. + */ + phys_addr = memblock_phys_alloc(PGDIR_SIZE, PGDIR_SIZE); + if (phys_addr) { + set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_KERNEL)); + continue; + } } } - kasan_populate_pmd(pgdp, vaddr, next); + kasan_populate_pgd_next(pgdp, vaddr, next, early); } while (pgdp++, vaddr = next, vaddr != end); } +asmlinkage void __init kasan_early_init(void) +{ + uintptr_t i; + + BUILD_BUG_ON(KASAN_SHADOW_OFFSET != + KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT))); + + for (i = 0; i < PTRS_PER_PTE; ++i) + set_pte(kasan_early_shadow_pte + i, + pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL)); + + for (i = 0; i < PTRS_PER_PMD; ++i) + set_pmd(kasan_early_shadow_pmd + i, + pfn_pmd(PFN_DOWN + (__pa((uintptr_t)kasan_early_shadow_pte)), + PAGE_TABLE)); + + if (pgtable_l4_enabled) { + for (i = 0; i < PTRS_PER_PUD; ++i) + set_pud(kasan_early_shadow_pud + i, + pfn_pud(PFN_DOWN + (__pa(((uintptr_t)kasan_early_shadow_pmd))), + PAGE_TABLE)); + } + + kasan_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START), + KASAN_SHADOW_START, KASAN_SHADOW_END, true); + + local_flush_tlb_all(); +} + +void __init kasan_swapper_init(void) +{ + kasan_populate_pgd(pgd_offset_k(KASAN_SHADOW_START), + KASAN_SHADOW_START, KASAN_SHADOW_END, true); + + local_flush_tlb_all(); +} + static void __init kasan_populate(void *start, void *end) { unsigned long vaddr = (unsigned long)start & PAGE_MASK; unsigned long vend = PAGE_ALIGN((unsigned long)end); - kasan_populate_pgd(vaddr, vend); + kasan_populate_pgd(pgd_offset_k(vaddr), vaddr, vend, false); local_flush_tlb_all(); memset(start, KASAN_SHADOW_INIT, end - start); } +static void __init kasan_shallow_populate_pud(pgd_t *pgdp, + unsigned long vaddr, unsigned long end, + bool kasan_populate) +{ + unsigned long next; + pud_t *pudp, *base_pud; + pmd_t *base_pmd; + bool is_kasan_pmd; + + base_pud = (pud_t *)pgd_page_vaddr(*pgdp); + pudp = base_pud + pud_index(vaddr); + + if (kasan_populate) + memcpy(base_pud, (void *)kasan_early_shadow_pgd_next, + sizeof(pud_t) * PTRS_PER_PUD); + + do { + next = pud_addr_end(vaddr, end); + is_kasan_pmd = (pud_pgtable(*pudp) == lm_alias(kasan_early_shadow_pmd)); + + if (is_kasan_pmd) { + base_pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + set_pud(pudp, pfn_pud(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE)); + } + } while (pudp++, vaddr = next, vaddr != end); +} + static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long end) { unsigned long next; void *p; pgd_t *pgd_k = pgd_offset_k(vaddr); + bool is_kasan_pgd_next; do { next = pgd_addr_end(vaddr, end); - if (pgd_page_vaddr(*pgd_k) == (unsigned long)lm_alias(kasan_early_shadow_pmd)) { + is_kasan_pgd_next = (pgd_page_vaddr(*pgd_k) == + (unsigned long)lm_alias(kasan_early_shadow_pgd_next)); + + if (is_kasan_pgd_next) { p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); set_pgd(pgd_k, pfn_pgd(PFN_DOWN(__pa(p)), PAGE_TABLE)); } + + if (IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) + continue; + + kasan_shallow_populate_pud(pgd_k, vaddr, next, is_kasan_pgd_next); } while (pgd_k++, vaddr = next, vaddr != end); } diff --git a/arch/riscv/mm/physaddr.c b/arch/riscv/mm/physaddr.c index e7fd0c253c..19cf25a74e 100644 --- a/arch/riscv/mm/physaddr.c +++ b/arch/riscv/mm/physaddr.c @@ -8,12 +8,10 @@ phys_addr_t __virt_to_phys(unsigned long x) { - phys_addr_t y = x - PAGE_OFFSET; - /* * Boundary checking aginst the kernel linear mapping space. */ - WARN(y >= KERN_VIRT_SIZE, + WARN(!is_linear_mapping(x) && !is_kernel_mapping(x), "virt_to_phys used for non-linear address: %pK (%pS)\n", (void *)x, (void *)x); diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 64f8201237..37ed760d00 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -32,7 +32,6 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, unsigned long size, unsigned long stride) { struct cpumask *cmask = mm_cpumask(mm); - struct cpumask hmask; unsigned int cpuid; bool broadcast; @@ -46,9 +45,7 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, unsigned long asid = atomic_long_read(&mm->context.id); if (broadcast) { - riscv_cpuid_to_hartid_mask(cmask, &hmask); - sbi_remote_sfence_vma_asid(cpumask_bits(&hmask), - start, size, asid); + sbi_remote_sfence_vma_asid(cmask, start, size, asid); } else if (size <= stride) { local_flush_tlb_page_asid(start, asid); } else { @@ -56,9 +53,7 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, } } else { if (broadcast) { - riscv_cpuid_to_hartid_mask(cmask, &hmask); - sbi_remote_sfence_vma(cpumask_bits(&hmask), - start, size); + sbi_remote_sfence_vma(cmask, start, size); } else if (size <= stride) { local_flush_tlb_page(start); } else { diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 75c1e99968..f42d9cd3b6 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -71,6 +71,7 @@ struct rv_jit_context { int ninsns; int epilogue_offset; int *offset; /* BPF to RV */ + int nexentries; unsigned long flags; int stack_size; }; diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index e6497424cb..529a83b85c 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -799,11 +799,10 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx); /* - * temp_tcc = tcc - 1; - * if (tcc < 0) + * if (--tcc < 0) * goto out; */ - emit(rv_addi(RV_REG_T1, RV_REG_TCC, -1), ctx); + emit(rv_addi(RV_REG_TCC, RV_REG_TCC, -1), ctx); off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_bcc(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); @@ -829,7 +828,6 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) if (is_12b_check(off, insn)) return -1; emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx); - emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx); /* Epilogue jumps to *(t0 + 4). */ __build_epilogue(true, ctx); return 0; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 3af4131c22..0bcda99d1d 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -5,6 +5,7 @@ * */ +#include #include #include #include "bpf_jit.h" @@ -27,6 +28,21 @@ static const int regmap[] = { [BPF_REG_AX] = RV_REG_T0, }; +static const int pt_regmap[] = { + [RV_REG_A0] = offsetof(struct pt_regs, a0), + [RV_REG_A1] = offsetof(struct pt_regs, a1), + [RV_REG_A2] = offsetof(struct pt_regs, a2), + [RV_REG_A3] = offsetof(struct pt_regs, a3), + [RV_REG_A4] = offsetof(struct pt_regs, a4), + [RV_REG_A5] = offsetof(struct pt_regs, a5), + [RV_REG_S1] = offsetof(struct pt_regs, s1), + [RV_REG_S2] = offsetof(struct pt_regs, s2), + [RV_REG_S3] = offsetof(struct pt_regs, s3), + [RV_REG_S4] = offsetof(struct pt_regs, s4), + [RV_REG_S5] = offsetof(struct pt_regs, s5), + [RV_REG_T0] = offsetof(struct pt_regs, t0), +}; + enum { RV_CTX_F_SEEN_TAIL_CALL = 0, RV_CTX_F_SEEN_CALL = RV_REG_RA, @@ -311,12 +327,12 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); - /* if (TCC-- < 0) + /* if (--TCC < 0) * goto out; */ - emit_addi(RV_REG_T1, tcc, -1, ctx); + emit_addi(RV_REG_TCC, tcc, -1, ctx); off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); - emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); + emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); /* prog = array->ptrs[index]; * if (!prog) @@ -336,7 +352,6 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) if (is_12b_check(off, insn)) return -1; emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); - emit_mv(RV_REG_TCC, RV_REG_T1, ctx); __build_epilogue(true, ctx); return 0; } @@ -440,6 +455,70 @@ static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) return 0; } +#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) +#define BPF_FIXUP_REG_MASK GENMASK(31, 27) + +bool ex_handler_bpf(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); + int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); + + *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; + regs->epc = (unsigned long)&ex->fixup - offset; + + return true; +} + +/* For accesses to BTF pointers, add an entry to the exception table */ +static int add_exception_handler(const struct bpf_insn *insn, + struct rv_jit_context *ctx, + int dst_reg, int insn_len) +{ + struct exception_table_entry *ex; + unsigned long pc; + off_t offset; + + if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM) + return 0; + + if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) + return -EINVAL; + + if (WARN_ON_ONCE(insn_len > ctx->ninsns)) + return -EINVAL; + + if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) + return -EINVAL; + + ex = &ctx->prog->aux->extable[ctx->nexentries]; + pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len]; + + offset = pc - (long)&ex->insn; + if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) + return -ERANGE; + ex->insn = offset; + + /* + * Since the extable follows the program, the fixup offset is always + * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value + * to keep things simple, and put the destination register in the upper + * bits. We don't need to worry about buildtime or runtime sort + * modifying the upper bits because the table is already sorted, and + * isn't part of the main exception table. + */ + offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); + if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) + return -ERANGE; + + ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | + FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); + ex->type = EX_TYPE_BPF; + + ctx->nexentries++; + return 0; +} + int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, bool extra_pass) { @@ -893,52 +972,86 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, /* LDX: dst = *(size *)(src + off) */ case BPF_LDX | BPF_MEM | BPF_B: - if (is_12b_int(off)) { - emit(rv_lbu(rd, off, rs), ctx); - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); - emit(rv_lbu(rd, 0, RV_REG_T1), ctx); - if (insn_is_zext(&insn[1])) - return 1; - break; case BPF_LDX | BPF_MEM | BPF_H: - if (is_12b_int(off)) { - emit(rv_lhu(rd, off, rs), ctx); - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); - emit(rv_lhu(rd, 0, RV_REG_T1), ctx); - if (insn_is_zext(&insn[1])) - return 1; - break; case BPF_LDX | BPF_MEM | BPF_W: - if (is_12b_int(off)) { - emit(rv_lwu(rd, off, rs), ctx); - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); - emit(rv_lwu(rd, 0, RV_REG_T1), ctx); - if (insn_is_zext(&insn[1])) - return 1; - break; case BPF_LDX | BPF_MEM | BPF_DW: - if (is_12b_int(off)) { - emit_ld(rd, off, rs, ctx); + case BPF_LDX | BPF_PROBE_MEM | BPF_B: + case BPF_LDX | BPF_PROBE_MEM | BPF_H: + case BPF_LDX | BPF_PROBE_MEM | BPF_W: + case BPF_LDX | BPF_PROBE_MEM | BPF_DW: + { + int insn_len, insns_start; + + switch (BPF_SIZE(code)) { + case BPF_B: + if (is_12b_int(off)) { + insns_start = ctx->ninsns; + emit(rv_lbu(rd, off, rs), ctx); + insn_len = ctx->ninsns - insns_start; + break; + } + + emit_imm(RV_REG_T1, off, ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); + insns_start = ctx->ninsns; + emit(rv_lbu(rd, 0, RV_REG_T1), ctx); + insn_len = ctx->ninsns - insns_start; + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_H: + if (is_12b_int(off)) { + insns_start = ctx->ninsns; + emit(rv_lhu(rd, off, rs), ctx); + insn_len = ctx->ninsns - insns_start; + break; + } + + emit_imm(RV_REG_T1, off, ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); + insns_start = ctx->ninsns; + emit(rv_lhu(rd, 0, RV_REG_T1), ctx); + insn_len = ctx->ninsns - insns_start; + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_W: + if (is_12b_int(off)) { + insns_start = ctx->ninsns; + emit(rv_lwu(rd, off, rs), ctx); + insn_len = ctx->ninsns - insns_start; + break; + } + + emit_imm(RV_REG_T1, off, ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); + insns_start = ctx->ninsns; + emit(rv_lwu(rd, 0, RV_REG_T1), ctx); + insn_len = ctx->ninsns - insns_start; + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_DW: + if (is_12b_int(off)) { + insns_start = ctx->ninsns; + emit_ld(rd, off, rs, ctx); + insn_len = ctx->ninsns - insns_start; + break; + } + + emit_imm(RV_REG_T1, off, ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); + insns_start = ctx->ninsns; + emit_ld(rd, 0, RV_REG_T1, ctx); + insn_len = ctx->ninsns - insns_start; break; } - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); - emit_ld(rd, 0, RV_REG_T1, ctx); + ret = add_exception_handler(insn, ctx, rd, insn_len); + if (ret) + return ret; break; - + } /* speculation barrier */ case BPF_ST | BPF_NOSPEC: break; diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 753d85bdfa..be743d700a 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -11,7 +11,7 @@ #include "bpf_jit.h" /* Number of iterations to try until offsets converge. */ -#define NR_JIT_ITERATIONS 16 +#define NR_JIT_ITERATIONS 32 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) { @@ -41,12 +41,12 @@ bool bpf_jit_needs_zext(void) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { + unsigned int prog_size = 0, extable_size = 0; bool tmp_blinded = false, extra_pass = false; struct bpf_prog *tmp, *orig_prog = prog; int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; - unsigned int image_size = 0; if (!prog->jit_requested) return orig_prog; @@ -73,7 +73,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (ctx->offset) { extra_pass = true; - image_size = sizeof(*ctx->insns) * ctx->ninsns; + prog_size = sizeof(*ctx->insns) * ctx->ninsns; goto skip_init_ctx; } @@ -102,10 +102,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (ctx->ninsns == prev_ninsns) { if (jit_data->header) break; + /* obtain the actual image size */ + extable_size = prog->aux->num_exentries * + sizeof(struct exception_table_entry); + prog_size = sizeof(*ctx->insns) * ctx->ninsns; - image_size = sizeof(*ctx->insns) * ctx->ninsns; jit_data->header = - bpf_jit_binary_alloc(image_size, + bpf_jit_binary_alloc(prog_size + extable_size, &jit_data->image, sizeof(u32), bpf_fill_ill_insns); @@ -131,9 +134,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) goto out_offset; } + if (extable_size) + prog->aux->extable = (void *)ctx->insns + prog_size; + skip_init_ctx: pass++; ctx->ninsns = 0; + ctx->nexentries = 0; bpf_jit_build_prologue(ctx); if (build_body(ctx, extra_pass, NULL)) { @@ -144,11 +151,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) bpf_jit_build_epilogue(ctx); if (bpf_jit_enable > 1) - bpf_jit_dump(prog->len, image_size, pass, ctx->insns); + bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); prog->bpf_func = (void *)ctx->insns; prog->jited = 1; - prog->jited_len = image_size; + prog->jited_len = prog_size; bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns); diff --git a/arch/sh/Kbuild b/arch/sh/Kbuild index 48c2a091a0..be17188097 100644 --- a/arch/sh/Kbuild +++ b/arch/sh/Kbuild @@ -2,3 +2,6 @@ obj-y += kernel/ mm/ boards/ obj-$(CONFIG_SH_FPU_EMU) += math-emu/ obj-$(CONFIG_USE_BUILTIN_DTB) += boot/dts/ + +# for cleaning +subdir- += boot diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 6904f4bdbf..2474a04cea 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -34,7 +34,6 @@ config SUPERH select HAVE_FAST_GUP if MMU select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER - select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_FTRACE_MCOUNT_RECORD select HAVE_HW_BREAKPOINT select HAVE_IOREMAP_PROT if MMU && !X2TLB @@ -56,7 +55,6 @@ config SUPERH select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select IRQ_FORCED_THREADING - select MAY_HAVE_SPARSE_IRQ select MODULES_USE_ELF_RELA select NEED_SG_DMA_LENGTH select NO_DMA if !MMU && !DMA_COHERENT diff --git a/arch/sh/Makefile b/arch/sh/Makefile index 88ddb6f1c7..b39412bf91 100644 --- a/arch/sh/Makefile +++ b/arch/sh/Makefile @@ -198,10 +198,6 @@ compressed: zImage archprepare: $(Q)$(MAKE) $(build)=arch/sh/tools include/generated/machtypes.h -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - $(Q)$(MAKE) $(clean)=arch/sh/kernel/vsyscall - archheaders: $(Q)$(MAKE) $(build)=arch/sh/kernel/syscalls all diff --git a/arch/sh/boards/mach-ap325rxa/setup.c b/arch/sh/boards/mach-ap325rxa/setup.c index bac8a058eb..c77b5f00a6 100644 --- a/arch/sh/boards/mach-ap325rxa/setup.c +++ b/arch/sh/boards/mach-ap325rxa/setup.c @@ -560,7 +560,7 @@ static void __init ap325rxa_mv_mem_reserve(void) if (!phys) panic("Failed to allocate CEU memory\n"); - memblock_free(phys, size); + memblock_phys_free(phys, size); memblock_remove(phys, size); ceu_dma_membase = phys; diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c index bab91a9912..4c9522dd35 100644 --- a/arch/sh/boards/mach-ecovec24/setup.c +++ b/arch/sh/boards/mach-ecovec24/setup.c @@ -886,7 +886,7 @@ static struct asoc_simple_card_info fsi_da7210_info = { .card = "FSIB-DA7210", .codec = "da7210.0-001a", .platform = "sh_fsi.0", - .daifmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBM_CFM, + .daifmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBP_CFP, .cpu_dai = { .name = "fsib-dai", }, @@ -1502,7 +1502,7 @@ static void __init ecovec_mv_mem_reserve(void) if (!phys) panic("Failed to allocate CEU0 memory\n"); - memblock_free(phys, size); + memblock_phys_free(phys, size); memblock_remove(phys, size); ceu0_dma_membase = phys; @@ -1510,7 +1510,7 @@ static void __init ecovec_mv_mem_reserve(void) if (!phys) panic("Failed to allocate CEU1 memory\n"); - memblock_free(phys, size); + memblock_phys_free(phys, size); memblock_remove(phys, size); ceu1_dma_membase = phys; } diff --git a/arch/sh/boards/mach-kfr2r09/setup.c b/arch/sh/boards/mach-kfr2r09/setup.c index eeb5ce341e..20f4db778e 100644 --- a/arch/sh/boards/mach-kfr2r09/setup.c +++ b/arch/sh/boards/mach-kfr2r09/setup.c @@ -633,7 +633,7 @@ static void __init kfr2r09_mv_mem_reserve(void) if (!phys) panic("Failed to allocate CEU memory\n"); - memblock_free(phys, size); + memblock_phys_free(phys, size); memblock_remove(phys, size); ceu_dma_membase = phys; diff --git a/arch/sh/boards/mach-migor/setup.c b/arch/sh/boards/mach-migor/setup.c index 6703a2122c..f60061283c 100644 --- a/arch/sh/boards/mach-migor/setup.c +++ b/arch/sh/boards/mach-migor/setup.c @@ -633,7 +633,7 @@ static void __init migor_mv_mem_reserve(void) if (!phys) panic("Failed to allocate CEU memory\n"); - memblock_free(phys, size); + memblock_phys_free(phys, size); memblock_remove(phys, size); ceu_dma_membase = phys; diff --git a/arch/sh/boards/mach-se/7724/setup.c b/arch/sh/boards/mach-se/7724/setup.c index 8d6541ba01..b60a2626e1 100644 --- a/arch/sh/boards/mach-se/7724/setup.c +++ b/arch/sh/boards/mach-se/7724/setup.c @@ -305,7 +305,7 @@ static struct asoc_simple_card_info fsi_ak4642_info = { .card = "FSIA-AK4642", .codec = "ak4642-codec.0-0012", .platform = "sh_fsi.0", - .daifmt = SND_SOC_DAIFMT_LEFT_J | SND_SOC_DAIFMT_CBM_CFM, + .daifmt = SND_SOC_DAIFMT_LEFT_J | SND_SOC_DAIFMT_CBP_CFP, .cpu_dai = { .name = "fsia-dai", }, @@ -966,7 +966,7 @@ static void __init ms7724se_mv_mem_reserve(void) if (!phys) panic("Failed to allocate CEU0 memory\n"); - memblock_free(phys, size); + memblock_phys_free(phys, size); memblock_remove(phys, size); ceu0_dma_membase = phys; @@ -974,7 +974,7 @@ static void __init ms7724se_mv_mem_reserve(void) if (!phys) panic("Failed to allocate CEU1 memory\n"); - memblock_free(phys, size); + memblock_phys_free(phys, size); memblock_remove(phys, size); ceu1_dma_membase = phys; } diff --git a/arch/sh/boards/of-generic.c b/arch/sh/boards/of-generic.c index 921d76fc33..f7f3e618e8 100644 --- a/arch/sh/boards/of-generic.c +++ b/arch/sh/boards/of-generic.c @@ -62,9 +62,8 @@ static void sh_of_smp_probe(void) init_cpu_possible(cpumask_of(0)); for_each_of_cpu_node(np) { - const __be32 *cell = of_get_property(np, "reg", NULL); - u64 id = -1; - if (cell) id = of_read_number(cell, of_n_addr_cells(np)); + u64 id = of_get_cpu_hwid(np, 0); + if (id < NR_CPUS) { if (!method) of_property_read_string(np, "enable-method", &method); diff --git a/arch/sh/boot/Makefile b/arch/sh/boot/Makefile index c081e7e2d6..1f5d2df3c7 100644 --- a/arch/sh/boot/Makefile +++ b/arch/sh/boot/Makefile @@ -19,16 +19,16 @@ CONFIG_ZERO_PAGE_OFFSET ?= 0x00001000 CONFIG_ENTRY_OFFSET ?= 0x00001000 CONFIG_PHYSICAL_START ?= $(CONFIG_MEMORY_START) -suffix-y := bin -suffix-$(CONFIG_KERNEL_GZIP) := gz -suffix-$(CONFIG_KERNEL_BZIP2) := bz2 -suffix-$(CONFIG_KERNEL_LZMA) := lzma -suffix-$(CONFIG_KERNEL_XZ) := xz -suffix-$(CONFIG_KERNEL_LZO) := lzo +suffix_y := bin +suffix_$(CONFIG_KERNEL_GZIP) := gz +suffix_$(CONFIG_KERNEL_BZIP2) := bz2 +suffix_$(CONFIG_KERNEL_LZMA) := lzma +suffix_$(CONFIG_KERNEL_XZ) := xz +suffix_$(CONFIG_KERNEL_LZO) := lzo targets := zImage vmlinux.srec romImage uImage uImage.srec uImage.gz \ - uImage.bz2 uImage.lzma uImage.xz uImage.lzo uImage.bin -extra-y += vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \ + uImage.bz2 uImage.lzma uImage.xz uImage.lzo uImage.bin \ + vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \ vmlinux.bin.xz vmlinux.bin.lzo subdir- := compressed romimage @@ -106,10 +106,10 @@ OBJCOPYFLAGS_uImage.srec := -I binary -O srec $(obj)/uImage.srec: $(obj)/uImage FORCE $(call if_changed,objcopy) -$(obj)/uImage: $(obj)/uImage.$(suffix-y) +$(obj)/uImage: $(obj)/uImage.$(suffix_y) @ln -sf $(notdir $<) $@ @echo ' Image $@ is ready' export CONFIG_PAGE_OFFSET CONFIG_MEMORY_START CONFIG_BOOT_LINK_OFFSET \ CONFIG_PHYSICAL_START CONFIG_ZERO_PAGE_OFFSET CONFIG_ENTRY_OFFSET \ - KERNEL_MEMORY suffix-y + KERNEL_MEMORY suffix_y diff --git a/arch/sh/boot/compressed/.gitignore b/arch/sh/boot/compressed/.gitignore index 37aa530573..cd16663bc7 100644 --- a/arch/sh/boot/compressed/.gitignore +++ b/arch/sh/boot/compressed/.gitignore @@ -1,7 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -ashiftrt.S -ashldi3.c -ashlsi3.S -ashrsi3.S -lshrsi3.S vmlinux.bin.* diff --git a/arch/sh/boot/compressed/Makefile b/arch/sh/boot/compressed/Makefile index 589d2d8a57..591125c42d 100644 --- a/arch/sh/boot/compressed/Makefile +++ b/arch/sh/boot/compressed/Makefile @@ -5,12 +5,18 @@ # create a compressed vmlinux image from the original vmlinux # -targets := vmlinux vmlinux.bin vmlinux.bin.gz \ - vmlinux.bin.bz2 vmlinux.bin.lzma \ - vmlinux.bin.xz vmlinux.bin.lzo \ - head_32.o misc.o piggy.o +OBJECTS := head_32.o misc.o cache.o piggy.o \ + ashiftrt.o ashldi3.o ashrsi3.o ashlsi3.o lshrsi3.o -OBJECTS = $(obj)/head_32.o $(obj)/misc.o $(obj)/cache.o +# These were previously generated files. When you are building the kernel +# with O=, make sure to remove the stale files in the output tree. Otherwise, +# the build system wrongly compiles the stale ones. +ifdef building_out_of_srctree +$(shell rm -f $(addprefix $(obj)/, ashiftrt.S ashldi3.c ashrsi3.S ashlsi3.S lshrsi3.S)) +endif + +targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 \ + vmlinux.bin.lzma vmlinux.bin.xz vmlinux.bin.lzo $(OBJECTS) GCOV_PROFILE := n @@ -33,42 +39,28 @@ ccflags-remove-$(CONFIG_MCOUNT) += -pg LDFLAGS_vmlinux := --oformat $(ld-bfd) -Ttext $(IMAGE_OFFSET) -e startup \ -T $(obj)/../../kernel/vmlinux.lds -# -# Pull in the necessary libgcc bits from the in-kernel implementation. -# -lib1funcs-y := ashiftrt.S ashldi3.c ashrsi3.S ashlsi3.S lshrsi3.S -lib1funcs-obj := \ - $(addsuffix .o, $(basename $(addprefix $(obj)/, $(lib1funcs-y)))) +KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING -lib1funcs-dir := $(srctree)/arch/$(SRCARCH)/lib - -KBUILD_CFLAGS += -I$(lib1funcs-dir) -DDISABLE_BRANCH_PROFILING - -$(addprefix $(obj)/,$(lib1funcs-y)): $(obj)/%: $(lib1funcs-dir)/% FORCE - $(call cmd,shipped) - -$(obj)/vmlinux: $(OBJECTS) $(obj)/piggy.o $(lib1funcs-obj) FORCE +$(obj)/vmlinux: $(addprefix $(obj)/, $(OBJECTS)) FORCE $(call if_changed,ld) $(obj)/vmlinux.bin: vmlinux FORCE $(call if_changed,objcopy) -vmlinux.bin.all-y := $(obj)/vmlinux.bin - -$(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE +$(obj)/vmlinux.bin.gz: $(obj)/vmlinux.bin FORCE $(call if_changed,gzip) -$(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE - $(call if_changed,bzip2) -$(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzma) -$(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,xzkern) -$(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzo) +$(obj)/vmlinux.bin.bz2: $(obj)/vmlinux.bin FORCE + $(call if_changed,bzip2_with_size) +$(obj)/vmlinux.bin.lzma: $(obj)/vmlinux.bin FORCE + $(call if_changed,lzma_with_size) +$(obj)/vmlinux.bin.xz: $(obj)/vmlinux.bin FORCE + $(call if_changed,xzkern_with_size) +$(obj)/vmlinux.bin.lzo: $(obj)/vmlinux.bin FORCE + $(call if_changed,lzo_with_size) OBJCOPYFLAGS += -R .empty_zero_page LDFLAGS_piggy.o := -r --format binary --oformat $(ld-bfd) -T -$(obj)/piggy.o: $(obj)/vmlinux.scr $(obj)/vmlinux.bin.$(suffix-y) FORCE +$(obj)/piggy.o: $(obj)/vmlinux.scr $(obj)/vmlinux.bin.$(suffix_y) FORCE $(call if_changed,ld) diff --git a/arch/sh/boot/compressed/misc.c b/arch/sh/boot/compressed/misc.c index a03b6680a9..ca05c99a3d 100644 --- a/arch/sh/boot/compressed/misc.c +++ b/arch/sh/boot/compressed/misc.c @@ -115,6 +115,9 @@ void __stack_chk_fail(void) void ftrace_stub(void) { } +void arch_ftrace_ops_list_func(void) +{ +} #define stackalign 4 diff --git a/arch/sh/boot/dts/Makefile b/arch/sh/boot/dts/Makefile index c17d65b82a..4a6dec9714 100644 --- a/arch/sh/boot/dts/Makefile +++ b/arch/sh/boot/dts/Makefile @@ -1,4 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -ifneq ($(CONFIG_BUILTIN_DTB_SOURCE),"") -obj-$(CONFIG_USE_BUILTIN_DTB) += $(patsubst "%",%,$(CONFIG_BUILTIN_DTB_SOURCE)).dtb.o -endif +obj-$(CONFIG_USE_BUILTIN_DTB) += $(addsuffix .dtb.o, $(CONFIG_BUILTIN_DTB_SOURCE)) diff --git a/arch/sh/boot/dts/j2_mimas_v2.dts b/arch/sh/boot/dts/j2_mimas_v2.dts index 9f4742fab3..fa9562f78d 100644 --- a/arch/sh/boot/dts/j2_mimas_v2.dts +++ b/arch/sh/boot/dts/j2_mimas_v2.dts @@ -88,6 +88,8 @@ uart0: serial@100 { clock-frequency = <125000000>; compatible = "xlnx,xps-uartlite-1.00.a"; current-speed = <19200>; + xlnx,use-parity = <0>; + xlnx,data-bits = <8>; device_type = "serial"; interrupts = <0x12>; port-number = <0>; diff --git a/arch/sh/configs/sdk7786_defconfig b/arch/sh/configs/sdk7786_defconfig index f776a1d0d2..a8662b6927 100644 --- a/arch/sh/configs/sdk7786_defconfig +++ b/arch/sh/configs/sdk7786_defconfig @@ -5,7 +5,6 @@ CONFIG_BSD_PROCESS_ACCT=y CONFIG_BSD_PROCESS_ACCT_V3=y CONFIG_AUDIT=y CONFIG_AUDITSYSCALL=y -CONFIG_TREE_PREEMPT_RCU=y CONFIG_RCU_TRACE=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y diff --git a/arch/sh/include/asm/bitops.h b/arch/sh/include/asm/bitops.h index 3b6c7b5b7e..10ceb0d6b5 100644 --- a/arch/sh/include/asm/bitops.h +++ b/arch/sh/include/asm/bitops.h @@ -68,6 +68,5 @@ static inline unsigned long __ffs(unsigned long word) #include #include -#include #endif /* __ASM_SH_BITOPS_H */ diff --git a/arch/sh/include/asm/cacheflush.h b/arch/sh/include/asm/cacheflush.h index 372afa82fe..481a664287 100644 --- a/arch/sh/include/asm/cacheflush.h +++ b/arch/sh/include/asm/cacheflush.h @@ -42,7 +42,7 @@ extern void flush_cache_page(struct vm_area_struct *vma, extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -extern void flush_dcache_page(struct page *page); +void flush_dcache_page(struct page *page); extern void flush_icache_range(unsigned long start, unsigned long end); #define flush_icache_user_range flush_icache_range extern void flush_icache_page(struct vm_area_struct *vma, diff --git a/arch/sh/include/asm/checksum_32.h b/arch/sh/include/asm/checksum_32.h index 1a391e3a76..a6501b856f 100644 --- a/arch/sh/include/asm/checksum_32.h +++ b/arch/sh/include/asm/checksum_32.h @@ -84,7 +84,8 @@ static inline __sum16 csum_fold(__wsum sum) */ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) { - unsigned int sum, __dummy0, __dummy1; + __wsum sum; + unsigned int __dummy0, __dummy1; __asm__ __volatile__( "mov.l @%1+, %0\n\t" @@ -197,6 +198,6 @@ static inline __wsum csum_and_copy_to_user(const void *src, { if (!access_ok(dst, len)) return 0; - return csum_partial_copy_generic((__force const void *)src, dst, len); + return csum_partial_copy_generic(src, (__force void *)dst, len); } #endif /* __ASM_SH_CHECKSUM_H */ diff --git a/arch/sh/include/asm/irq.h b/arch/sh/include/asm/irq.h index 839551ce39..1c4923502f 100644 --- a/arch/sh/include/asm/irq.h +++ b/arch/sh/include/asm/irq.h @@ -5,17 +5,6 @@ #include #include -/* - * Only legacy non-sparseirq platforms have to set a reasonably sane - * value here. sparseirq platforms allocate their irq_descs on the fly, - * so will expand automatically based on the number of registered IRQs. - */ -#ifdef CONFIG_SPARSE_IRQ -# define NR_IRQS 8 -#else -# define NR_IRQS 512 -#endif - /* * This is a special IRQ number for indicating that no IRQ has been * triggered and to simply ignore the IRQ dispatch. This is a special diff --git a/arch/sh/include/asm/kprobes.h b/arch/sh/include/asm/kprobes.h index 6171682f77..eeba83e0a7 100644 --- a/arch/sh/include/asm/kprobes.h +++ b/arch/sh/include/asm/kprobes.h @@ -26,7 +26,7 @@ typedef insn_size_t kprobe_opcode_t; struct kprobe; void arch_remove_kprobe(struct kprobe *); -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); /* Architecture specific copy of original instruction*/ struct arch_specific_insn { diff --git a/arch/sh/include/asm/processor_32.h b/arch/sh/include/asm/processor_32.h index aa92cc9338..45240ec6b8 100644 --- a/arch/sh/include/asm/processor_32.h +++ b/arch/sh/include/asm/processor_32.h @@ -180,7 +180,7 @@ static inline void show_code(struct pt_regs *regs) } #endif -extern unsigned long get_wchan(struct task_struct *p); +extern unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[15]) diff --git a/arch/sh/include/asm/spinlock_types.h b/arch/sh/include/asm/spinlock_types.h index e82369f286..907bda4b16 100644 --- a/arch/sh/include/asm/spinlock_types.h +++ b/arch/sh/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef __ASM_SH_SPINLOCK_TYPES_H #define __ASM_SH_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/sh/include/asm/syscall_32.h b/arch/sh/include/asm/syscall_32.h index cb51a75283..d87738eebe 100644 --- a/arch/sh/include/asm/syscall_32.h +++ b/arch/sh/include/asm/syscall_32.h @@ -57,18 +57,6 @@ static inline void syscall_get_arguments(struct task_struct *task, args[0] = regs->regs[4]; } -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - const unsigned long *args) -{ - regs->regs[1] = args[5]; - regs->regs[0] = args[4]; - regs->regs[7] = args[3]; - regs->regs[6] = args[2]; - regs->regs[5] = args[1]; - regs->regs[4] = args[0]; -} - static inline int syscall_get_arch(struct task_struct *task) { int arch = AUDIT_ARCH_SH; diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index 73f3b48d4a..8867bb04b0 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h @@ -68,7 +68,7 @@ struct __large_struct { unsigned long buf[100]; }; ({ \ long __gu_err = -EFAULT; \ unsigned long __gu_val = 0; \ - const __typeof__(*(ptr)) *__gu_addr = (ptr); \ + const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ if (likely(access_ok(__gu_addr, (size)))) \ __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ @@ -124,7 +124,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) * Clear the area and return remaining number of bytes * (on failure. Usually it's 0.) */ -__kernel_size_t __clear_user(void *addr, __kernel_size_t size); +__kernel_size_t __clear_user(void __user *addr, __kernel_size_t size); #define clear_user(addr,n) \ ({ \ diff --git a/arch/sh/kernel/cpu/sh4/sq.c b/arch/sh/kernel/cpu/sh4/sq.c index d432164b23..a76b94e41e 100644 --- a/arch/sh/kernel/cpu/sh4/sq.c +++ b/arch/sh/kernel/cpu/sh4/sq.c @@ -324,6 +324,7 @@ static struct attribute *sq_sysfs_attrs[] = { &mapping_attr.attr, NULL, }; +ATTRIBUTE_GROUPS(sq_sysfs); static const struct sysfs_ops sq_sysfs_ops = { .show = sq_sysfs_show, @@ -332,7 +333,7 @@ static const struct sysfs_ops sq_sysfs_ops = { static struct kobj_type ktype_percpu_entry = { .sysfs_ops = &sq_sysfs_ops, - .default_attrs = sq_sysfs_attrs, + .default_groups = sq_sysfs_groups, }; static int sq_dev_add(struct device *dev, struct subsys_interface *sif) diff --git a/arch/sh/kernel/crash_dump.c b/arch/sh/kernel/crash_dump.c index a9086127b1..5b41b59698 100644 --- a/arch/sh/kernel/crash_dump.c +++ b/arch/sh/kernel/crash_dump.c @@ -26,7 +26,7 @@ ssize_t copy_oldmem_page(unsigned long pfn, char *buf, size_t csize, unsigned long offset, int userbuf) { - void *vaddr; + void __iomem *vaddr; if (!csize) return 0; @@ -34,7 +34,7 @@ ssize_t copy_oldmem_page(unsigned long pfn, char *buf, vaddr = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE); if (userbuf) { - if (copy_to_user(buf, (vaddr + offset), csize)) { + if (copy_to_user((void __user *)buf, (vaddr + offset), csize)) { iounmap(vaddr); return -EFAULT; } diff --git a/arch/sh/kernel/ftrace.c b/arch/sh/kernel/ftrace.c index 295c43315b..930001bb8c 100644 --- a/arch/sh/kernel/ftrace.c +++ b/arch/sh/kernel/ftrace.c @@ -252,11 +252,6 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) return ftrace_modify_code(rec->ip, old, new); } - -int __init ftrace_dyn_arch_init(void) -{ - return 0; -} #endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_FUNCTION_GRAPH_TRACER diff --git a/arch/sh/kernel/kprobes.c b/arch/sh/kernel/kprobes.c index 1c7f358ef0..aed1ea8e2c 100644 --- a/arch/sh/kernel/kprobes.c +++ b/arch/sh/kernel/kprobes.c @@ -207,7 +207,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, ri->fp = NULL; /* Replace the return addr with trampoline addr */ - regs->pr = (unsigned long)kretprobe_trampoline; + regs->pr = (unsigned long)__kretprobe_trampoline; } static int __kprobes kprobe_handler(struct pt_regs *regs) @@ -293,17 +293,17 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) */ static void __used kretprobe_trampoline_holder(void) { - asm volatile (".globl kretprobe_trampoline\n" - "kretprobe_trampoline:\n\t" + asm volatile (".globl __kretprobe_trampoline\n" + "__kretprobe_trampoline:\n\t" "nop\n"); } /* - * Called when we hit the probe point at kretprobe_trampoline + * Called when we hit the probe point at __kretprobe_trampoline */ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) { - regs->pc = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL); + regs->pc = __kretprobe_trampoline_handler(regs, NULL); return 1; } @@ -442,7 +442,7 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, } static struct kprobe trampoline_p = { - .addr = (kprobe_opcode_t *)&kretprobe_trampoline, + .addr = (kprobe_opcode_t *)&__kretprobe_trampoline, .pre_handler = trampoline_probe_handler }; diff --git a/arch/sh/kernel/process_32.c b/arch/sh/kernel/process_32.c index 717de05c81..1c28e3cddb 100644 --- a/arch/sh/kernel/process_32.c +++ b/arch/sh/kernel/process_32.c @@ -182,13 +182,10 @@ __switch_to(struct task_struct *prev, struct task_struct *next) return prev; } -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { unsigned long pc; - if (!p || p == current || task_is_running(p)) - return 0; - /* * The same comment as on the Alpha applies here, too ... */ diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl index 208f131659..2de85c977f 100644 --- a/arch/sh/kernel/syscalls/syscall.tbl +++ b/arch/sh/kernel/syscalls/syscall.tbl @@ -451,3 +451,5 @@ 446 common landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/sh/kernel/traps.c b/arch/sh/kernel/traps.c index e76b221570..01884054ae 100644 --- a/arch/sh/kernel/traps.c +++ b/arch/sh/kernel/traps.c @@ -20,7 +20,7 @@ static DEFINE_SPINLOCK(die_lock); -void die(const char *str, struct pt_regs *regs, long err) +void __noreturn die(const char *str, struct pt_regs *regs, long err) { static int die_counter; @@ -57,7 +57,7 @@ void die(const char *str, struct pt_regs *regs, long err) if (panic_on_oops) panic("Fatal exception"); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } void die_if_kernel(const char *str, struct pt_regs *regs, long err) diff --git a/arch/sh/kernel/traps_32.c b/arch/sh/kernel/traps_32.c index b62ad0ba23..b3c715bc25 100644 --- a/arch/sh/kernel/traps_32.c +++ b/arch/sh/kernel/traps_32.c @@ -490,7 +490,7 @@ asmlinkage void do_address_error(struct pt_regs *regs, inc_unaligned_user_access(); oldfs = force_uaccess_begin(); - if (copy_from_user(&instruction, (insn_size_t *)(regs->pc & ~1), + if (copy_from_user(&instruction, (insn_size_t __user *)(regs->pc & ~1), sizeof(instruction))) { force_uaccess_end(oldfs); goto uspace_segv; @@ -614,7 +614,7 @@ asmlinkage void do_reserved_inst(void) unsigned short inst = 0; int err; - get_user(inst, (unsigned short*)regs->pc); + get_user(inst, (unsigned short __user *)regs->pc); err = do_fpu_inst(inst, regs); if (!err) { @@ -699,9 +699,9 @@ asmlinkage void do_illegal_slot_inst(void) return; #ifdef CONFIG_SH_FPU_EMU - get_user(inst, (unsigned short *)regs->pc + 1); + get_user(inst, (unsigned short __user *)regs->pc + 1); if (!do_fpu_inst(inst, regs)) { - get_user(inst, (unsigned short *)regs->pc); + get_user(inst, (unsigned short __user *)regs->pc); if (!emulate_branch(inst, regs)) return; /* fault in branch.*/ diff --git a/arch/sh/math-emu/math.c b/arch/sh/math-emu/math.c index 615ba932c3..cdaef6501d 100644 --- a/arch/sh/math-emu/math.c +++ b/arch/sh/math-emu/math.c @@ -51,8 +51,8 @@ #define Rn (regs->regs[n]) #define Rm (regs->regs[m]) -#define WRITE(d,a) ({if(put_user(d, (typeof (d)*)a)) return -EFAULT;}) -#define READ(d,a) ({if(get_user(d, (typeof (d)*)a)) return -EFAULT;}) +#define MWRITE(d,a) ({if(put_user(d, (typeof (d) __user *)a)) return -EFAULT;}) +#define MREAD(d,a) ({if(get_user(d, (typeof (d) __user *)a)) return -EFAULT;}) #define PACK_S(r,f) FP_PACK_SP(&r,f) #define UNPACK_S(f,r) FP_UNPACK_SP(f,&r) @@ -157,11 +157,11 @@ fmov_idx_reg(struct sh_fpu_soft_struct *fregs, struct pt_regs *regs, int m, { if (FPSCR_SZ) { FMOV_EXT(n); - READ(FRn, Rm + R0 + 4); + MREAD(FRn, Rm + R0 + 4); n++; - READ(FRn, Rm + R0); + MREAD(FRn, Rm + R0); } else { - READ(FRn, Rm + R0); + MREAD(FRn, Rm + R0); } return 0; @@ -173,11 +173,11 @@ fmov_mem_reg(struct sh_fpu_soft_struct *fregs, struct pt_regs *regs, int m, { if (FPSCR_SZ) { FMOV_EXT(n); - READ(FRn, Rm + 4); + MREAD(FRn, Rm + 4); n++; - READ(FRn, Rm); + MREAD(FRn, Rm); } else { - READ(FRn, Rm); + MREAD(FRn, Rm); } return 0; @@ -189,12 +189,12 @@ fmov_inc_reg(struct sh_fpu_soft_struct *fregs, struct pt_regs *regs, int m, { if (FPSCR_SZ) { FMOV_EXT(n); - READ(FRn, Rm + 4); + MREAD(FRn, Rm + 4); n++; - READ(FRn, Rm); + MREAD(FRn, Rm); Rm += 8; } else { - READ(FRn, Rm); + MREAD(FRn, Rm); Rm += 4; } @@ -207,11 +207,11 @@ fmov_reg_idx(struct sh_fpu_soft_struct *fregs, struct pt_regs *regs, int m, { if (FPSCR_SZ) { FMOV_EXT(m); - WRITE(FRm, Rn + R0 + 4); + MWRITE(FRm, Rn + R0 + 4); m++; - WRITE(FRm, Rn + R0); + MWRITE(FRm, Rn + R0); } else { - WRITE(FRm, Rn + R0); + MWRITE(FRm, Rn + R0); } return 0; @@ -223,11 +223,11 @@ fmov_reg_mem(struct sh_fpu_soft_struct *fregs, struct pt_regs *regs, int m, { if (FPSCR_SZ) { FMOV_EXT(m); - WRITE(FRm, Rn + 4); + MWRITE(FRm, Rn + 4); m++; - WRITE(FRm, Rn); + MWRITE(FRm, Rn); } else { - WRITE(FRm, Rn); + MWRITE(FRm, Rn); } return 0; @@ -240,12 +240,12 @@ fmov_reg_dec(struct sh_fpu_soft_struct *fregs, struct pt_regs *regs, int m, if (FPSCR_SZ) { FMOV_EXT(m); Rn -= 8; - WRITE(FRm, Rn + 4); + MWRITE(FRm, Rn + 4); m++; - WRITE(FRm, Rn); + MWRITE(FRm, Rn); } else { Rn -= 4; - WRITE(FRm, Rn); + MWRITE(FRm, Rn); } return 0; @@ -445,11 +445,11 @@ id_sys(struct sh_fpu_soft_struct *fregs, struct pt_regs *regs, u16 code) case 0x4052: case 0x4062: Rn -= 4; - WRITE(*reg, Rn); + MWRITE(*reg, Rn); break; case 0x4056: case 0x4066: - READ(*reg, Rn); + MREAD(*reg, Rn); Rn += 4; break; default: diff --git a/arch/sh/mm/alignment.c b/arch/sh/mm/alignment.c index fb517b82a8..3a76a766f4 100644 --- a/arch/sh/mm/alignment.c +++ b/arch/sh/mm/alignment.c @@ -140,7 +140,7 @@ static int alignment_proc_open(struct inode *inode, struct file *file) static ssize_t alignment_proc_write(struct file *file, const char __user *buffer, size_t count, loff_t *pos) { - int *data = PDE_DATA(file_inode(file)); + int *data = pde_data(file_inode(file)); char mode; if (count > 0) { @@ -161,7 +161,7 @@ static const struct proc_ops alignment_proc_ops = { }; /* - * This needs to be done after sysctl_init, otherwise sys/ will be + * This needs to be done after sysctl_init_bases(), otherwise sys/ will be * overwritten. Actually, this shouldn't be in sys/ at all since * it isn't a sysctl, and it doesn't contain sysctl information. * We now locate it in /proc/cpu/alignment instead. diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 88a1f453d7..e175667b13 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -238,8 +238,6 @@ no_context(struct pt_regs *regs, unsigned long error_code, show_fault_oops(regs, address); die("Oops", regs, error_code); - bust_spinlocks(0); - do_exit(SIGKILL); } static void @@ -487,17 +485,15 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, if (mm_fault_error(regs, error_code, address, fault)) return; - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* - * No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ - goto retry; - } + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + goto retry; } mmap_read_unlock(mm); diff --git a/arch/sh/mm/nommu.c b/arch/sh/mm/nommu.c index 8b4504413c..78c4b6e6d3 100644 --- a/arch/sh/mm/nommu.c +++ b/arch/sh/mm/nommu.c @@ -28,9 +28,9 @@ __kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n) return 0; } -__kernel_size_t __clear_user(void *to, __kernel_size_t n) +__kernel_size_t __clear_user(void __user *to, __kernel_size_t n) { - memset(to, 0, n); + memset((__force void *)to, 0, n); return 0; } diff --git a/arch/sparc/Kbuild b/arch/sparc/Kbuild index c9e574906a..71cb3d934b 100644 --- a/arch/sparc/Kbuild +++ b/arch/sparc/Kbuild @@ -9,3 +9,6 @@ obj-y += math-emu/ obj-y += net/ obj-y += crypto/ obj-$(CONFIG_SPARC64) += vdso/ + +# for cleaning +subdir- += boot diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index b120ed947f..1cab1b284f 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -53,8 +53,9 @@ config SPARC32 def_bool !64BIT select ARCH_32BIT_OFF_T select ARCH_HAS_SYNC_DMA_FOR_CPU - select GENERIC_ATOMIC64 select CLZ_TAB + select DMA_DIRECT_REMAP + select GENERIC_ATOMIC64 select HAVE_UID16 select OLD_SIGACTION select ZONE_DMA @@ -96,6 +97,9 @@ config SPARC64 select PCI_DOMAINS if PCI select ARCH_HAS_GIGANTIC_PAGE select HAVE_SOFTIRQ_ON_OWN_STACK + select HAVE_SETUP_PER_CPU_AREA + select NEED_PER_CPU_EMBED_FIRST_CHUNK + select NEED_PER_CPU_PAGE_FIRST_CHUNK config ARCH_PROC_KCORE_TEXT def_bool y @@ -122,15 +126,6 @@ config AUDIT_ARCH bool default y -config HAVE_SETUP_PER_CPU_AREA - def_bool y if SPARC64 - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y if SPARC64 - -config NEED_PER_CPU_PAGE_FIRST_CHUNK - def_bool y if SPARC64 - config MMU bool default y diff --git a/arch/sparc/Makefile b/arch/sparc/Makefile index 24fb5a99f4..c7008bbebc 100644 --- a/arch/sparc/Makefile +++ b/arch/sparc/Makefile @@ -75,9 +75,6 @@ install: sh $(srctree)/$(boot)/install.sh $(KERNELRELEASE) $(KBUILD_IMAGE) \ System.map "$(INSTALL_PATH)" -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - archheaders: $(Q)$(MAKE) $(build)=arch/sparc/kernel/syscalls all diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c index 86a654cce5..06b7becfcb 100644 --- a/arch/sparc/crypto/sha1_glue.c +++ b/arch/sparc/crypto/sha1_glue.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -26,17 +27,6 @@ asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data, unsigned int rounds); -static int sha1_sparc64_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} - static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data, unsigned int len, unsigned int partial) { @@ -128,7 +118,7 @@ static int sha1_sparc64_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_sparc64_init, + .init = sha1_base_init, .update = sha1_sparc64_update, .final = sha1_sparc64_final, .export = sha1_sparc64_export, diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c index 60ec524cf9..285561a1cd 100644 --- a/arch/sparc/crypto/sha256_glue.c +++ b/arch/sparc/crypto/sha256_glue.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -26,38 +27,6 @@ asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data, unsigned int rounds); -static int sha224_sparc64_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - - return 0; -} - -static int sha256_sparc64_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; -} - static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data, unsigned int len, unsigned int partial) { @@ -158,7 +127,7 @@ static int sha256_sparc64_import(struct shash_desc *desc, const void *in) static struct shash_alg sha256_alg = { .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_sparc64_init, + .init = sha256_base_init, .update = sha256_sparc64_update, .final = sha256_sparc64_final, .export = sha256_sparc64_export, @@ -176,7 +145,7 @@ static struct shash_alg sha256_alg = { static struct shash_alg sha224_alg = { .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_sparc64_init, + .init = sha224_base_init, .update = sha256_sparc64_update, .final = sha224_sparc64_final, .descsize = sizeof(struct sha256_state), diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index 273ce21918..d66efa4ec5 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -25,38 +26,6 @@ asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data, unsigned int rounds); -static int sha512_sparc64_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA512_H0; - sctx->state[1] = SHA512_H1; - sctx->state[2] = SHA512_H2; - sctx->state[3] = SHA512_H3; - sctx->state[4] = SHA512_H4; - sctx->state[5] = SHA512_H5; - sctx->state[6] = SHA512_H6; - sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static int sha384_sparc64_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA384_H0; - sctx->state[1] = SHA384_H1; - sctx->state[2] = SHA384_H2; - sctx->state[3] = SHA384_H3; - sctx->state[4] = SHA384_H4; - sctx->state[5] = SHA384_H5; - sctx->state[6] = SHA384_H6; - sctx->state[7] = SHA384_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data, unsigned int len, unsigned int partial) { @@ -146,7 +115,7 @@ static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash) static struct shash_alg sha512 = { .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_sparc64_init, + .init = sha512_base_init, .update = sha512_sparc64_update, .final = sha512_sparc64_final, .descsize = sizeof(struct sha512_state), @@ -161,7 +130,7 @@ static struct shash_alg sha512 = { static struct shash_alg sha384 = { .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_sparc64_init, + .init = sha384_base_init, .update = sha512_sparc64_update, .final = sha384_sparc64_final, .descsize = sizeof(struct sha512_state), diff --git a/arch/sparc/include/asm/bitops_32.h b/arch/sparc/include/asm/bitops_32.h index 0ceff3b915..889afa9f99 100644 --- a/arch/sparc/include/asm/bitops_32.h +++ b/arch/sparc/include/asm/bitops_32.h @@ -100,7 +100,6 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) #include #include #include -#include #include #include diff --git a/arch/sparc/include/asm/bitops_64.h b/arch/sparc/include/asm/bitops_64.h index ca7ea59134..005a8ae858 100644 --- a/arch/sparc/include/asm/bitops_64.h +++ b/arch/sparc/include/asm/bitops_64.h @@ -52,8 +52,6 @@ unsigned int __arch_hweight8(unsigned int w); #include #endif /* __KERNEL__ */ -#include - #ifdef __KERNEL__ #include diff --git a/arch/sparc/include/asm/kprobes.h b/arch/sparc/include/asm/kprobes.h index bfcaa6326c..06c2bc767e 100644 --- a/arch/sparc/include/asm/kprobes.h +++ b/arch/sparc/include/asm/kprobes.h @@ -24,7 +24,7 @@ do { flushi(&(p)->ainsn.insn[0]); \ flushi(&(p)->ainsn.insn[1]); \ } while (0) -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); /* Architecture specific copy of original instruction*/ struct arch_specific_insn { diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h index b6242f7771..647bf0ac7b 100644 --- a/arch/sparc/include/asm/processor_32.h +++ b/arch/sparc/include/asm/processor_32.h @@ -89,7 +89,7 @@ static inline void start_thread(struct pt_regs * regs, unsigned long pc, /* Free all resources held by a thread. */ #define release_thread(tsk) do { } while(0) -unsigned long get_wchan(struct task_struct *); +unsigned long __get_wchan(struct task_struct *); #define task_pt_regs(tsk) ((tsk)->thread.kregs) #define KSTK_EIP(tsk) ((tsk)->thread.kregs->pc) diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index 5cf145f18f..ae851e8fce 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -183,7 +183,7 @@ do { \ /* Free all resources held by a thread. */ #define release_thread(tsk) do { } while (0) -unsigned long get_wchan(struct task_struct *task); +unsigned long __get_wchan(struct task_struct *task); #define task_pt_regs(tsk) (task_thread_info(tsk)->kregs) #define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc) diff --git a/arch/sparc/include/asm/ptrace.h b/arch/sparc/include/asm/ptrace.h index 71dd82b43c..d1419e6690 100644 --- a/arch/sparc/include/asm/ptrace.h +++ b/arch/sparc/include/asm/ptrace.h @@ -26,12 +26,12 @@ static inline bool pt_regs_clear_syscall(struct pt_regs *regs) return (regs->tstate &= ~TSTATE_SYSCALL); } -#define arch_ptrace_stop_needed(exit_code, info) \ +#define arch_ptrace_stop_needed() \ ({ flush_user_windows(); \ get_thread_wsaved() != 0; \ }) -#define arch_ptrace_stop(exit_code, info) \ +#define arch_ptrace_stop() \ synchronize_user_stack() #define current_pt_regs() \ @@ -129,12 +129,12 @@ static inline bool pt_regs_clear_syscall(struct pt_regs *regs) return (regs->psr &= ~PSR_SYSCALL); } -#define arch_ptrace_stop_needed(exit_code, info) \ +#define arch_ptrace_stop_needed() \ ({ flush_user_windows(); \ current_thread_info()->w_saved != 0; \ }) -#define arch_ptrace_stop(exit_code, info) \ +#define arch_ptrace_stop() \ synchronize_user_stack() #define current_pt_regs() \ diff --git a/arch/sparc/include/asm/syscall.h b/arch/sparc/include/asm/syscall.h index 62a5a78804..20c109ac8c 100644 --- a/arch/sparc/include/asm/syscall.h +++ b/arch/sparc/include/asm/syscall.h @@ -117,16 +117,6 @@ static inline void syscall_get_arguments(struct task_struct *task, } } -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - const unsigned long *args) -{ - unsigned int i; - - for (i = 0; i < 6; i++) - regs->u_regs[UREG_I0 + i] = args[i]; -} - static inline int syscall_get_arch(struct task_struct *task) { #if defined(CONFIG_SPARC64) && defined(CONFIG_COMPAT) diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index e80ee8641a..2672dd03fa 100644 --- a/arch/sparc/include/uapi/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -124,6 +124,9 @@ #define SO_BUF_LOCK 0x0051 +#define SO_RESERVE_MEM 0x0052 + + #if !defined(__KERNEL__) diff --git a/arch/sparc/kernel/audit.c b/arch/sparc/kernel/audit.c index a6e91bf34d..b092274eca 100644 --- a/arch/sparc/kernel/audit.c +++ b/arch/sparc/kernel/audit.c @@ -48,15 +48,17 @@ int audit_classify_syscall(int abi, unsigned int syscall) #endif switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/sparc/kernel/compat_audit.c b/arch/sparc/kernel/compat_audit.c index 10eeb4f15b..f1ea0005a7 100644 --- a/arch/sparc/kernel/compat_audit.c +++ b/arch/sparc/kernel/compat_audit.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #define __32bit_syscall_numbers__ +#include #include #include "kernel.h" @@ -32,14 +33,16 @@ int sparc32_classify_syscall(unsigned int syscall) { switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/sparc/kernel/ftrace.c b/arch/sparc/kernel/ftrace.c index 684b84ce39..eaead3da8e 100644 --- a/arch/sparc/kernel/ftrace.c +++ b/arch/sparc/kernel/ftrace.c @@ -82,11 +82,6 @@ int ftrace_update_ftrace_func(ftrace_func_t func) new = ftrace_call_replace(ip, (unsigned long)func); return ftrace_modify_code(ip, old, new); } - -int __init ftrace_dyn_arch_init(void) -{ - return 0; -} #endif #ifdef CONFIG_FUNCTION_GRAPH_TRACER diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c index 7ceae24b0c..57a72c46ed 100644 --- a/arch/sparc/kernel/ioport.c +++ b/arch/sparc/kernel/ioport.c @@ -52,17 +52,6 @@ #include #include -/* This function must make sure that caches and memory are coherent after DMA - * On LEON systems without cache snooping it flushes the entire D-CACHE. - */ -static inline void dma_make_coherent(unsigned long pa, unsigned long len) -{ - if (sparc_cpu_model == sparc_leon) { - if (!sparc_leon3_snooping_enabled()) - leon_flush_dcache_all(); - } -} - static void __iomem *_sparc_ioremap(struct resource *res, u32 bus, u32 pa, int sz); static void __iomem *_sparc_alloc_io(unsigned int busno, unsigned long phys, unsigned long size, char *name); @@ -311,68 +300,19 @@ arch_initcall(sparc_register_ioport); #endif /* CONFIG_SBUS */ - -/* Allocate and map kernel buffer using consistent mode DMA for a device. - * hwdev should be valid struct pci_dev pointer for PCI devices. - */ -void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, - gfp_t gfp, unsigned long attrs) -{ - unsigned long addr; - void *va; - - if (!size || size > 256 * 1024) /* __get_free_pages() limit */ - return NULL; - - size = PAGE_ALIGN(size); - va = (void *) __get_free_pages(gfp | __GFP_ZERO, get_order(size)); - if (!va) { - printk("%s: no %zd pages\n", __func__, size >> PAGE_SHIFT); - return NULL; - } - - addr = sparc_dma_alloc_resource(dev, size); - if (!addr) - goto err_nomem; - - srmmu_mapiorange(0, virt_to_phys(va), addr, size); - - *dma_handle = virt_to_phys(va); - return (void *)addr; - -err_nomem: - free_pages((unsigned long)va, get_order(size)); - return NULL; -} - -/* Free and unmap a consistent DMA buffer. - * cpu_addr is what was returned arch_dma_alloc, size must be the same as what - * was passed into arch_dma_alloc, and likewise dma_addr must be the same as - * what *dma_ndler was set to. +/* + * IIep is write-through, not flushing on cpu to device transfer. * - * References to the memory and mappings associated with cpu_addr/dma_addr - * past this call are illegal. + * On LEON systems without cache snooping, the entire D-CACHE must be flushed to + * make DMA to cacheable memory coherent. */ -void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_addr, unsigned long attrs) -{ - size = PAGE_ALIGN(size); - - if (!sparc_dma_free_resource(cpu_addr, size)) - return; - - dma_make_coherent(dma_addr, size); - srmmu_unmapiorange((unsigned long)cpu_addr, size); - free_pages((unsigned long)phys_to_virt(dma_addr), get_order(size)); -} - -/* IIep is write-through, not flushing on cpu to device transfer. */ - void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, enum dma_data_direction dir) { - if (dir != PCI_DMA_TODEVICE) - dma_make_coherent(paddr, PAGE_ALIGN(size)); + if (dir != PCI_DMA_TODEVICE && + sparc_cpu_model == sparc_leon && + !sparc_leon3_snooping_enabled()) + leon_flush_dcache_all(); } #ifdef CONFIG_PROC_FS diff --git a/arch/sparc/kernel/kprobes.c b/arch/sparc/kernel/kprobes.c index 4c05a4ee6a..535c7b35cb 100644 --- a/arch/sparc/kernel/kprobes.c +++ b/arch/sparc/kernel/kprobes.c @@ -440,7 +440,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, /* Replace the return addr with trampoline addr */ regs->u_regs[UREG_RETPC] = - ((unsigned long)kretprobe_trampoline) - 8; + ((unsigned long)__kretprobe_trampoline) - 8; } /* @@ -451,7 +451,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, { unsigned long orig_ret_address = 0; - orig_ret_address = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL); + orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); regs->tpc = orig_ret_address; regs->tnpc = orig_ret_address + 4; @@ -465,13 +465,13 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, static void __used kretprobe_trampoline_holder(void) { - asm volatile(".global kretprobe_trampoline\n" - "kretprobe_trampoline:\n" + asm volatile(".global __kretprobe_trampoline\n" + "__kretprobe_trampoline:\n" "\tnop\n" "\tnop\n"); } static struct kprobe trampoline_p = { - .addr = (kprobe_opcode_t *) &kretprobe_trampoline, + .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, .pre_handler = trampoline_probe_handler }; @@ -482,7 +482,7 @@ int __init arch_init_kprobes(void) int __kprobes arch_trampoline_kprobe(struct kprobe *p) { - if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline) + if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) return 1; return 0; diff --git a/arch/sparc/kernel/led.c b/arch/sparc/kernel/led.c index 3a66e62eb2..ab657b3597 100644 --- a/arch/sparc/kernel/led.c +++ b/arch/sparc/kernel/led.c @@ -114,18 +114,16 @@ static const struct proc_ops led_proc_ops = { }; #endif -static struct proc_dir_entry *led; - #define LED_VERSION "0.1" static int __init led_init(void) { timer_setup(&led_blink_timer, led_blink, 0); - led = proc_create("led", 0, NULL, &led_proc_ops); - if (!led) +#ifdef CONFIG_PROC_FS + if (!proc_create("led", 0, NULL, &led_proc_ops)) return -ENOMEM; - +#endif printk(KERN_INFO "led: version %s, Lars Kotthoff \n", LED_VERSION); diff --git a/arch/sparc/kernel/pci_msi.c b/arch/sparc/kernel/pci_msi.c index fb5899cbfa..9ed1198576 100644 --- a/arch/sparc/kernel/pci_msi.c +++ b/arch/sparc/kernel/pci_msi.c @@ -146,13 +146,13 @@ static int sparc64_setup_msi_irq(unsigned int *irq_p, msiqid = pick_msiq(pbm); err = ops->msi_setup(pbm, msiqid, msi, - (entry->msi_attrib.is_64 ? 1 : 0)); + (entry->pci.msi_attrib.is_64 ? 1 : 0)); if (err) goto out_msi_free; pbm->msi_irq_table[msi - pbm->msi_first] = *irq_p; - if (entry->msi_attrib.is_64) { + if (entry->pci.msi_attrib.is_64) { msg.address_hi = pbm->msi64_start >> 32; msg.address_lo = pbm->msi64_start & 0xffffffff; } else { diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index bbbe0cfef7..2dc0bf9fe6 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -365,7 +365,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, return 0; } -unsigned long get_wchan(struct task_struct *task) +unsigned long __get_wchan(struct task_struct *task) { unsigned long pc, fp, bias = 0; unsigned long task_base = (unsigned long) task; @@ -373,9 +373,6 @@ unsigned long get_wchan(struct task_struct *task) struct reg_window32 *rw; int count = 0; - if (!task || task == current || task_is_running(task)) - goto out; - fp = task_thread_info(task)->ksp + bias; do { /* Bogus frame pointer? */ diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index d1cc410d2f..f5b2cac866 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -663,7 +663,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return 0; } -unsigned long get_wchan(struct task_struct *task) +unsigned long __get_wchan(struct task_struct *task) { unsigned long pc, fp, bias = 0; struct thread_info *tp; @@ -671,9 +671,6 @@ unsigned long get_wchan(struct task_struct *task) unsigned long ret = 0; int count = 0; - if (!task || task == current || task_is_running(task)) - goto out; - tp = task_thread_info(task); bias = STACK_BIAS; fp = task_thread_info(task)->ksp + bias; diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index 0224d8f19e..a1f78e9dda 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -1526,50 +1526,6 @@ void smp_send_stop(void) smp_call_function(stop_this_cpu, NULL, 0); } -/** - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu - * @cpu: cpu to allocate for - * @size: size allocation in bytes - * @align: alignment - * - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper - * does the right thing for NUMA regardless of the current - * configuration. - * - * RETURNS: - * Pointer to the allocated area on success, NULL on failure. - */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, - size_t align) -{ - const unsigned long goal = __pa(MAX_DMA_ADDRESS); -#ifdef CONFIG_NUMA - int node = cpu_to_node(cpu); - void *ptr; - - if (!node_online(node) || !NODE_DATA(node)) { - ptr = memblock_alloc_from(size, align, goal); - pr_info("cpu %d has no node %d or node-local memory\n", - cpu, node); - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", - cpu, size, __pa(ptr)); - } else { - ptr = memblock_alloc_try_nid(size, align, goal, - MEMBLOCK_ALLOC_ACCESSIBLE, node); - pr_debug("per cpu data for cpu%d %lu bytes on node%d at " - "%016lx\n", cpu, size, node, __pa(ptr)); - } - return ptr; -#else - return memblock_alloc_from(size, align, goal); -#endif -} - -static void __init pcpu_free_bootmem(void *ptr, size_t size) -{ - memblock_free(__pa(ptr), size); -} - static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) { if (cpu_to_node(from) == cpu_to_node(to)) @@ -1578,57 +1534,9 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) return REMOTE_DISTANCE; } -static void __init pcpu_populate_pte(unsigned long addr) +static int __init pcpu_cpu_to_node(int cpu) { - pgd_t *pgd = pgd_offset_k(addr); - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - - if (pgd_none(*pgd)) { - pud_t *new; - - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - pgd_populate(&init_mm, pgd, new); - } - - p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { - pud_t *new; - - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - p4d_populate(&init_mm, p4d, new); - } - - pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { - pmd_t *new; - - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - pud_populate(&init_mm, pud, new); - } - - pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) { - pte_t *new; - - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - pmd_populate_kernel(&init_mm, pmd, new); - } - - return; - -err_alloc: - panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + return cpu_to_node(cpu); } void __init setup_per_cpu_areas(void) @@ -1641,8 +1549,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, 4 << 20, pcpu_cpu_distance, - pcpu_alloc_bootmem, - pcpu_free_bootmem); + pcpu_cpu_to_node); if (rc) pr_warn("PERCPU: %s allocator failed (%d), " "falling back to page size\n", @@ -1650,9 +1557,7 @@ void __init setup_per_cpu_areas(void) } if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, - pcpu_alloc_bootmem, - pcpu_free_bootmem, - pcpu_populate_pte); + pcpu_cpu_to_node); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl index c37764dc76..4398cc6fb6 100644 --- a/arch/sparc/kernel/syscalls/syscall.tbl +++ b/arch/sparc/kernel/syscalls/syscall.tbl @@ -494,3 +494,5 @@ 446 common landlock_restrict_self sys_landlock_restrict_self # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/sparc/kernel/traps_32.c b/arch/sparc/kernel/traps_32.c index 5630e5a395..179aabfa71 100644 --- a/arch/sparc/kernel/traps_32.c +++ b/arch/sparc/kernel/traps_32.c @@ -86,9 +86,7 @@ void __noreturn die_if_kernel(char *str, struct pt_regs *regs) } printk("Instruction DUMP:"); instruction_dump ((unsigned long *) regs->pc); - if(regs->psr & PSR_PS) - do_exit(SIGKILL); - do_exit(SIGSEGV); + make_task_dead((regs->psr & PSR_PS) ? SIGKILL : SIGSEGV); } void do_hw_interrupt(struct pt_regs *regs, unsigned long type) diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index 6863025ed5..21077821f4 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -2559,9 +2559,7 @@ void __noreturn die_if_kernel(char *str, struct pt_regs *regs) } if (panic_on_oops) panic("Fatal exception"); - if (regs->tstate & TSTATE_PRIV) - do_exit(SIGKILL); - do_exit(SIGSEGV); + make_task_dead((regs->tstate & TSTATE_PRIV)? SIGKILL : SIGSEGV); } EXPORT_SYMBOL(die_if_kernel); diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index fa858626b8..ad569d9bd1 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -200,17 +200,15 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + /* No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ - goto retry; - } + goto retry; } mmap_read_unlock(mm); @@ -248,7 +246,6 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, } unhandled_fault(address, tsk, regs); - do_exit(SIGKILL); /* * We ran out of memory, or some other thing happened to us that made diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 9a9652a15f..253e070432 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -437,17 +437,15 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* No need to mmap_read_unlock(mm) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + /* No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ - goto retry; - } + goto retry; } mmap_read_unlock(mm); diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index 0dce4b7ff7..9122057871 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c @@ -266,7 +266,7 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign default: printk(KERN_ERR "TSB[%s:%d]: Impossible TSB size %lu, killing process.\n", current->comm, current->pid, tsb_bytes); - do_exit(SIGSEGV); + BUG(); } tte |= pte_sz_bits(page_sz); diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c index 9a2f20cbd4..b1e38784eb 100644 --- a/arch/sparc/net/bpf_jit_comp_64.c +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -227,7 +227,7 @@ static const int bpf2sparc[] = { [BPF_REG_AX] = G7, - /* temporary register for internal BPF JIT */ + /* temporary register for BPF JIT */ [TMP_REG_1] = G1, [TMP_REG_2] = G2, [TMP_REG_3] = G3, @@ -867,7 +867,7 @@ static void emit_tail_call(struct jit_ctx *ctx) emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx); #define OFFSET2 13 - emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx); + emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET2, ctx); emit_nop(ctx); emit_alu_K(ADD, tmp, 1, ctx); diff --git a/arch/um/Kconfig b/arch/um/Kconfig index c18b45f75d..4d398b80ae 100644 --- a/arch/um/Kconfig +++ b/arch/um/Kconfig @@ -14,14 +14,13 @@ config UML select HAVE_ARCH_SECCOMP_FILTER select HAVE_ASM_MODVERSIONS select HAVE_UID16 - select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_BUGVERBOSE select NO_DMA if !UML_DMA_EMULATION + select OF_EARLY_FLATTREE if OF select GENERIC_IRQ_SHOW select GENERIC_CPU_DEVICES select HAVE_GCC_PLUGINS - select SET_FS select TRACE_IRQFLAGS_SUPPORT select TTY # Needed for line.c select HAVE_ARCH_VMAP_STACK diff --git a/arch/um/drivers/net_kern.c b/arch/um/drivers/net_kern.c index 2fc0b038ff..59331384c2 100644 --- a/arch/um/drivers/net_kern.c +++ b/arch/um/drivers/net_kern.c @@ -276,7 +276,7 @@ static const struct ethtool_ops uml_net_ethtool_ops = { void uml_net_setup_etheraddr(struct net_device *dev, char *str) { - unsigned char *addr = dev->dev_addr; + u8 addr[ETH_ALEN]; char *end; int i; @@ -316,6 +316,7 @@ void uml_net_setup_etheraddr(struct net_device *dev, char *str) addr[0] | 0x02, addr[1], addr[2], addr[3], addr[4], addr[5]); } + eth_hw_addr_set(dev, addr); return; random: diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c index fefd343412..69d2d0049a 100644 --- a/arch/um/drivers/ubd_kern.c +++ b/arch/um/drivers/ubd_kern.c @@ -855,8 +855,8 @@ static const struct attribute_group *ubd_attr_groups[] = { NULL, }; -static void ubd_disk_register(int major, u64 size, int unit, - struct gendisk *disk) +static int ubd_disk_register(int major, u64 size, int unit, + struct gendisk *disk) { disk->major = major; disk->first_minor = unit << UBD_SHIFT; @@ -873,7 +873,7 @@ static void ubd_disk_register(int major, u64 size, int unit, disk->private_data = &ubd_devs[unit]; disk->queue = ubd_devs[unit].queue; - device_add_disk(&ubd_devs[unit].pdev.dev, disk, ubd_attr_groups); + return device_add_disk(&ubd_devs[unit].pdev.dev, disk, ubd_attr_groups); } #define ROUND_BLOCK(n) ((n + (SECTOR_SIZE - 1)) & (-SECTOR_SIZE)) @@ -920,10 +920,15 @@ static int ubd_add(int n, char **error_out) blk_queue_write_cache(ubd_dev->queue, true, false); blk_queue_max_segments(ubd_dev->queue, MAX_SG); blk_queue_segment_boundary(ubd_dev->queue, PAGE_SIZE - 1); - ubd_disk_register(UBD_MAJOR, ubd_dev->size, n, disk); + err = ubd_disk_register(UBD_MAJOR, ubd_dev->size, n, disk); + if (err) + goto out_cleanup_disk; + ubd_gendisk[n] = disk; return 0; +out_cleanup_disk: + blk_cleanup_disk(disk); out_cleanup_tags: blk_mq_free_tag_set(&ubd_dev->tag_set); out: diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index cde6db184c..4fc1a5d70d 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -1441,7 +1441,9 @@ static int vector_net_load_bpf_flash(struct net_device *dev, } static void vector_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct vector_private *vp = netdev_priv(netdev); diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index 7755cb4ff9..ba562d68dc 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c @@ -21,6 +21,7 @@ * Based on Virtio MMIO driver by Pawel Moll, copyright 2011-2014, ARM Ltd. */ #include +#include #include #include #include @@ -49,6 +50,7 @@ struct virtio_uml_platform_data { struct virtio_uml_device { struct virtio_device vdev; struct platform_device *pdev; + struct virtio_uml_platform_data *pdata; spinlock_t sock_lock; int sock, req_fd, irq; @@ -149,7 +151,7 @@ static int vhost_user_recv(struct virtio_uml_device *vu_dev, if (rc == -ECONNRESET && vu_dev->registered) { struct virtio_uml_platform_data *pdata; - pdata = vu_dev->pdev->dev.platform_data; + pdata = vu_dev->pdata; virtio_break_device(&vu_dev->vdev); schedule_work(&pdata->conn_broken_wk); @@ -1115,21 +1117,63 @@ void virtio_uml_set_no_vq_suspend(struct virtio_device *vdev, no_vq_suspend ? "dis" : "en"); } +static void vu_of_conn_broken(struct work_struct *wk) +{ + /* + * We can't remove the device from the devicetree so the only thing we + * can do is warn. + */ + WARN_ON(1); +} + /* Platform device */ +static struct virtio_uml_platform_data * +virtio_uml_create_pdata(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct virtio_uml_platform_data *pdata; + int ret; + + if (!np) + return ERR_PTR(-EINVAL); + + pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return ERR_PTR(-ENOMEM); + + INIT_WORK(&pdata->conn_broken_wk, vu_of_conn_broken); + pdata->pdev = pdev; + + ret = of_property_read_string(np, "socket-path", &pdata->socket_path); + if (ret) + return ERR_PTR(ret); + + ret = of_property_read_u32(np, "virtio-device-id", + &pdata->virtio_device_id); + if (ret) + return ERR_PTR(ret); + + return pdata; +} + static int virtio_uml_probe(struct platform_device *pdev) { struct virtio_uml_platform_data *pdata = pdev->dev.platform_data; struct virtio_uml_device *vu_dev; int rc; - if (!pdata) - return -EINVAL; + if (!pdata) { + pdata = virtio_uml_create_pdata(pdev); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + } vu_dev = kzalloc(sizeof(*vu_dev), GFP_KERNEL); if (!vu_dev) return -ENOMEM; + vu_dev->pdata = pdata; vu_dev->vdev.dev.parent = &pdev->dev; vu_dev->vdev.dev.release = virtio_uml_release_dev; vu_dev->vdev.config = &virtio_uml_config_ops; diff --git a/arch/um/include/asm/processor-generic.h b/arch/um/include/asm/processor-generic.h index b5cf0ed116..6a4fe8b4e6 100644 --- a/arch/um/include/asm/processor-generic.h +++ b/arch/um/include/asm/processor-generic.h @@ -11,7 +11,6 @@ struct pt_regs; struct task_struct; #include -#include #include #include @@ -105,7 +104,8 @@ extern struct cpuinfo_um boot_cpu_data; #define current_cpu_data boot_cpu_data #define cache_line_size() (boot_cpu_data.cache_alignment) +extern unsigned long get_thread_reg(int reg, jmp_buf *buf); #define KSTK_REG(tsk, reg) get_thread_reg(reg, &tsk->thread.switch_buf) -extern unsigned long get_wchan(struct task_struct *p); +extern unsigned long __get_wchan(struct task_struct *p); #endif diff --git a/arch/um/include/asm/syscall-generic.h b/arch/um/include/asm/syscall-generic.h index 2984feb9d5..172b74143c 100644 --- a/arch/um/include/asm/syscall-generic.h +++ b/arch/um/include/asm/syscall-generic.h @@ -62,20 +62,6 @@ static inline void syscall_get_arguments(struct task_struct *task, *args = UPT_SYSCALL_ARG6(r); } -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - const unsigned long *args) -{ - struct uml_pt_regs *r = ®s->regs; - - UPT_SYSCALL_ARG1(r) = *args++; - UPT_SYSCALL_ARG2(r) = *args++; - UPT_SYSCALL_ARG3(r) = *args++; - UPT_SYSCALL_ARG4(r) = *args++; - UPT_SYSCALL_ARG5(r) = *args++; - UPT_SYSCALL_ARG6(r) = *args; -} - /* See arch/x86/um/asm/syscall.h for syscall_get_arch() definition. */ #endif /* __UM_SYSCALL_GENERIC_H */ diff --git a/arch/um/include/asm/thread_info.h b/arch/um/include/asm/thread_info.h index 3b1cb8b3b1..1395cbd7e3 100644 --- a/arch/um/include/asm/thread_info.h +++ b/arch/um/include/asm/thread_info.h @@ -22,9 +22,6 @@ struct thread_info { __u32 cpu; /* current CPU */ int preempt_count; /* 0 => preemptable, <0 => BUG */ - mm_segment_t addr_limit; /* thread address space: - 0-0xBFFFFFFF for user - 0-0xFFFFFFFF for kernel */ struct thread_info *real_thread; /* Points to non-IRQ stack */ unsigned long aux_fp_regs[FP_SIZE]; /* auxiliary fp_regs to save/restore them out-of-band */ @@ -36,7 +33,6 @@ struct thread_info { .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ .real_thread = NULL, \ } diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h index 191ef36dd5..17d18cfd82 100644 --- a/arch/um/include/asm/uaccess.h +++ b/arch/um/include/asm/uaccess.h @@ -8,6 +8,7 @@ #define __UM_UACCESS_H #include +#include #define __under_task_size(addr, size) \ (((unsigned long) (addr) < TASK_SIZE) && \ @@ -39,8 +40,24 @@ static inline int __access_ok(unsigned long addr, unsigned long size) { return __addr_range_nowrap(addr, size) && (__under_task_size(addr, size) || - __access_ok_vsyscall(addr, size) || - uaccess_kernel()); + __access_ok_vsyscall(addr, size)); } +/* no pagefaults for kernel addresses in um */ +#define HAVE_GET_KERNEL_NOFAULT 1 + +#define __get_kernel_nofault(dst, src, type, err_label) \ +do { \ + *((type *)dst) = get_unaligned((type *)(src)); \ + if (0) /* make sure the label looks used to the compiler */ \ + goto err_label; \ +} while (0) + +#define __put_kernel_nofault(dst, src, type, err_label) \ +do { \ + put_unaligned(*((type *)src), (type *)(dst)); \ + if (0) /* make sure the label looks used to the compiler */ \ + goto err_label; \ +} while (0) + #endif diff --git a/arch/um/include/shared/common-offsets.h b/arch/um/include/shared/common-offsets.h index edc90ab737..96195483fb 100644 --- a/arch/um/include/shared/common-offsets.h +++ b/arch/um/include/shared/common-offsets.h @@ -9,32 +9,17 @@ DEFINE(UM_KERN_PAGE_MASK, PAGE_MASK); DEFINE(UM_KERN_PAGE_SHIFT, PAGE_SHIFT); DEFINE(UM_NSEC_PER_SEC, NSEC_PER_SEC); -DEFINE(UM_ELF_CLASS, ELF_CLASS); -DEFINE(UM_ELFCLASS32, ELFCLASS32); -DEFINE(UM_ELFCLASS64, ELFCLASS64); - -DEFINE(UM_NR_CPUS, NR_CPUS); - DEFINE(UM_GFP_KERNEL, GFP_KERNEL); DEFINE(UM_GFP_ATOMIC, GFP_ATOMIC); -/* For crypto assembler code. */ -DEFINE(crypto_tfm_ctx_offset, offsetof(struct crypto_tfm, __crt_ctx)); - DEFINE(UM_THREAD_SIZE, THREAD_SIZE); -DEFINE(UM_HZ, HZ); - -DEFINE(UM_USEC_PER_SEC, USEC_PER_SEC); DEFINE(UM_NSEC_PER_SEC, NSEC_PER_SEC); DEFINE(UM_NSEC_PER_USEC, NSEC_PER_USEC); #ifdef CONFIG_PRINTK DEFINE(UML_CONFIG_PRINTK, CONFIG_PRINTK); #endif -#ifdef CONFIG_NO_HZ_COMMON -DEFINE(UML_CONFIG_NO_HZ_COMMON, CONFIG_NO_HZ_COMMON); -#endif #ifdef CONFIG_UML_X86 DEFINE(UML_CONFIG_UML_X86, CONFIG_UML_X86); #endif diff --git a/arch/um/include/shared/irq_user.h b/arch/um/include/shared/irq_user.h index 86a8a573b6..da0f6eea30 100644 --- a/arch/um/include/shared/irq_user.h +++ b/arch/um/include/shared/irq_user.h @@ -20,6 +20,5 @@ void sigio_run_timetravel_handlers(void); extern void free_irq_by_fd(int fd); extern void deactivate_fd(int fd, int irqnum); extern int deactivate_all_fds(void); -extern int activate_ipi(int fd, int pid); #endif diff --git a/arch/um/include/shared/kern_util.h b/arch/um/include/shared/kern_util.h index a2cfd42608..d8b8b4f07e 100644 --- a/arch/um/include/shared/kern_util.h +++ b/arch/um/include/shared/kern_util.h @@ -53,13 +53,11 @@ extern void do_uml_exitcalls(void); extern int __cant_sleep(void); extern int get_current_pid(void); extern int copy_from_user_proc(void *to, void *from, int size); -extern int cpu(void); extern char *uml_strdup(const char *string); extern unsigned long to_irq_stack(unsigned long *mask_out); extern unsigned long from_irq_stack(int nested); -extern void syscall_trace(struct uml_pt_regs *regs, int entryexit); extern int singlestepping(void *t); extern void segv_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs); diff --git a/arch/um/include/shared/mem_user.h b/arch/um/include/shared/mem_user.h index cb84414e3e..11a723a585 100644 --- a/arch/um/include/shared/mem_user.h +++ b/arch/um/include/shared/mem_user.h @@ -46,16 +46,11 @@ extern int iomem_size; #define ROUND_4M(n) ((((unsigned long) (n)) + (1 << 22)) & ~((1 << 22) - 1)) -extern int init_mem_user(void); -extern void setup_memory(void *entry); extern unsigned long find_iomem(char *driver, unsigned long *len_out); extern void mem_total_pages(unsigned long physmem, unsigned long iomem, unsigned long highmem); -extern unsigned long get_vm(unsigned long len); extern void setup_physmem(unsigned long start, unsigned long usable, unsigned long len, unsigned long long highmem); -extern void add_iomem(char *name, int fd, unsigned long size); -extern unsigned long phys_offset(unsigned long phys); extern void map_memory(unsigned long virt, unsigned long phys, unsigned long len, int r, int w, int x); diff --git a/arch/um/include/shared/net_kern.h b/arch/um/include/shared/net_kern.h index a87be13c5b..441a8a3093 100644 --- a/arch/um/include/shared/net_kern.h +++ b/arch/um/include/shared/net_kern.h @@ -59,8 +59,6 @@ struct transport { const int setup_size; }; -extern struct net_device *ether_init(int); -extern unsigned short ether_protocol(struct sk_buff *); extern int tap_setup_common(char *str, char *type, char **dev_name, char **mac_out, char **gate_addr); extern void register_transport(struct transport *new); diff --git a/arch/um/include/shared/net_user.h b/arch/um/include/shared/net_user.h index 1b0531769a..ba92a4d935 100644 --- a/arch/um/include/shared/net_user.h +++ b/arch/um/include/shared/net_user.h @@ -24,7 +24,6 @@ struct net_user_info { int mtu; }; -extern void ether_user_init(void *data, void *dev); extern void iter_addresses(void *d, void (*cb)(unsigned char *, unsigned char *, void *), void *arg); diff --git a/arch/um/include/shared/os.h b/arch/um/include/shared/os.h index 03ffbdddcc..00214059d9 100644 --- a/arch/um/include/shared/os.h +++ b/arch/um/include/shared/os.h @@ -159,20 +159,11 @@ extern int os_create_unix_socket(const char *file, int len, int close_on_exec); extern int os_shutdown_socket(int fd, int r, int w); extern void os_close_file(int fd); extern int os_rcv_fd(int fd, int *helper_pid_out); -extern int create_unix_socket(char *file, int len, int close_on_exec); extern int os_connect_socket(const char *name); extern int os_file_type(char *file); extern int os_file_mode(const char *file, struct openflags *mode_out); extern int os_lock_file(int fd, int excl); extern void os_flush_stdout(void); -extern int os_stat_filesystem(char *path, long *bsize_out, - long long *blocks_out, long long *bfree_out, - long long *bavail_out, long long *files_out, - long long *ffree_out, void *fsid_out, - int fsid_size, long *namelen_out, - long *spare_out); -extern int os_change_dir(char *dir); -extern int os_fchange_dir(int fd); extern unsigned os_major(unsigned long long dev); extern unsigned os_minor(unsigned long long dev); extern unsigned long long os_makedev(unsigned major, unsigned minor); @@ -232,7 +223,6 @@ extern char *get_umid(void); /* signal.c */ extern void timer_set_signal_handler(void); extern void set_sigstack(void *sig_stack, int size); -extern void remove_sigstack(void); extern void set_handler(int sig); extern void send_sigio_to_self(void); extern int change_sig(int signal, int on); @@ -266,7 +256,6 @@ extern int os_timer_create(void); extern int os_timer_set_interval(unsigned long long nsecs); extern int os_timer_one_shot(unsigned long long nsecs); extern void os_timer_disable(void); -extern void uml_idle_timer(void); extern long long os_persistent_clock_emulation(void); extern long long os_nsecs(void); @@ -290,8 +279,6 @@ extern int is_skas_winch(int pid, int fd, void *data); extern int start_userspace(unsigned long stub_stack); extern int copy_context_skas0(unsigned long stack, int pid); extern void userspace(struct uml_pt_regs *regs, unsigned long *aux_fp_regs); -extern int map_stub_pages(int fd, unsigned long code, unsigned long data, - unsigned long stack); extern void new_thread(void *stack, jmp_buf *buf, void (*handler)(void)); extern void switch_threads(jmp_buf *me, jmp_buf *you); extern int start_idle_thread(void *stack, jmp_buf *switch_buf); diff --git a/arch/um/include/shared/registers.h b/arch/um/include/shared/registers.h index fbb709a222..2f9c3ce5b4 100644 --- a/arch/um/include/shared/registers.h +++ b/arch/um/include/shared/registers.h @@ -7,7 +7,6 @@ #define __REGISTERS_H #include -#include extern int save_i387_registers(int pid, unsigned long *fp_regs); extern int restore_i387_registers(int pid, unsigned long *fp_regs); @@ -19,7 +18,6 @@ extern int save_registers(int pid, struct uml_pt_regs *regs); extern int restore_pid_registers(int pid, struct uml_pt_regs *regs); extern int init_pid_registers(int pid); extern void get_safe_registers(unsigned long *regs, unsigned long *fp_regs); -extern unsigned long get_thread_reg(int reg, jmp_buf *buf); extern int get_fp_registers(int pid, unsigned long *regs); extern int put_fp_registers(int pid, unsigned long *regs); diff --git a/arch/um/include/shared/sigio.h b/arch/um/include/shared/sigio.h index 8fe8f57c05..e60c8b2278 100644 --- a/arch/um/include/shared/sigio.h +++ b/arch/um/include/shared/sigio.h @@ -7,7 +7,6 @@ #define __SIGIO_H__ extern int write_sigio_irq(int fd); -extern int register_sigio_fd(int fd); extern void sigio_lock(void); extern void sigio_unlock(void); diff --git a/arch/um/kernel/Makefile b/arch/um/kernel/Makefile index 1d18e4e469..1c2d4b29a3 100644 --- a/arch/um/kernel/Makefile +++ b/arch/um/kernel/Makefile @@ -16,11 +16,13 @@ extra-y := vmlinux.lds obj-y = config.o exec.o exitcode.o irq.o ksyms.o mem.o \ physmem.o process.o ptrace.o reboot.o sigio.o \ - signal.o syscall.o sysrq.o time.o tlb.o trap.o \ + signal.o sysrq.o time.o tlb.o trap.o \ um_arch.o umid.o maccess.o kmsg_dump.o capflags.o skas/ +obj-y += load_file.o obj-$(CONFIG_BLK_DEV_INITRD) += initrd.o obj-$(CONFIG_GPROF) += gprof_syms.o +obj-$(CONFIG_OF) += dtb.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_GENERIC_PCI_IOMAP) += ioport.o diff --git a/arch/um/kernel/exec.c b/arch/um/kernel/exec.c index 4d84981003..c85e40c727 100644 --- a/arch/um/kernel/exec.c +++ b/arch/um/kernel/exec.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/arch/um/kernel/initrd.c b/arch/um/kernel/initrd.c index c1981ffb71..47b8cb1a11 100644 --- a/arch/um/kernel/initrd.c +++ b/arch/um/kernel/initrd.c @@ -10,37 +10,21 @@ #include #include +#include "um_arch.h" + /* Changed by uml_initrd_setup, which is a setup */ static char *initrd __initdata = NULL; -static int load_initrd(char *filename, void *buf, int size); int __init read_initrd(void) { + unsigned long long size; void *area; - long long size; - int err; - if (initrd == NULL) + if (!initrd) return 0; - err = os_file_size(initrd, &size); - if (err) - return 0; - - /* - * This is necessary because alloc_bootmem craps out if you - * ask for no memory. - */ - if (size == 0) { - printk(KERN_ERR "\"%s\" is a zero-size initrd\n", initrd); - return 0; - } - - area = memblock_alloc(size, SMP_CACHE_BYTES); + area = uml_load_file(initrd, &size); if (!area) - panic("%s: Failed to allocate %llu bytes\n", __func__, size); - - if (load_initrd(initrd, area, size) == -1) return 0; initrd_start = (unsigned long) area; @@ -59,25 +43,3 @@ __uml_setup("initrd=", uml_initrd_setup, " This is used to boot UML from an initrd image. The argument is the\n" " name of the file containing the image.\n\n" ); - -static int load_initrd(char *filename, void *buf, int size) -{ - int fd, n; - - fd = os_open_file(filename, of_read(OPENFLAGS()), 0); - if (fd < 0) { - printk(KERN_ERR "Opening '%s' failed - err = %d\n", filename, - -fd); - return -1; - } - n = os_read_file(fd, buf, size); - if (n != size) { - printk(KERN_ERR "Read of %d bytes from '%s' failed, " - "err = %d\n", size, - filename, -n); - return -1; - } - - os_close_file(fd); - return 0; -} diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c index 8e636ce029..15295c3237 100644 --- a/arch/um/kernel/mem.c +++ b/arch/um/kernel/mem.c @@ -47,7 +47,7 @@ void __init mem_init(void) */ brk_end = (unsigned long) UML_ROUND_UP(sbrk(0)); map_memory(brk_end, __pa(brk_end), uml_reserved - brk_end, 1, 1, 0); - memblock_free(__pa(brk_end), uml_reserved - brk_end); + memblock_free((void *)brk_end, uml_reserved - brk_end); uml_reserved = brk_end; /* this will put all low memory onto the freelists */ @@ -85,8 +85,7 @@ static void __init one_md_table_init(pud_t *pud) __func__, PAGE_SIZE, PAGE_SIZE); set_pud(pud, __pud(_KERNPG_TABLE + (unsigned long) __pa(pmd_table))); - if (pmd_table != pmd_offset(pud, 0)) - BUG(); + BUG_ON(pmd_table != pmd_offset(pud, 0)); #endif } diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index 457a38db36..4a420778ed 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -31,6 +31,7 @@ #include #include #include +#include #include /* @@ -263,11 +264,6 @@ int clear_user_proc(void __user *buf, int size) return clear_user(buf, size); } -int cpu(void) -{ - return current_thread_info()->cpu; -} - static atomic_t using_sysemu = ATOMIC_INIT(0); int sysemu_supported; @@ -364,14 +360,11 @@ unsigned long arch_align_stack(unsigned long sp) } #endif -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { unsigned long stack_page, sp, ip; bool seen_sched = 0; - if ((p == NULL) || (p == current) || task_is_running(p)) - return 0; - stack_page = (unsigned long) task_stack_page(p); /* Bail if the process has no kernel stack for some reason */ if (stack_page == 0) diff --git a/arch/um/kernel/skas/uaccess.c b/arch/um/kernel/skas/uaccess.c index a509be9110..aaee96f071 100644 --- a/arch/um/kernel/skas/uaccess.c +++ b/arch/um/kernel/skas/uaccess.c @@ -146,11 +146,6 @@ static int copy_chunk_from_user(unsigned long from, int len, void *arg) unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) { - if (uaccess_kernel()) { - memcpy(to, (__force void*)from, n); - return 0; - } - return buffer_op((unsigned long) from, n, 0, copy_chunk_from_user, &to); } EXPORT_SYMBOL(raw_copy_from_user); @@ -166,11 +161,6 @@ static int copy_chunk_to_user(unsigned long to, int len, void *arg) unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - if (uaccess_kernel()) { - memcpy((__force void *) to, from, n); - return 0; - } - return buffer_op((unsigned long) to, n, 1, copy_chunk_to_user, &from); } EXPORT_SYMBOL(raw_copy_to_user); @@ -196,12 +186,6 @@ long strncpy_from_user(char *dst, const char __user *src, long count) if (!access_ok(src, 1)) return -EFAULT; - - if (uaccess_kernel()) { - strncpy(dst, (__force void *) src, count); - return strnlen(dst, count); - } - n = buffer_op((unsigned long) src, count, 0, strncpy_chunk_from_user, &ptr); if (n != 0) @@ -218,11 +202,6 @@ static int clear_chunk(unsigned long addr, int len, void *unused) unsigned long __clear_user(void __user *mem, unsigned long len) { - if (uaccess_kernel()) { - memset((__force void*)mem, 0, len); - return 0; - } - return buffer_op((unsigned long) mem, len, 1, clear_chunk, NULL); } EXPORT_SYMBOL(__clear_user); @@ -245,10 +224,6 @@ long strnlen_user(const char __user *str, long len) if (!access_ok(str, 1)) return -EFAULT; - - if (uaccess_kernel()) - return strnlen((__force char*)str, len) + 1; - n = buffer_op((unsigned long) str, len, 0, strnlen_chunk, &count); if (n == 0) return count + 1; @@ -348,7 +323,6 @@ EXPORT_SYMBOL(arch_futex_atomic_op_inuser); * 0 - On success * -EFAULT - User access resulted in a page fault * -EAGAIN - Atomic operation was unable to complete due to contention - * -ENOSYS - Function not implemented (only if !HAVE_FUTEX_CMPXCHG) */ int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index c32efb09db..d1d5d0be03 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -87,12 +87,10 @@ int handle_page_fault(unsigned long address, unsigned long ip, } BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_RETRY) { - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - goto retry; - } + goto retry; } pmd = pmd_off(mm, address); @@ -127,7 +125,6 @@ int handle_page_fault(unsigned long address, unsigned long ip, pagefault_out_of_memory(); return 0; } -EXPORT_SYMBOL(handle_page_fault); static void show_segv_info(struct uml_pt_regs *regs) { diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index a149a5e9a1..abceeabe29 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -29,6 +29,8 @@ #include #include +#include "um_arch.h" + #define DEFAULT_COMMAND_LINE_ROOT "root=98:0" #define DEFAULT_COMMAND_LINE_CONSOLE "console=tty" @@ -407,6 +409,7 @@ void __init setup_arch(char **cmdline_p) stack_protections((unsigned long) &init_thread_info); setup_physmem(uml_physmem, uml_reserved, physmem_size, highmem); mem_total_pages(physmem_size, iomem_size, highmem); + uml_dtb_init(); read_initrd(); paging_init(); @@ -421,6 +424,10 @@ void __init check_bugs(void) os_check_bugs(); } +void apply_retpolines(s32 *start, s32 *end) +{ +} + void apply_alternatives(struct alt_instr *start, struct alt_instr *end) { } diff --git a/arch/um/os-Linux/sigio.c b/arch/um/os-Linux/sigio.c index 9e71794839..37d60e72cf 100644 --- a/arch/um/os-Linux/sigio.c +++ b/arch/um/os-Linux/sigio.c @@ -3,6 +3,7 @@ * Copyright (C) 2002 - 2008 Jeff Dike (jdike@{addtoit,linux.intel}.com) */ +#include #include #include #include @@ -50,7 +51,7 @@ static struct pollfds all_sigio_fds; static int write_sigio_thread(void *unused) { - struct pollfds *fds, tmp; + struct pollfds *fds; struct pollfd *p; int i, n, respond_fd; char c; @@ -77,9 +78,7 @@ static int write_sigio_thread(void *unused) "write_sigio_thread : " "read on socket failed, " "err = %d\n", errno); - tmp = current_poll; - current_poll = next_poll; - next_poll = tmp; + swap(current_poll, next_poll); respond_fd = sigio_private[1]; } else { diff --git a/arch/x86/Kbuild b/arch/x86/Kbuild index 30dec01975..f384cb1a4f 100644 --- a/arch/x86/Kbuild +++ b/arch/x86/Kbuild @@ -25,3 +25,6 @@ obj-y += platform/ obj-y += net/ obj-$(CONFIG_KEXEC_FILE) += purgatory/ + +# for cleaning +subdir- += boot tools diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 1f96809606..9f5bd41bf6 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -61,8 +61,9 @@ config X86 select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI select ARCH_32BIT_OFF_T if X86_32 select ARCH_CLOCKSOURCE_INIT + select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_HUGEPAGE_MIGRATION if X86_64 && HUGETLB_PAGE && MIGRATION - select ARCH_ENABLE_MEMORY_HOTPLUG if X86_64 || (X86_32 && HIGHMEM) + select ARCH_ENABLE_MEMORY_HOTPLUG if X86_64 select ARCH_ENABLE_MEMORY_HOTREMOVE if MEMORY_HOTPLUG select ARCH_ENABLE_SPLIT_PMD_PTLOCK if (PGTABLE_LEVELS > 2) && (X86_64 || X86_PAE) select ARCH_ENABLE_THP_MIGRATION if X86_64 && TRANSPARENT_HUGEPAGE @@ -77,7 +78,7 @@ config X86 select ARCH_HAS_FILTER_PGPROT select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_GCOV_PROFILE_ALL - select ARCH_HAS_KCOV if X86_64 && STACK_VALIDATION + select ARCH_HAS_KCOV if X86_64 select ARCH_HAS_MEM_ENCRYPT select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE @@ -103,6 +104,7 @@ config X86 select ARCH_SUPPORTS_ACPI select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_DEBUG_PAGEALLOC + select ARCH_SUPPORTS_PAGE_TABLE_CHECK if X86_64 select ARCH_SUPPORTS_NUMA_BALANCING if X86_64 select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096 select ARCH_SUPPORTS_LTO_CLANG @@ -125,6 +127,7 @@ config X86 select CLOCKSOURCE_VALIDATE_LAST_CYCLE select CLOCKSOURCE_WATCHDOG select DCACHE_WORD_ACCESS + select DYNAMIC_SIGFRAME select EDAC_ATOMIC_SCRUB select EDAC_SUPPORT select GENERIC_CLOCKEVENTS_BROADCAST if X86_64 || (X86_32 && X86_LOCAL_APIC) @@ -134,7 +137,6 @@ config X86 select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP select GENERIC_ENTRY - select GENERIC_FIND_FIRST_BIT select GENERIC_IOMAP select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select GENERIC_IRQ_MATRIX_ALLOCATOR if X86_LOCAL_APIC @@ -184,12 +186,15 @@ config X86 select HAVE_CONTEXT_TRACKING_OFFSTACK if HAVE_CONTEXT_TRACKING select HAVE_C_RECORDMCOUNT select HAVE_OBJTOOL_MCOUNT if STACK_VALIDATION + select HAVE_BUILDTIME_MCOUNT_SORT select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_DYNAMIC_FTRACE_WITH_ARGS if X86_64 select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + select HAVE_SAMPLE_FTRACE_DIRECT if X86_64 + select HAVE_SAMPLE_FTRACE_DIRECT_MULTI if X86_64 select HAVE_EBPF_JIT select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EISA @@ -197,7 +202,7 @@ config X86 select HAVE_FAST_GUP select HAVE_FENTRY if X86_64 || DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD - select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_GRAPH_TRACER if X86_32 || (X86_64 && DYNAMIC_FTRACE) select HAVE_FUNCTION_TRACER select HAVE_GCC_PLUGINS select HAVE_HW_BREAKPOINT @@ -235,6 +240,7 @@ config X86 select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION select HAVE_FUNCTION_ARG_ACCESS_API + select HAVE_SETUP_PER_CPU_AREA select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 @@ -248,6 +254,8 @@ config X86 select HAVE_GENERIC_VDSO select HOTPLUG_SMT if SMP select IRQ_FORCED_THREADING + select NEED_PER_CPU_EMBED_FIRST_CHUNK + select NEED_PER_CPU_PAGE_FIRST_CHUNK select NEED_SG_DMA_LENGTH select PCI_DOMAINS if PCI select PCI_LOCKLESS_CONFIG if PCI @@ -265,6 +273,7 @@ config X86 select HAVE_ARCH_KCSAN if X86_64 select X86_FEATURE_NAMES if PROC_FS select PROC_PID_ARCH_STATUS if PROC_FS + select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI config INSTRUCTION_DECODER @@ -327,15 +336,6 @@ config ARCH_HAS_CPU_RELAX config ARCH_HAS_FILTER_PGPROT def_bool y -config HAVE_SETUP_PER_CPU_AREA - def_bool y - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y - -config NEED_PER_CPU_PAGE_FIRST_CHUNK - def_bool y - config ARCH_HIBERNATION_POSSIBLE def_bool y @@ -468,6 +468,18 @@ config RETPOLINE branches. Requires a compiler with -mindirect-branch=thunk-extern support for full protection. The kernel may run slower. +config CC_HAS_SLS + def_bool $(cc-option,-mharden-sls=all) + +config SLS + bool "Mitigate Straight-Line-Speculation" + depends on CC_HAS_SLS && X86_64 + default n + help + Compile the kernel with straight-line-speculation options to guard + against straight line speculation. The kernel image might be slightly + larger. + config X86_CPU_RESCTRL bool "x86 CPU resource control support" depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD) @@ -610,9 +622,7 @@ config X86_INTEL_MID depends on X86_IO_APIC select I2C select DW_APB_TIMER - select APB_TIMER select INTEL_SCU_PCI - select MFD_INTEL_MSIC help Select to build a kernel capable of supporting Intel MID (Mobile Internet Device) platform systems which do not have the PCI legacy @@ -1001,6 +1011,17 @@ config NR_CPUS This is purely to save memory: each supported CPU adds about 8KB to the kernel image. +config SCHED_CLUSTER + bool "Cluster scheduler support" + depends on SMP + default y + help + Cluster scheduler support improves the CPU scheduler's decision + making when dealing with machines that have clusters of CPUs. + Cluster usually means a couple of CPUs which are placed closely + by sharing mid-level caches, last-level cache tags or internal + busses. + config SCHED_SMT def_bool y if SMP @@ -1510,16 +1531,20 @@ config X86_CPA_STATISTICS helps to determine the effectiveness of preserving large and huge page mappings when mapping protections are changed. +config X86_MEM_ENCRYPT + select ARCH_HAS_FORCE_DMA_UNENCRYPTED + select DYNAMIC_PHYSICAL_MASK + select ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS + def_bool n + config AMD_MEM_ENCRYPT bool "AMD Secure Memory Encryption (SME) support" depends on X86_64 && CPU_SUP_AMD select DMA_COHERENT_POOL - select DYNAMIC_PHYSICAL_MASK select ARCH_USE_MEMREMAP_PROT - select ARCH_HAS_FORCE_DMA_UNENCRYPTED select INSTRUCTION_DECODER - select ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS select ARCH_HAS_CC_PLATFORM + select X86_MEM_ENCRYPT help Say yes to enable support for the encryption of system memory. This requires an AMD processor that supports Secure Memory @@ -1544,6 +1569,7 @@ config NUMA depends on SMP depends on X86_64 || (X86_32 && HIGHMEM64G && X86_BIGSMP) default y if X86_BIGSMP + select USE_PERCPU_NUMA_NODE_ID help Enable NUMA (Non-Uniform Memory Access) support. @@ -1616,7 +1642,7 @@ config ARCH_SELECT_MEMORY_MODEL config ARCH_MEMORY_PROBE bool "Enable sysfs memory/probe interface" - depends on X86_64 && MEMORY_HOTPLUG + depends on MEMORY_HOTPLUG help This option enables a sysfs memory/probe interface for testing. See Documentation/admin-guide/mm/memory-hotplug.rst for more information. @@ -1904,6 +1930,7 @@ config X86_SGX select SRCU select MMU_NOTIFIER select NUMA_KEEP_MEMINFO if NUMA + select XARRAY_MULTI help Intel(R) Software Guard eXtensions (SGX) is a set of CPU instructions that can be used by applications to set aside private regions of code @@ -1933,7 +1960,7 @@ config EFI config EFI_STUB bool "EFI stub support" - depends on EFI && !X86_USE_3DNOW + depends on EFI depends on $(cc-option,-mabi=ms) || X86_32 select RELOCATABLE help @@ -2391,21 +2418,33 @@ config MODIFY_LDT_SYSCALL Saying 'N' here may make sense for embedded or server kernels. +config STRICT_SIGALTSTACK_SIZE + bool "Enforce strict size checking for sigaltstack" + depends on DYNAMIC_SIGFRAME + help + For historical reasons MINSIGSTKSZ is a constant which became + already too small with AVX512 support. Add a mechanism to + enforce strict checking of the sigaltstack size against the + real size of the FPU frame. This option enables the check + by default. It can also be controlled via the kernel command + line option 'strict_sas_size' independent of this config + switch. Enabling it might break existing applications which + allocate a too small sigaltstack but 'work' because they + never get a signal delivered. + + Say 'N' unless you want to really enforce this check. + source "kernel/livepatch/Kconfig" endmenu config ARCH_HAS_ADD_PAGES def_bool y - depends on X86_64 && ARCH_ENABLE_MEMORY_HOTPLUG + depends on ARCH_ENABLE_MEMORY_HOTPLUG config ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE def_bool y -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - menu "Power management and ACPI options" config ARCH_HIBERNATION_HEADER diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index 814fe0d349..542377cd41 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -342,10 +342,6 @@ config X86_USE_PPRO_CHECKSUM def_bool y depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM -config X86_USE_3DNOW - def_bool y - depends on (MCYRIXIII || MK7 || MGEODE_LX) && !UML - # # P6_NOPs are a relatively minor optimization that require a family >= # 6 processor, except that it is broken on certain VIA chips. @@ -508,3 +504,16 @@ config CPU_SUP_ZHAOXIN CPU might render the kernel unbootable. If unsure, say N. + +config CPU_SUP_VORTEX_32 + default y + bool "Support Vortex processors" if PROCESSOR_SELECT + depends on X86_32 + help + This enables detection, tunings and quirks for Vortex processors + + You need this enabled if you want your kernel to run on a + Vortex CPU. Disabling this option on other types of CPUs + makes the kernel a tiny bit smaller. + + If unsure, say N. diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 7488cfbbd2..e84cdd409b 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -12,6 +12,18 @@ else KBUILD_DEFCONFIG := $(ARCH)_defconfig endif +ifdef CONFIG_CC_IS_GCC +RETPOLINE_CFLAGS := $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register) +RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch-cs-prefix) +RETPOLINE_VDSO_CFLAGS := $(call cc-option,-mindirect-branch=thunk-inline -mindirect-branch-register) +endif +ifdef CONFIG_CC_IS_CLANG +RETPOLINE_CFLAGS := -mretpoline-external-thunk +RETPOLINE_VDSO_CFLAGS := -mretpoline +endif +export RETPOLINE_CFLAGS +export RETPOLINE_VDSO_CFLAGS + # For gcc stack alignment is specified with -mpreferred-stack-boundary, # clang has the option -mstack-alignment for that purpose. ifneq ($(call cc-option, -mpreferred-stack-boundary=4),) @@ -179,6 +191,10 @@ ifdef CONFIG_RETPOLINE endif endif +ifdef CONFIG_SLS + KBUILD_CFLAGS += -mharden-sls=all +endif + KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE) ifdef CONFIG_LTO_CLANG @@ -283,8 +299,6 @@ endif archclean: $(Q)rm -rf $(objtree)/arch/i386 $(Q)rm -rf $(objtree)/arch/x86_64 - $(Q)$(MAKE) $(clean)=$(boot) - $(Q)$(MAKE) $(clean)=arch/x86/tools define archhelp echo '* bzImage - Compressed kernel image (arch/x86/boot/bzImage)' @@ -299,7 +313,7 @@ define archhelp echo ' isoimage - Create a boot CD-ROM image (arch/x86/boot/image.iso)' echo ' bzdisk/fdimage*/hdimage/isoimage also accept:' echo ' FDARGS="..." arguments for the booted kernel' - echo ' FDINITRD=file initrd for the booted kernel' + echo ' FDINITRD=file initrd for the booted kernel' echo '' echo ' kvm_guest.config - Enable Kconfig items for running this kernel as a KVM guest' echo ' xen.config - Enable Kconfig items for running this kernel as a Xen guest' diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index e118136460..6115274fe1 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -126,17 +126,17 @@ vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += $(obj)/vmlinux.relocs $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE $(call if_changed,gzip) $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE - $(call if_changed,bzip2) + $(call if_changed,bzip2_with_size) $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzma) + $(call if_changed,lzma_with_size) $(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,xzkern) + $(call if_changed,xzkern_with_size) $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzo) + $(call if_changed,lzo_with_size) $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lz4) + $(call if_changed,lz4_with_size) $(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE - $(call if_changed,zstd22) + $(call if_changed,zstd22_with_size) suffix-$(CONFIG_KERNEL_GZIP) := gz suffix-$(CONFIG_KERNEL_BZIP2) := bz2 diff --git a/arch/x86/boot/compressed/efi_thunk_64.S b/arch/x86/boot/compressed/efi_thunk_64.S index 8bb92e9f4e..67e7edcdfe 100644 --- a/arch/x86/boot/compressed/efi_thunk_64.S +++ b/arch/x86/boot/compressed/efi_thunk_64.S @@ -26,8 +26,6 @@ SYM_FUNC_START(__efi64_thunk) push %rbp push %rbx - leaq 1f(%rip), %rbp - movl %ds, %eax push %rax movl %es, %eax @@ -35,6 +33,11 @@ SYM_FUNC_START(__efi64_thunk) movl %ss, %eax push %rax + /* Copy args passed on stack */ + movq 0x30(%rsp), %rbp + movq 0x38(%rsp), %rbx + movq 0x40(%rsp), %rax + /* * Convert x86-64 ABI params to i386 ABI */ @@ -44,13 +47,18 @@ SYM_FUNC_START(__efi64_thunk) movl %ecx, 0x8(%rsp) movl %r8d, 0xc(%rsp) movl %r9d, 0x10(%rsp) + movl %ebp, 0x14(%rsp) + movl %ebx, 0x18(%rsp) + movl %eax, 0x1c(%rsp) - leaq 0x14(%rsp), %rbx + leaq 0x20(%rsp), %rbx sgdt (%rbx) addq $16, %rbx sidt (%rbx) + leaq 1f(%rip), %rbp + /* * Switch to IDT and GDT with 32-bit segments. This is the firmware GDT * and IDT that was installed when the kernel started executing. The @@ -93,7 +101,7 @@ SYM_FUNC_START(__efi64_thunk) pop %rbx pop %rbp - ret + RET SYM_FUNC_END(__efi64_thunk) .code32 diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index 67c3208b66..411b268bc0 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -32,10 +32,6 @@ #include #include -/* Macros used by the included decompressor code below. */ -#define STATIC -#include - #define _SETUP #include /* For COMMAND_LINE_SIZE */ #undef _SETUP diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index 743f13ea25..a4339cb2d2 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -28,6 +28,9 @@ /* Macros used by the included decompressor code below. */ #define STATIC static +/* Define an externally visible malloc()/free(). */ +#define MALLOC_VISIBLE +#include /* * Provide definitions of memzero and memmove as some of the decompressors will diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 3113925685..16ed360b66 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -14,6 +14,8 @@ #undef CONFIG_KASAN #undef CONFIG_KASAN_GENERIC +#define __NO_FORTIFY + /* cpu_feature_enabled() cannot be used this early */ #define USE_EARLY_PGTABLE_L5 @@ -44,6 +46,8 @@ extern char _head[], _end[]; /* misc.c */ extern memptr free_mem_ptr; extern memptr free_mem_end_ptr; +void *malloc(int size); +void free(void *where); extern struct boot_params *boot_params; void __putstr(const char *s); void __puthex(unsigned long value); diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index 2a78746f5a..a1733319a2 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "misc.h" #include #include #include diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 670e998fe9..28bcf04c02 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -122,7 +122,7 @@ static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, static bool early_setup_sev_es(void) { if (!sev_es_negotiate_protocol()) - sev_es_terminate(GHCB_SEV_ES_REASON_PROTOCOL_UNSUPPORTED); + sev_es_terminate(GHCB_SEV_ES_PROT_UNSUPPORTED); if (set_page_decrypted((unsigned long)&boot_ghcb_page)) return false; @@ -175,7 +175,7 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) enum es_result result; if (!boot_ghcb && !early_setup_sev_es()) - sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + sev_es_terminate(GHCB_SEV_ES_GEN_REQ); vc_ghcb_invalidate(boot_ghcb); result = vc_init_em_ctxt(&ctxt, regs, exit_code); @@ -202,5 +202,5 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) if (result == ES_OK) vc_finish_insn(&ctxt); else if (result != ES_RETRY) - sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + sev_es_terminate(GHCB_SEV_ES_GEN_REQ); } diff --git a/arch/x86/boot/genimage.sh b/arch/x86/boot/genimage.sh index 0673fdfc1a..c9299aeb73 100644 --- a/arch/x86/boot/genimage.sh +++ b/arch/x86/boot/genimage.sh @@ -120,12 +120,13 @@ efiarch() { } # Get the combined sizes in bytes of the files given, counting sparse -# files as full length, and padding each file to a 4K block size +# files as full length, and padding each file to cluster size +cluster=16384 filesizes() { local t=0 local s for s in $(ls -lnL "$@" 2>/dev/null | awk '/^-/{ print $5; }'); do - t=$((t + ((s+4095)/4096)*4096)) + t=$((t + ((s+cluster-1)/cluster)*cluster)) done echo $t } @@ -230,14 +231,14 @@ genhdimage() { ptype='-T 0xef' # EFI system partition, no GPT fi sizes=$(filesizes "$FBZIMAGE" "${FDINITRDS[@]}" "$efishell") - # Allow 1% + 1 MiB for filesystem and partition table overhead, - # syslinux, and config files + # Allow 1% + 2 MiB for filesystem and partition table overhead, + # syslinux, and config files; this is probably excessive... megs=$(((sizes + sizes/100 + 2*1024*1024 - 1)/(1024*1024))) $dd if=/dev/zero of="$FIMAGE" bs=$((1024*1024)) count=$megs 2>/dev/null - mpartition -I -c -s 32 -h 64 -t $megs $ptype -b 512 -a h: + mpartition -I -c -s 32 -h 64 $ptype -b 64 -a p: $dd if="$mbr" of="$FIMAGE" bs=440 count=1 conv=notrunc 2>/dev/null - mformat -v 'LINUX_BOOT' -s 32 -h 64 -t $megs h: - syslinux --offset $((512*512)) "$FIMAGE" + mformat -v 'LINUX_BOOT' -s 32 -h 64 -c $((cluster/512)) -t $megs h: + syslinux --offset $((64*512)) "$FIMAGE" do_mcopy h: } diff --git a/arch/x86/boot/mtools.conf.in b/arch/x86/boot/mtools.conf.in index 9e2662d013..174c605087 100644 --- a/arch/x86/boot/mtools.conf.in +++ b/arch/x86/boot/mtools.conf.in @@ -14,7 +14,8 @@ drive v: drive w: file="@OBJ@/fdimage" cylinders=80 heads=2 sectors=36 filter -# Hard disk +# Hard disk (h: for the filesystem, p: for format - old mtools bug?) drive h: + file="@OBJ@/hdimage" offset=32768 mformat_only +drive p: file="@OBJ@/hdimage" partition=1 mformat_only - diff --git a/arch/x86/boot/string.h b/arch/x86/boot/string.h index a232da487c..e5d2c6b8c2 100644 --- a/arch/x86/boot/string.h +++ b/arch/x86/boot/string.h @@ -8,8 +8,10 @@ #undef memcmp void *memcpy(void *dst, const void *src, size_t len); +void *memmove(void *dst, const void *src, size_t len); void *memset(void *dst, int c, size_t len); int memcmp(const void *s1, const void *s2, size_t len); +int bcmp(const void *s1, const void *s2, size_t len); /* Access builtin version by default. */ #define memcpy(d,s,l) __builtin_memcpy(d,s,l) @@ -25,6 +27,7 @@ extern size_t strnlen(const char *s, size_t maxlen); extern unsigned int atou(const char *s); extern unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base); +long simple_strtol(const char *cp, char **endp, unsigned int base); int kstrtoull(const char *s, unsigned int base, unsigned long long *res); int boot_kstrtoul(const char *s, unsigned int base, unsigned long *res); diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig index 99398cbdae..71124cf863 100644 --- a/arch/x86/configs/i386_defconfig +++ b/arch/x86/configs/i386_defconfig @@ -1,4 +1,3 @@ -# CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_AUDIT=y diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index d7298b104a..92b1169ec9 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -1,4 +1,3 @@ -# CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_AUDIT=y diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index f307c93fc9..c3af959648 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -62,7 +62,9 @@ obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o obj-$(CONFIG_CRYPTO_BLAKE2S_X86) += blake2s-x86_64.o -blake2s-x86_64-y := blake2s-core.o blake2s-glue.o +blake2s-x86_64-y := blake2s-shash.o +obj-$(if $(CONFIG_CRYPTO_BLAKE2S_X86),y) += libblake2s-x86_64.o +libblake2s-x86_64-y := blake2s-core.o blake2s-glue.o obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o diff --git a/arch/x86/crypto/blake2s-core.S b/arch/x86/crypto/blake2s-core.S index 2ca79974f8..b50b35ff1f 100644 --- a/arch/x86/crypto/blake2s-core.S +++ b/arch/x86/crypto/blake2s-core.S @@ -171,7 +171,7 @@ SYM_FUNC_START(blake2s_compress_ssse3) movdqu %xmm1,0x10(%rdi) movdqu %xmm14,0x20(%rdi) .Lendofloop: - ret + RET SYM_FUNC_END(blake2s_compress_ssse3) #ifdef CONFIG_AS_AVX512 @@ -251,6 +251,6 @@ SYM_FUNC_START(blake2s_compress_avx512) vmovdqu %xmm1,0x10(%rdi) vmovdqu %xmm4,0x20(%rdi) vzeroupper - retq + RET SYM_FUNC_END(blake2s_compress_avx512) #endif /* CONFIG_AS_AVX512 */ diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c index a40365ab30..69853c13e8 100644 --- a/arch/x86/crypto/blake2s-glue.c +++ b/arch/x86/crypto/blake2s-glue.c @@ -5,7 +5,6 @@ #include #include -#include #include #include @@ -28,9 +27,8 @@ asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3); static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512); -void blake2s_compress_arch(struct blake2s_state *state, - const u8 *block, size_t nblocks, - const u32 inc) +void blake2s_compress(struct blake2s_state *state, const u8 *block, + size_t nblocks, const u32 inc) { /* SIMD disables preemption, so relax after processing each page. */ BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8); @@ -56,49 +54,12 @@ void blake2s_compress_arch(struct blake2s_state *state, block += blocks * BLAKE2S_BLOCK_SIZE; } while (nblocks); } -EXPORT_SYMBOL(blake2s_compress_arch); - -static int crypto_blake2s_update_x86(struct shash_desc *desc, - const u8 *in, unsigned int inlen) -{ - return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch); -} - -static int crypto_blake2s_final_x86(struct shash_desc *desc, u8 *out) -{ - return crypto_blake2s_final(desc, out, blake2s_compress_arch); -} - -#define BLAKE2S_ALG(name, driver_name, digest_size) \ - { \ - .base.cra_name = name, \ - .base.cra_driver_name = driver_name, \ - .base.cra_priority = 200, \ - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ - .base.cra_module = THIS_MODULE, \ - .digestsize = digest_size, \ - .setkey = crypto_blake2s_setkey, \ - .init = crypto_blake2s_init, \ - .update = crypto_blake2s_update_x86, \ - .final = crypto_blake2s_final_x86, \ - .descsize = sizeof(struct blake2s_state), \ - } - -static struct shash_alg blake2s_algs[] = { - BLAKE2S_ALG("blake2s-128", "blake2s-128-x86", BLAKE2S_128_HASH_SIZE), - BLAKE2S_ALG("blake2s-160", "blake2s-160-x86", BLAKE2S_160_HASH_SIZE), - BLAKE2S_ALG("blake2s-224", "blake2s-224-x86", BLAKE2S_224_HASH_SIZE), - BLAKE2S_ALG("blake2s-256", "blake2s-256-x86", BLAKE2S_256_HASH_SIZE), -}; +EXPORT_SYMBOL(blake2s_compress); static int __init blake2s_mod_init(void) { - if (!boot_cpu_has(X86_FEATURE_SSSE3)) - return 0; - - static_branch_enable(&blake2s_use_ssse3); + if (boot_cpu_has(X86_FEATURE_SSSE3)) + static_branch_enable(&blake2s_use_ssse3); if (IS_ENABLED(CONFIG_AS_AVX512) && boot_cpu_has(X86_FEATURE_AVX) && @@ -109,26 +70,9 @@ static int __init blake2s_mod_init(void) XFEATURE_MASK_AVX512, NULL)) static_branch_enable(&blake2s_use_avx512); - return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? - crypto_register_shashes(blake2s_algs, - ARRAY_SIZE(blake2s_algs)) : 0; -} - -static void __exit blake2s_mod_exit(void) -{ - if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && boot_cpu_has(X86_FEATURE_SSSE3)) - crypto_unregister_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs)); + return 0; } module_init(blake2s_mod_init); -module_exit(blake2s_mod_exit); -MODULE_ALIAS_CRYPTO("blake2s-128"); -MODULE_ALIAS_CRYPTO("blake2s-128-x86"); -MODULE_ALIAS_CRYPTO("blake2s-160"); -MODULE_ALIAS_CRYPTO("blake2s-160-x86"); -MODULE_ALIAS_CRYPTO("blake2s-224"); -MODULE_ALIAS_CRYPTO("blake2s-224-x86"); -MODULE_ALIAS_CRYPTO("blake2s-256"); -MODULE_ALIAS_CRYPTO("blake2s-256-x86"); MODULE_LICENSE("GPL v2"); diff --git a/arch/x86/crypto/curve25519-x86_64.c b/arch/x86/crypto/curve25519-x86_64.c index 38caf61cd5..d55fa9e9b9 100644 --- a/arch/x86/crypto/curve25519-x86_64.c +++ b/arch/x86/crypto/curve25519-x86_64.c @@ -64,10 +64,9 @@ static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2) /* Return the carry bit in a register */ " adcx %%r11, %1;" - : "+&r" (f2), "=&r" (carry_r) - : "r" (out), "r" (f1) - : "%r8", "%r9", "%r10", "%r11", "memory", "cc" - ); + : "+&r"(f2), "=&r"(carry_r) + : "r"(out), "r"(f1) + : "%r8", "%r9", "%r10", "%r11", "memory", "cc"); return carry_r; } @@ -108,10 +107,9 @@ static inline void fadd(u64 *out, const u64 *f1, const u64 *f2) " cmovc %0, %%rax;" " add %%rax, %%r8;" " movq %%r8, 0(%1);" - : "+&r" (f2) - : "r" (out), "r" (f1) - : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc" - ); + : "+&r"(f2) + : "r"(out), "r"(f1) + : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"); } /* Computes the field subtraction of two field elements */ @@ -151,10 +149,9 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2) " movq %%r9, 8(%0);" " movq %%r10, 16(%0);" " movq %%r11, 24(%0);" - : - : "r" (out), "r" (f1), "r" (f2) - : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc" - ); + : + : "r"(out), "r"(f1), "r"(f2) + : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"); } /* Computes a field multiplication: out <- f1 * f2 @@ -162,239 +159,400 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2) static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) { asm volatile( + /* Compute the raw multiplication: tmp <- src1 * src2 */ /* Compute src1[0] * src2 */ - " movq 0(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" + " movq 0(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " movq %%r8, 0(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " movq %%r10, 8(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + /* Compute src1[1] * src2 */ - " movq 8(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 8(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 8(%2), %%r8;" + " movq %%r8, 8(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 16(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[2] * src2 */ - " movq 16(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 16(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 16(%2), %%r8;" + " movq %%r8, 16(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 24(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[3] * src2 */ - " movq 24(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);" + " movq 24(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 24(%2), %%r8;" + " movq %%r8, 24(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 32(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " movq %%rbx, 40(%2);" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " movq %%r14, 48(%2);" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + " movq %%rax, 56(%2);" + /* Line up pointers */ - " mov %0, %1;" " mov %2, %0;" + " mov %3, %2;" /* Wrap the result back into the field */ /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" - " xor %k3, %k3;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" + " mulxq 32(%0), %%r8, %%r13;" + " xor %k1, %k1;" + " adoxq 0(%0), %%r8;" + " mulxq 40(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" + " adoxq 8(%0), %%r9;" + " mulxq 48(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" + " adoxq 16(%0), %%r10;" + " mulxq 56(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" - " adcx %3, %%rax;" - " adox %3, %%rax;" + " adoxq 24(%0), %%r11;" + " adcx %1, %%rax;" + " adox %1, %%rax;" " imul %%rdx, %%rax;" /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" - " adcx %3, %%r9;" - " movq %%r9, 8(%0);" - " adcx %3, %%r10;" - " movq %%r10, 16(%0);" - " adcx %3, %%r11;" - " movq %%r11, 24(%0);" + " adcx %1, %%r9;" + " movq %%r9, 8(%2);" + " adcx %1, %%r10;" + " movq %%r10, 16(%2);" + " adcx %1, %%r11;" + " movq %%r11, 24(%2);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 0(%0);" - : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2) - : - : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc" - ); + " movq %%r8, 0(%2);" + : "+&r"(f1), "+&r"(f2), "+&r"(tmp) + : "r"(out) + : "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", + "%r14", "memory", "cc"); } /* Computes two field multiplications: - * out[0] <- f1[0] * f2[0] - * out[1] <- f1[1] * f2[1] - * Uses the 16-element buffer tmp for intermediate results. */ + * out[0] <- f1[0] * f2[0] + * out[1] <- f1[1] * f2[1] + * Uses the 16-element buffer tmp for intermediate results: */ static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) { asm volatile( + /* Compute the raw multiplication tmp[0] <- f1[0] * f2[0] */ /* Compute src1[0] * src2 */ - " movq 0(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" + " movq 0(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " movq %%r8, 0(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " movq %%r10, 8(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + /* Compute src1[1] * src2 */ - " movq 8(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 8(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 8(%2), %%r8;" + " movq %%r8, 8(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 16(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[2] * src2 */ - " movq 16(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 16(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 16(%2), %%r8;" + " movq %%r8, 16(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 24(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[3] * src2 */ - " movq 24(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);" + " movq 24(%0), %%rdx;" + " mulxq 0(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 24(%2), %%r8;" + " movq %%r8, 24(%2);" + " mulxq 8(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 32(%2);" + " mulxq 16(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " movq %%rbx, 40(%2);" + " mov $0, %%r8;" + " mulxq 24(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " movq %%r14, 48(%2);" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + " movq %%rax, 56(%2);" /* Compute the raw multiplication tmp[1] <- f1[1] * f2[1] */ /* Compute src1[0] * src2 */ - " movq 32(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 64(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" + " movq 32(%0), %%rdx;" + " mulxq 32(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " movq %%r8, 64(%2);" + " mulxq 40(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " movq %%r10, 72(%2);" + " mulxq 48(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " mulxq 56(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + /* Compute src1[1] * src2 */ - " movq 40(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 40(%0), %%rdx;" + " mulxq 32(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 72(%2), %%r8;" + " movq %%r8, 72(%2);" + " mulxq 40(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 80(%2);" + " mulxq 48(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 56(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[2] * src2 */ - " movq 48(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + " movq 48(%0), %%rdx;" + " mulxq 32(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 80(%2), %%r8;" + " movq %%r8, 80(%2);" + " mulxq 40(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 88(%2);" + " mulxq 48(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " mov $0, %%r8;" + " mulxq 56(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + /* Compute src1[3] * src2 */ - " movq 56(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 104(%0);" " mov $0, %%r8;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 112(%0);" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 120(%0);" + " movq 56(%0), %%rdx;" + " mulxq 32(%1), %%r8, %%r9;" + " xor %%r10d, %%r10d;" + " adcxq 88(%2), %%r8;" + " movq %%r8, 88(%2);" + " mulxq 40(%1), %%r10, %%r11;" + " adox %%r9, %%r10;" + " adcx %%rbx, %%r10;" + " movq %%r10, 96(%2);" + " mulxq 48(%1), %%rbx, %%r13;" + " adox %%r11, %%rbx;" + " adcx %%r14, %%rbx;" + " movq %%rbx, 104(%2);" + " mov $0, %%r8;" + " mulxq 56(%1), %%r14, %%rdx;" + " adox %%r13, %%r14;" + " adcx %%rax, %%r14;" + " movq %%r14, 112(%2);" + " mov $0, %%rax;" + " adox %%rdx, %%rax;" + " adcx %%r8, %%rax;" + " movq %%rax, 120(%2);" + /* Line up pointers */ - " mov %0, %1;" " mov %2, %0;" + " mov %3, %2;" /* Wrap the results back into the field */ /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" - " xor %k3, %k3;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" + " mulxq 32(%0), %%r8, %%r13;" + " xor %k1, %k1;" + " adoxq 0(%0), %%r8;" + " mulxq 40(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" + " adoxq 8(%0), %%r9;" + " mulxq 48(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" + " adoxq 16(%0), %%r10;" + " mulxq 56(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" - " adcx %3, %%rax;" - " adox %3, %%rax;" + " adoxq 24(%0), %%r11;" + " adcx %1, %%rax;" + " adox %1, %%rax;" " imul %%rdx, %%rax;" /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" - " adcx %3, %%r9;" - " movq %%r9, 8(%0);" - " adcx %3, %%r10;" - " movq %%r10, 16(%0);" - " adcx %3, %%r11;" - " movq %%r11, 24(%0);" + " adcx %1, %%r9;" + " movq %%r9, 8(%2);" + " adcx %1, %%r10;" + " movq %%r10, 16(%2);" + " adcx %1, %%r11;" + " movq %%r11, 24(%2);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 0(%0);" + " movq %%r8, 0(%2);" /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 96(%1), %%r8, %%r13;" - " xor %k3, %k3;" - " adoxq 64(%1), %%r8;" - " mulxq 104(%1), %%r9, %%rbx;" + " mulxq 96(%0), %%r8, %%r13;" + " xor %k1, %k1;" + " adoxq 64(%0), %%r8;" + " mulxq 104(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 72(%1), %%r9;" - " mulxq 112(%1), %%r10, %%r13;" + " adoxq 72(%0), %%r9;" + " mulxq 112(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 80(%1), %%r10;" - " mulxq 120(%1), %%r11, %%rax;" + " adoxq 80(%0), %%r10;" + " mulxq 120(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 88(%1), %%r11;" - " adcx %3, %%rax;" - " adox %3, %%rax;" + " adoxq 88(%0), %%r11;" + " adcx %1, %%rax;" + " adox %1, %%rax;" " imul %%rdx, %%rax;" /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" - " adcx %3, %%r9;" - " movq %%r9, 40(%0);" - " adcx %3, %%r10;" - " movq %%r10, 48(%0);" - " adcx %3, %%r11;" - " movq %%r11, 56(%0);" + " adcx %1, %%r9;" + " movq %%r9, 40(%2);" + " adcx %1, %%r10;" + " movq %%r10, 48(%2);" + " adcx %1, %%r11;" + " movq %%r11, 56(%2);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 32(%0);" - : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2) - : - : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc" - ); + " movq %%r8, 32(%2);" + : "+&r"(f1), "+&r"(f2), "+&r"(tmp) + : "r"(out) + : "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", + "%r14", "memory", "cc"); } -/* Computes the field multiplication of four-element f1 with value in f2 */ +/* Computes the field multiplication of four-element f1 with value in f2 + * Requires f2 to be smaller than 2^17 */ static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2) { register u64 f2_r asm("rdx") = f2; asm volatile( /* Compute the raw multiplication of f1*f2 */ - " mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */ - " mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */ + " mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */ + " mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */ " add %%rcx, %%r9;" " mov $0, %%rcx;" - " mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */ + " mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */ " adcx %%rbx, %%r10;" - " mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */ + " mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */ " adcx %%r13, %%r11;" " adcx %%rcx, %%rax;" @@ -418,17 +576,17 @@ static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2) " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" " movq %%r8, 0(%1);" - : "+&r" (f2_r) - : "r" (out), "r" (f1) - : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "memory", "cc" - ); + : "+&r"(f2_r) + : "r"(out), "r"(f1) + : "%rax", "%rbx", "%rcx", "%r8", "%r9", "%r10", "%r11", "%r13", + "memory", "cc"); } /* Computes p1 <- bit ? p2 : p1 in constant time */ static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2) { asm volatile( - /* Invert the polarity of bit to match cmov expectations */ + /* Transfer bit into CF flag */ " add $18446744073709551615, %0;" /* cswap p1[0], p2[0] */ @@ -502,10 +660,9 @@ static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2) " cmovc %%r10, %%r9;" " movq %%r8, 56(%1);" " movq %%r9, 56(%2);" - : "+&r" (bit) - : "r" (p1), "r" (p2) - : "%r8", "%r9", "%r10", "memory", "cc" - ); + : "+&r"(bit) + : "r"(p1), "r"(p2) + : "%r8", "%r9", "%r10", "memory", "cc"); } /* Computes the square of a field element: out <- f * f @@ -516,15 +673,22 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) /* Compute the raw multiplication: tmp <- f * f */ /* Step 1: Compute all partial products */ - " movq 0(%1), %%rdx;" /* f[0] */ - " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */ - " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ - " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ - " movq 24(%1), %%rdx;" /* f[3] */ - " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ - " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ - " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ - " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ + " movq 0(%0), %%rdx;" /* f[0] */ + " mulxq 8(%0), %%r8, %%r14;" + " xor %%r15d, %%r15d;" /* f[1]*f[0] */ + " mulxq 16(%0), %%r9, %%r10;" + " adcx %%r14, %%r9;" /* f[2]*f[0] */ + " mulxq 24(%0), %%rax, %%rcx;" + " adcx %%rax, %%r10;" /* f[3]*f[0] */ + " movq 24(%0), %%rdx;" /* f[3] */ + " mulxq 8(%0), %%r11, %%rbx;" + " adcx %%rcx, %%r11;" /* f[1]*f[3] */ + " mulxq 16(%0), %%rax, %%r13;" + " adcx %%rax, %%rbx;" /* f[2]*f[3] */ + " movq 8(%0), %%rdx;" + " adcx %%r15, %%r13;" /* f1 */ + " mulxq 16(%0), %%rax, %%rcx;" + " mov $0, %%r14;" /* f[2]*f[1] */ /* Step 2: Compute two parallel carry chains */ " xor %%r15d, %%r15d;" @@ -542,39 +706,50 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) " adcx %%r14, %%r14;" /* Step 3: Compute intermediate squares */ - " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ - " movq %%rax, 0(%0);" - " add %%rcx, %%r8;" " movq %%r8, 8(%0);" - " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ - " adcx %%rax, %%r9;" " movq %%r9, 16(%0);" - " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);" - " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ - " adcx %%rax, %%r11;" " movq %%r11, 32(%0);" - " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);" - " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ - " adcx %%rax, %%r13;" " movq %%r13, 48(%0);" - " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);" + " movq 0(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ + " movq %%rax, 0(%1);" + " add %%rcx, %%r8;" + " movq %%r8, 8(%1);" + " movq 8(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ + " adcx %%rax, %%r9;" + " movq %%r9, 16(%1);" + " adcx %%rcx, %%r10;" + " movq %%r10, 24(%1);" + " movq 16(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ + " adcx %%rax, %%r11;" + " movq %%r11, 32(%1);" + " adcx %%rcx, %%rbx;" + " movq %%rbx, 40(%1);" + " movq 24(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ + " adcx %%rax, %%r13;" + " movq %%r13, 48(%1);" + " adcx %%rcx, %%r14;" + " movq %%r14, 56(%1);" /* Line up pointers */ - " mov %0, %1;" - " mov %2, %0;" + " mov %1, %0;" + " mov %2, %1;" /* Wrap the result back into the field */ /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" + " mulxq 32(%0), %%r8, %%r13;" " xor %%ecx, %%ecx;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" + " adoxq 0(%0), %%r8;" + " mulxq 40(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" + " adoxq 8(%0), %%r9;" + " mulxq 48(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" + " adoxq 16(%0), %%r10;" + " mulxq 56(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" + " adoxq 24(%0), %%r11;" " adcx %%rcx, %%rax;" " adox %%rcx, %%rax;" " imul %%rdx, %%rax;" @@ -582,40 +757,47 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" " adcx %%rcx, %%r9;" - " movq %%r9, 8(%0);" + " movq %%r9, 8(%1);" " adcx %%rcx, %%r10;" - " movq %%r10, 16(%0);" + " movq %%r10, 16(%1);" " adcx %%rcx, %%r11;" - " movq %%r11, 24(%0);" + " movq %%r11, 24(%1);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 0(%0);" - : "+&r" (tmp), "+&r" (f), "+&r" (out) - : - : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc" - ); + " movq %%r8, 0(%1);" + : "+&r"(f), "+&r"(tmp) + : "r"(out) + : "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", + "%r13", "%r14", "%r15", "memory", "cc"); } /* Computes two field squarings: - * out[0] <- f[0] * f[0] - * out[1] <- f[1] * f[1] + * out[0] <- f[0] * f[0] + * out[1] <- f[1] * f[1] * Uses the 16-element buffer tmp for intermediate results */ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) { asm volatile( /* Step 1: Compute all partial products */ - " movq 0(%1), %%rdx;" /* f[0] */ - " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */ - " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ - " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ - " movq 24(%1), %%rdx;" /* f[3] */ - " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ - " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ - " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ - " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ + " movq 0(%0), %%rdx;" /* f[0] */ + " mulxq 8(%0), %%r8, %%r14;" + " xor %%r15d, %%r15d;" /* f[1]*f[0] */ + " mulxq 16(%0), %%r9, %%r10;" + " adcx %%r14, %%r9;" /* f[2]*f[0] */ + " mulxq 24(%0), %%rax, %%rcx;" + " adcx %%rax, %%r10;" /* f[3]*f[0] */ + " movq 24(%0), %%rdx;" /* f[3] */ + " mulxq 8(%0), %%r11, %%rbx;" + " adcx %%rcx, %%r11;" /* f[1]*f[3] */ + " mulxq 16(%0), %%rax, %%r13;" + " adcx %%rax, %%rbx;" /* f[2]*f[3] */ + " movq 8(%0), %%rdx;" + " adcx %%r15, %%r13;" /* f1 */ + " mulxq 16(%0), %%rax, %%rcx;" + " mov $0, %%r14;" /* f[2]*f[1] */ /* Step 2: Compute two parallel carry chains */ " xor %%r15d, %%r15d;" @@ -633,29 +815,47 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) " adcx %%r14, %%r14;" /* Step 3: Compute intermediate squares */ - " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ - " movq %%rax, 0(%0);" - " add %%rcx, %%r8;" " movq %%r8, 8(%0);" - " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ - " adcx %%rax, %%r9;" " movq %%r9, 16(%0);" - " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);" - " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ - " adcx %%rax, %%r11;" " movq %%r11, 32(%0);" - " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);" - " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ - " adcx %%rax, %%r13;" " movq %%r13, 48(%0);" - " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);" + " movq 0(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ + " movq %%rax, 0(%1);" + " add %%rcx, %%r8;" + " movq %%r8, 8(%1);" + " movq 8(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ + " adcx %%rax, %%r9;" + " movq %%r9, 16(%1);" + " adcx %%rcx, %%r10;" + " movq %%r10, 24(%1);" + " movq 16(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ + " adcx %%rax, %%r11;" + " movq %%r11, 32(%1);" + " adcx %%rcx, %%rbx;" + " movq %%rbx, 40(%1);" + " movq 24(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ + " adcx %%rax, %%r13;" + " movq %%r13, 48(%1);" + " adcx %%rcx, %%r14;" + " movq %%r14, 56(%1);" /* Step 1: Compute all partial products */ - " movq 32(%1), %%rdx;" /* f[0] */ - " mulxq 40(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */ - " mulxq 48(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ - " mulxq 56(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ - " movq 56(%1), %%rdx;" /* f[3] */ - " mulxq 40(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ - " mulxq 48(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ - " movq 40(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ - " mulxq 48(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ + " movq 32(%0), %%rdx;" /* f[0] */ + " mulxq 40(%0), %%r8, %%r14;" + " xor %%r15d, %%r15d;" /* f[1]*f[0] */ + " mulxq 48(%0), %%r9, %%r10;" + " adcx %%r14, %%r9;" /* f[2]*f[0] */ + " mulxq 56(%0), %%rax, %%rcx;" + " adcx %%rax, %%r10;" /* f[3]*f[0] */ + " movq 56(%0), %%rdx;" /* f[3] */ + " mulxq 40(%0), %%r11, %%rbx;" + " adcx %%rcx, %%r11;" /* f[1]*f[3] */ + " mulxq 48(%0), %%rax, %%r13;" + " adcx %%rax, %%rbx;" /* f[2]*f[3] */ + " movq 40(%0), %%rdx;" + " adcx %%r15, %%r13;" /* f1 */ + " mulxq 48(%0), %%rax, %%rcx;" + " mov $0, %%r14;" /* f[2]*f[1] */ /* Step 2: Compute two parallel carry chains */ " xor %%r15d, %%r15d;" @@ -673,37 +873,48 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) " adcx %%r14, %%r14;" /* Step 3: Compute intermediate squares */ - " movq 32(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ - " movq %%rax, 64(%0);" - " add %%rcx, %%r8;" " movq %%r8, 72(%0);" - " movq 40(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ - " adcx %%rax, %%r9;" " movq %%r9, 80(%0);" - " adcx %%rcx, %%r10;" " movq %%r10, 88(%0);" - " movq 48(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ - " adcx %%rax, %%r11;" " movq %%r11, 96(%0);" - " adcx %%rcx, %%rbx;" " movq %%rbx, 104(%0);" - " movq 56(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ - " adcx %%rax, %%r13;" " movq %%r13, 112(%0);" - " adcx %%rcx, %%r14;" " movq %%r14, 120(%0);" + " movq 32(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ + " movq %%rax, 64(%1);" + " add %%rcx, %%r8;" + " movq %%r8, 72(%1);" + " movq 40(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ + " adcx %%rax, %%r9;" + " movq %%r9, 80(%1);" + " adcx %%rcx, %%r10;" + " movq %%r10, 88(%1);" + " movq 48(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ + " adcx %%rax, %%r11;" + " movq %%r11, 96(%1);" + " adcx %%rcx, %%rbx;" + " movq %%rbx, 104(%1);" + " movq 56(%0), %%rdx;" + " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ + " adcx %%rax, %%r13;" + " movq %%r13, 112(%1);" + " adcx %%rcx, %%r14;" + " movq %%r14, 120(%1);" /* Line up pointers */ - " mov %0, %1;" - " mov %2, %0;" + " mov %1, %0;" + " mov %2, %1;" /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" + " mulxq 32(%0), %%r8, %%r13;" " xor %%ecx, %%ecx;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" + " adoxq 0(%0), %%r8;" + " mulxq 40(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" + " adoxq 8(%0), %%r9;" + " mulxq 48(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" + " adoxq 16(%0), %%r10;" + " mulxq 56(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" + " adoxq 24(%0), %%r11;" " adcx %%rcx, %%rax;" " adox %%rcx, %%rax;" " imul %%rdx, %%rax;" @@ -711,32 +922,32 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" " adcx %%rcx, %%r9;" - " movq %%r9, 8(%0);" + " movq %%r9, 8(%1);" " adcx %%rcx, %%r10;" - " movq %%r10, 16(%0);" + " movq %%r10, 16(%1);" " adcx %%rcx, %%r11;" - " movq %%r11, 24(%0);" + " movq %%r11, 24(%1);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 0(%0);" + " movq %%r8, 0(%1);" /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ " mov $38, %%rdx;" - " mulxq 96(%1), %%r8, %%r13;" + " mulxq 96(%0), %%r8, %%r13;" " xor %%ecx, %%ecx;" - " adoxq 64(%1), %%r8;" - " mulxq 104(%1), %%r9, %%rbx;" + " adoxq 64(%0), %%r8;" + " mulxq 104(%0), %%r9, %%rbx;" " adcx %%r13, %%r9;" - " adoxq 72(%1), %%r9;" - " mulxq 112(%1), %%r10, %%r13;" + " adoxq 72(%0), %%r9;" + " mulxq 112(%0), %%r10, %%r13;" " adcx %%rbx, %%r10;" - " adoxq 80(%1), %%r10;" - " mulxq 120(%1), %%r11, %%rax;" + " adoxq 80(%0), %%r10;" + " mulxq 120(%0), %%r11, %%rax;" " adcx %%r13, %%r11;" - " adoxq 88(%1), %%r11;" + " adoxq 88(%0), %%r11;" " adcx %%rcx, %%rax;" " adox %%rcx, %%rax;" " imul %%rdx, %%rax;" @@ -744,21 +955,21 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) /* Step 2: Fold the carry back into dst */ " add %%rax, %%r8;" " adcx %%rcx, %%r9;" - " movq %%r9, 40(%0);" + " movq %%r9, 40(%1);" " adcx %%rcx, %%r10;" - " movq %%r10, 48(%0);" + " movq %%r10, 48(%1);" " adcx %%rcx, %%r11;" - " movq %%r11, 56(%0);" + " movq %%r11, 56(%1);" /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ " mov $0, %%rax;" " cmovc %%rdx, %%rax;" " add %%rax, %%r8;" - " movq %%r8, 32(%0);" - : "+&r" (tmp), "+&r" (f), "+&r" (out) - : - : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc" - ); + " movq %%r8, 32(%1);" + : "+&r"(f), "+&r"(tmp) + : "r"(out) + : "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", + "%r13", "%r14", "%r15", "memory", "cc"); } static void point_add_and_double(u64 *q, u64 *p01_tmp1, u64 *tmp2) diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c index e7cb68a3db..787c234d24 100644 --- a/arch/x86/crypto/des3_ede_glue.c +++ b/arch/x86/crypto/des3_ede_glue.c @@ -164,7 +164,7 @@ static int cbc_encrypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes)) { + while (walk.nbytes) { nbytes = __cbc_encrypt(ctx, &walk); err = skcipher_walk_done(&walk, nbytes); } @@ -243,7 +243,7 @@ static int cbc_decrypt(struct skcipher_request *req) err = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes)) { + while (walk.nbytes) { nbytes = __cbc_decrypt(ctx, &walk); err = skcipher_walk_done(&walk, nbytes); } diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index ccb9d32768..8874208440 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -268,19 +268,16 @@ 1: popl %ds 2: popl %es 3: popl %fs - addl $(4 + \pop), %esp /* pop the unused "gs" slot */ +4: addl $(4 + \pop), %esp /* pop the unused "gs" slot */ IRET_FRAME -.pushsection .fixup, "ax" -4: movl $0, (%esp) - jmp 1b -5: movl $0, (%esp) - jmp 2b -6: movl $0, (%esp) - jmp 3b -.popsection - _ASM_EXTABLE(1b, 4b) - _ASM_EXTABLE(2b, 5b) - _ASM_EXTABLE(3b, 6b) + + /* + * There is no _ASM_EXTABLE_TYPE_REG() for ASM, however since this is + * ASM the registers are known and we can trivially hard-code them. + */ + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_POP_ZERO|EX_REG_DS) + _ASM_EXTABLE_TYPE(2b, 3b, EX_TYPE_POP_ZERO|EX_REG_ES) + _ASM_EXTABLE_TYPE(3b, 4b, EX_TYPE_POP_ZERO|EX_REG_FS) .endm .macro RESTORE_ALL_NMI cr3_reg:req pop=0 @@ -740,7 +737,7 @@ SYM_FUNC_START(schedule_tail_wrapper) popl %eax FRAME_END - ret + RET SYM_FUNC_END(schedule_tail_wrapper) .popsection @@ -925,10 +922,8 @@ SYM_FUNC_START(entry_SYSENTER_32) sti sysexit -.pushsection .fixup, "ax" -2: movl $0, PT_FS(%esp) - jmp 1b -.popsection +2: movl $0, PT_FS(%esp) + jmp 1b _ASM_EXTABLE(1b, 2b) .Lsysenter_fix_flags: @@ -996,8 +991,7 @@ restore_all_switch_stack: */ iret -.section .fixup, "ax" -SYM_CODE_START(asm_iret_error) +.Lasm_iret_error: pushl $0 # no error code pushl $iret_error @@ -1014,9 +1008,8 @@ SYM_CODE_START(asm_iret_error) #endif jmp handle_exception -SYM_CODE_END(asm_iret_error) -.previous - _ASM_EXTABLE(.Lirq_return, asm_iret_error) + + _ASM_EXTABLE(.Lirq_return, .Lasm_iret_error) SYM_FUNC_END(entry_INT80_32) .macro FIXUP_ESPFIX_STACK @@ -1248,14 +1241,14 @@ SYM_CODE_START(asm_exc_nmi) SYM_CODE_END(asm_exc_nmi) .pushsection .text, "ax" -SYM_CODE_START(rewind_stack_do_exit) +SYM_CODE_START(rewind_stack_and_make_dead) /* Prevent any naive code from trying to unwind to our caller. */ xorl %ebp, %ebp movl PER_CPU_VAR(cpu_current_top_of_stack), %esi leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp - call do_exit + call make_task_dead 1: jmp 1b -SYM_CODE_END(rewind_stack_do_exit) +SYM_CODE_END(rewind_stack_and_make_dead) .popsection diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 97b1f84bb5..466df3e502 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -738,14 +738,10 @@ SYM_FUNC_START(asm_load_gs_index) 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE swapgs FRAME_END - ret -SYM_FUNC_END(asm_load_gs_index) -EXPORT_SYMBOL(asm_load_gs_index) + RET - _ASM_EXTABLE(.Lgs_change, .Lbad_gs) - .section .fixup, "ax" /* running with kernelgs */ -SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) +.Lbad_gs: swapgs /* switch back to user gs */ .macro ZAP_GS /* This can't be a string because the preprocessor needs to see it. */ @@ -756,8 +752,11 @@ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) xorl %eax, %eax movl %eax, %gs jmp 2b -SYM_CODE_END(.Lbad_gs) - .previous + + _ASM_EXTABLE(.Lgs_change, .Lbad_gs) + +SYM_FUNC_END(asm_load_gs_index) +EXPORT_SYMBOL(asm_load_gs_index) #ifdef CONFIG_XEN_PV /* @@ -889,7 +888,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) * is needed here. */ SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx - ret + RET .Lparanoid_entry_checkgs: /* EBX = 1 -> kernel GSBASE active, no restore required */ @@ -910,7 +909,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) .Lparanoid_kernel_gsbase: FENCE_SWAPGS_KERNEL_ENTRY - ret + RET SYM_CODE_END(paranoid_entry) /* @@ -989,7 +988,7 @@ SYM_CODE_START_LOCAL(error_entry) movq %rax, %rsp /* switch stack */ ENCODE_FRAME_POINTER pushq %r12 - ret + RET /* * There are two places in the kernel that can potentially fault with @@ -1020,7 +1019,7 @@ SYM_CODE_START_LOCAL(error_entry) */ .Lerror_entry_done_lfence: FENCE_SWAPGS_KERNEL_ENTRY - ret + RET .Lbstep_iret: /* Fix truncated RIP */ @@ -1428,7 +1427,7 @@ SYM_CODE_END(ignore_sysret) #endif .pushsection .text, "ax" -SYM_CODE_START(rewind_stack_do_exit) +SYM_CODE_START(rewind_stack_and_make_dead) UNWIND_HINT_FUNC /* Prevent any naive code from trying to unwind to our caller. */ xorl %ebp, %ebp @@ -1437,6 +1436,6 @@ SYM_CODE_START(rewind_stack_do_exit) leaq -PTREGS_SIZE(%rax), %rsp UNWIND_HINT_REGS - call do_exit -SYM_CODE_END(rewind_stack_do_exit) + call make_task_dead +SYM_CODE_END(rewind_stack_and_make_dead) .popsection diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 960a021d54..320480a8db 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -453,3 +453,5 @@ 446 i386 landlock_restrict_self sys_landlock_restrict_self 447 i386 memfd_secret sys_memfd_secret 448 i386 process_mrelease sys_process_mrelease +449 i386 futex_waitv sys_futex_waitv +450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 18b5500ea8..c84d12608c 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -370,6 +370,8 @@ 446 common landlock_restrict_self sys_landlock_restrict_self 447 common memfd_secret sys_memfd_secret 448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node # # Due to a historical design error, certain syscalls are numbered differently diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index a2dddcc189..693f8b9031 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -172,7 +172,7 @@ $(obj)/vdso32.so.dbg: $(obj)/vdso32/vdso32.lds $(vobjs32) FORCE # The DSO images are built using a special linker script. # quiet_cmd_vdso = VDSO $@ - cmd_vdso = $(LD) -nostdlib -o $@ \ + cmd_vdso = $(LD) -o $@ \ $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ -T $(filter %.lds,$^) $(filter %.o,$^) && \ sh $(srctree)/$(src)/checkundef.sh '$(NM)' '$@' diff --git a/arch/x86/entry/vdso/vdso-layout.lds.S b/arch/x86/entry/vdso/vdso-layout.lds.S index dc8da76958..bafa73f09e 100644 --- a/arch/x86/entry/vdso/vdso-layout.lds.S +++ b/arch/x86/entry/vdso/vdso-layout.lds.S @@ -77,7 +77,6 @@ SECTIONS .text : { *(.text*) - *(.fixup) } :text =0x90909090, diff --git a/arch/x86/events/amd/iommu.c b/arch/x86/events/amd/iommu.c index 913745f141..b15f7b950d 100644 --- a/arch/x86/events/amd/iommu.c +++ b/arch/x86/events/amd/iommu.c @@ -161,7 +161,7 @@ static int get_next_avail_iommu_bnk_cntr(struct perf_event *event) raw_spin_lock_irqsave(&piommu->lock, flags); - for (bank = 0, shift = 0; bank < max_banks; bank++) { + for (bank = 0; bank < max_banks; bank++) { for (cntr = 0; cntr < max_cntrs; cntr++) { shift = bank + (bank*3) + cntr; if (piommu->cntr_assign_mask & BIT_ULL(shift)) { diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 81d5e0a1f4..e686c5e053 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -66,6 +66,8 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_enable_all, *x86_pmu.enable_all); DEFINE_STATIC_CALL_NULL(x86_pmu_enable, *x86_pmu.enable); DEFINE_STATIC_CALL_NULL(x86_pmu_disable, *x86_pmu.disable); +DEFINE_STATIC_CALL_NULL(x86_pmu_assign, *x86_pmu.assign); + DEFINE_STATIC_CALL_NULL(x86_pmu_add, *x86_pmu.add); DEFINE_STATIC_CALL_NULL(x86_pmu_del, *x86_pmu.del); DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read); @@ -1215,6 +1217,8 @@ static inline void x86_assign_hw_event(struct perf_event *event, hwc->last_cpu = smp_processor_id(); hwc->last_tag = ++cpuc->tags[i]; + static_call_cond(x86_pmu_assign)(event, idx); + switch (hwc->idx) { case INTEL_PMC_IDX_FIXED_BTS: case INTEL_PMC_IDX_FIXED_VLBR: @@ -2005,6 +2009,8 @@ static void x86_pmu_static_call_update(void) static_call_update(x86_pmu_enable, x86_pmu.enable); static_call_update(x86_pmu_disable, x86_pmu.disable); + static_call_update(x86_pmu_assign, x86_pmu.assign); + static_call_update(x86_pmu_add, x86_pmu.add); static_call_update(x86_pmu_del, x86_pmu.del); static_call_update(x86_pmu_read, x86_pmu.read); @@ -2470,7 +2476,7 @@ static int x86_pmu_event_init(struct perf_event *event) if (READ_ONCE(x86_pmu.attr_rdpmc) && !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) - event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; + event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; return err; } @@ -2504,7 +2510,7 @@ void perf_clear_dirty_counters(void) static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) { - if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) + if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) return; /* @@ -2525,7 +2531,7 @@ static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) { - if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) + if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) return; if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) @@ -2536,7 +2542,7 @@ static int x86_pmu_event_idx(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; - if (!(hwc->flags & PERF_X86_EVENT_RDPMC_ALLOWED)) + if (!(hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT)) return 0; if (is_metric_idx(hwc->idx)) @@ -2719,7 +2725,7 @@ void arch_perf_update_userpage(struct perf_event *event, userpg->cap_user_time = 0; userpg->cap_user_time_zero = 0; userpg->cap_user_rdpmc = - !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); + !!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT); userpg->pmc_width = x86_pmu.cntval_bits; if (!using_native_sched_clock() || !sched_clock_stable()) @@ -2762,11 +2768,10 @@ static bool perf_hw_regs(struct pt_regs *regs) void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct unwind_state state; unsigned long addr; - if (guest_cbs && guest_cbs->is_in_guest()) { + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; } @@ -2866,11 +2871,10 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *ent void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stack_frame frame; const struct stack_frame __user *fp; - if (guest_cbs && guest_cbs->is_in_guest()) { + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; } @@ -2947,21 +2951,19 @@ static unsigned long code_segment_base(struct pt_regs *regs) unsigned long perf_instruction_pointer(struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - if (guest_cbs && guest_cbs->is_in_guest()) - return guest_cbs->get_guest_ip(); + if (perf_guest_state()) + return perf_guest_get_ip(); return regs->ip + code_segment_base(regs); } unsigned long perf_misc_flags(struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + unsigned int guest_state = perf_guest_state(); int misc = 0; - if (guest_cbs && guest_cbs->is_in_guest()) { - if (guest_cbs->is_user_mode()) + if (guest_state) { + if (guest_state & PERF_GUEST_USER) misc |= PERF_RECORD_MISC_GUEST_USER; else misc |= PERF_RECORD_MISC_GUEST_KERNEL; diff --git a/arch/x86/events/intel/bts.c b/arch/x86/events/intel/bts.c index 6320d2cfd9..974e917e65 100644 --- a/arch/x86/events/intel/bts.c +++ b/arch/x86/events/intel/bts.c @@ -209,6 +209,12 @@ static void bts_update(struct bts_ctx *bts) } else { local_set(&buf->data_size, head); } + + /* + * Since BTS is coherent, just add compiler barrier to ensure + * BTS updating is ordered against bts::handle::event. + */ + barrier(); } static int diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 97ede6fb15..a3c7ca876a 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2145,19 +2145,19 @@ static __initconst const u64 knl_hw_cache_extra_regs * However, there are some cases which may change PEBS status, e.g. PMI * throttle. The PEBS_ENABLE should be updated where the status changes. */ -static void __intel_pmu_disable_all(void) +static __always_inline void __intel_pmu_disable_all(bool bts) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0); - if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) + if (bts && test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) intel_pmu_disable_bts(); } -static void intel_pmu_disable_all(void) +static __always_inline void intel_pmu_disable_all(void) { - __intel_pmu_disable_all(); + __intel_pmu_disable_all(true); intel_pmu_pebs_disable_all(); intel_pmu_lbr_disable_all(); } @@ -2188,6 +2188,47 @@ static void intel_pmu_enable_all(int added) __intel_pmu_enable_all(added, false); } +static noinline int +__intel_pmu_snapshot_branch_stack(struct perf_branch_entry *entries, + unsigned int cnt, unsigned long flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + intel_pmu_lbr_read(); + cnt = min_t(unsigned int, cnt, x86_pmu.lbr_nr); + + memcpy(entries, cpuc->lbr_entries, sizeof(struct perf_branch_entry) * cnt); + intel_pmu_enable_all(0); + local_irq_restore(flags); + return cnt; +} + +static int +intel_pmu_snapshot_branch_stack(struct perf_branch_entry *entries, unsigned int cnt) +{ + unsigned long flags; + + /* must not have branches... */ + local_irq_save(flags); + __intel_pmu_disable_all(false); /* we don't care about BTS */ + __intel_pmu_lbr_disable(); + /* ... until here */ + return __intel_pmu_snapshot_branch_stack(entries, cnt, flags); +} + +static int +intel_pmu_snapshot_arch_branch_stack(struct perf_branch_entry *entries, unsigned int cnt) +{ + unsigned long flags; + + /* must not have branches... */ + local_irq_save(flags); + __intel_pmu_disable_all(false); /* we don't care about BTS */ + __intel_pmu_arch_lbr_disable(); + /* ... until here */ + return __intel_pmu_snapshot_branch_stack(entries, cnt, flags); +} + /* * Workaround for: * Intel Errata AAK100 (model 26) @@ -2404,6 +2445,12 @@ static void intel_pmu_disable_event(struct perf_event *event) intel_pmu_pebs_disable(event); } +static void intel_pmu_assign_event(struct perf_event *event, int idx) +{ + if (is_pebs_pt(event)) + perf_report_aux_output_id(event, idx); +} + static void intel_pmu_del_event(struct perf_event *event) { if (needs_branch_stack(event)) @@ -2788,7 +2835,6 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) { struct perf_sample_data data; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - struct perf_guest_info_callbacks *guest_cbs; int bit; int handled = 0; u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); @@ -2855,12 +2901,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) */ if (__test_and_clear_bit(GLOBAL_STATUS_TRACE_TOPAPMI_BIT, (unsigned long *)&status)) { handled++; - - guest_cbs = perf_get_guest_cbs(); - if (unlikely(guest_cbs && guest_cbs->is_in_guest() && - guest_cbs->handle_intel_pt_intr)) - guest_cbs->handle_intel_pt_intr(); - else + if (!perf_guest_handle_intel_pt_intr()) intel_pt_interrupt(); } @@ -2934,7 +2975,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs) apic_write(APIC_LVTPC, APIC_DM_NMI); intel_bts_disable_local(); cpuc->enabled = 0; - __intel_pmu_disable_all(); + __intel_pmu_disable_all(true); handled = intel_pmu_drain_bts_buffer(); handled += intel_bts_interrupt(); status = intel_pmu_get_status(); @@ -4501,8 +4542,16 @@ static int intel_pmu_check_period(struct perf_event *event, u64 value) return intel_pmu_has_bts_period(event, value) ? -EINVAL : 0; } +static void intel_aux_output_init(void) +{ + /* Refer also intel_pmu_aux_output_match() */ + if (x86_pmu.intel_cap.pebs_output_pt_available) + x86_pmu.assign = intel_pmu_assign_event; +} + static int intel_pmu_aux_output_match(struct perf_event *event) { + /* intel_pmu_assign_event() is needed, refer intel_aux_output_init() */ if (!x86_pmu.intel_cap.pebs_output_pt_available) return 0; @@ -6316,9 +6365,23 @@ __init int intel_pmu_init(void) x86_pmu.lbr_nr = 0; } - if (x86_pmu.lbr_nr) + if (x86_pmu.lbr_nr) { + intel_pmu_lbr_init(); + pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr); + /* only support branch_stack snapshot for perfmon >= v2 */ + if (x86_pmu.disable_all == intel_pmu_disable_all) { + if (boot_cpu_has(X86_FEATURE_ARCH_LBR)) { + static_call_update(perf_snapshot_branch_stack, + intel_pmu_snapshot_arch_branch_stack); + } else { + static_call_update(perf_snapshot_branch_stack, + intel_pmu_snapshot_branch_stack); + } + } + } + intel_pmu_check_extra_regs(x86_pmu.extra_regs); /* Support full width counters using alternative MSR range */ @@ -6334,6 +6397,8 @@ __init int intel_pmu_init(void) if (is_hybrid()) intel_pmu_check_hybrid_pmus((u64)fixed_mask); + intel_aux_output_init(); + return 0; } diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 4dbb55a43d..2e215369df 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1302,7 +1302,7 @@ void intel_pmu_pebs_disable_all(void) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); if (cpuc->pebs_enabled) - wrmsrl(MSR_IA32_PEBS_ENABLE, 0); + __intel_pmu_pebs_disable_all(); } static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs) diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index f455dd93f9..669c2be147 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -8,14 +8,6 @@ #include "../perf_event.h" -static const enum { - LBR_EIP_FLAGS = 1, - LBR_TSX = 2, -} lbr_desc[LBR_FORMAT_MAX_KNOWN + 1] = { - [LBR_FORMAT_EIP_FLAGS] = LBR_EIP_FLAGS, - [LBR_FORMAT_EIP_FLAGS2] = LBR_EIP_FLAGS | LBR_TSX, -}; - /* * Intel LBR_SELECT bits * Intel Vol3a, April 2011, Section 16.7 Table 16-10 @@ -228,20 +220,6 @@ static void __intel_pmu_lbr_enable(bool pmi) wrmsrl(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN); } -static void __intel_pmu_lbr_disable(void) -{ - u64 debugctl; - - if (static_cpu_has(X86_FEATURE_ARCH_LBR)) { - wrmsrl(MSR_ARCH_LBR_CTL, 0); - return; - } - - rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); - debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); - wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); -} - void intel_pmu_lbr_reset_32(void) { int i; @@ -257,7 +235,7 @@ void intel_pmu_lbr_reset_64(void) for (i = 0; i < x86_pmu.lbr_nr; i++) { wrmsrl(x86_pmu.lbr_from + i, 0); wrmsrl(x86_pmu.lbr_to + i, 0); - if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) + if (x86_pmu.lbr_has_info) wrmsrl(x86_pmu.lbr_info + i, 0); } } @@ -279,6 +257,8 @@ void intel_pmu_lbr_reset(void) cpuc->last_task_ctx = NULL; cpuc->last_log_id = 0; + if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && cpuc->lbr_select) + wrmsrl(MSR_LBR_SELECT, 0); } /* @@ -317,11 +297,10 @@ enum { */ static inline bool lbr_from_signext_quirk_needed(void) { - int lbr_format = x86_pmu.intel_cap.lbr_format; bool tsx_support = boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM); - return !tsx_support && (lbr_desc[lbr_format] & LBR_TSX); + return !tsx_support && x86_pmu.lbr_has_tsx; } static DEFINE_STATIC_KEY_FALSE(lbr_from_quirk_key); @@ -439,12 +418,12 @@ rdlbr_all(struct lbr_entry *lbr, unsigned int idx, bool need_info) void intel_pmu_lbr_restore(void *ctx) { - bool need_info = x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct x86_perf_task_context *task_ctx = ctx; - int i; - unsigned lbr_idx, mask; + bool need_info = x86_pmu.lbr_has_info; u64 tos = task_ctx->tos; + unsigned lbr_idx, mask; + int i; mask = x86_pmu.lbr_nr - 1; for (i = 0; i < task_ctx->valid_lbrs; i++) { @@ -456,7 +435,7 @@ void intel_pmu_lbr_restore(void *ctx) lbr_idx = (tos - i) & mask; wrlbr_from(lbr_idx, 0); wrlbr_to(lbr_idx, 0); - if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) + if (need_info) wrlbr_info(lbr_idx, 0); } @@ -531,9 +510,9 @@ static void __intel_pmu_lbr_restore(void *ctx) void intel_pmu_lbr_save(void *ctx) { - bool need_info = x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct x86_perf_task_context *task_ctx = ctx; + bool need_info = x86_pmu.lbr_has_info; unsigned lbr_idx, mask; u64 tos; int i; @@ -779,8 +758,12 @@ void intel_pmu_lbr_disable_all(void) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - if (cpuc->lbr_users && !vlbr_exclude_host()) + if (cpuc->lbr_users && !vlbr_exclude_host()) { + if (static_cpu_has(X86_FEATURE_ARCH_LBR)) + return __intel_pmu_arch_lbr_disable(); + __intel_pmu_lbr_disable(); + } } void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) @@ -824,7 +807,6 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) { bool need_info = false, call_stack = false; unsigned long mask = x86_pmu.lbr_nr - 1; - int lbr_format = x86_pmu.intel_cap.lbr_format; u64 tos = intel_pmu_lbr_tos(); int i; int out = 0; @@ -839,9 +821,7 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) for (i = 0; i < num; i++) { unsigned long lbr_idx = (tos - i) & mask; u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0; - int skip = 0; u16 cycles = 0; - int lbr_flags = lbr_desc[lbr_format]; from = rdlbr_from(lbr_idx, NULL); to = rdlbr_to(lbr_idx, NULL); @@ -853,38 +833,40 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) if (call_stack && !from) break; - if (lbr_format == LBR_FORMAT_INFO && need_info) { - u64 info; + if (x86_pmu.lbr_has_info) { + if (need_info) { + u64 info; - info = rdlbr_info(lbr_idx, NULL); - mis = !!(info & LBR_INFO_MISPRED); - pred = !mis; - in_tx = !!(info & LBR_INFO_IN_TX); - abort = !!(info & LBR_INFO_ABORT); - cycles = (info & LBR_INFO_CYCLES); - } + info = rdlbr_info(lbr_idx, NULL); + mis = !!(info & LBR_INFO_MISPRED); + pred = !mis; + cycles = (info & LBR_INFO_CYCLES); + if (x86_pmu.lbr_has_tsx) { + in_tx = !!(info & LBR_INFO_IN_TX); + abort = !!(info & LBR_INFO_ABORT); + } + } + } else { + int skip = 0; - if (lbr_format == LBR_FORMAT_TIME) { - mis = !!(from & LBR_FROM_FLAG_MISPRED); - pred = !mis; - skip = 1; - cycles = ((to >> 48) & LBR_INFO_CYCLES); + if (x86_pmu.lbr_from_flags) { + mis = !!(from & LBR_FROM_FLAG_MISPRED); + pred = !mis; + skip = 1; + } + if (x86_pmu.lbr_has_tsx) { + in_tx = !!(from & LBR_FROM_FLAG_IN_TX); + abort = !!(from & LBR_FROM_FLAG_ABORT); + skip = 3; + } + from = (u64)((((s64)from) << skip) >> skip); - to = (u64)((((s64)to) << 16) >> 16); + if (x86_pmu.lbr_to_cycles) { + cycles = ((to >> 48) & LBR_INFO_CYCLES); + to = (u64)((((s64)to) << 16) >> 16); + } } - if (lbr_flags & LBR_EIP_FLAGS) { - mis = !!(from & LBR_FROM_FLAG_MISPRED); - pred = !mis; - skip = 1; - } - if (lbr_flags & LBR_TSX) { - in_tx = !!(from & LBR_FROM_FLAG_IN_TX); - abort = !!(from & LBR_FROM_FLAG_ABORT); - skip = 3; - } - from = (u64)((((s64)from) << skip) >> skip); - /* * Some CPUs report duplicated abort records, * with the second entry not having an abort bit set. @@ -911,37 +893,40 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) cpuc->lbr_stack.hw_idx = tos; } +static DEFINE_STATIC_KEY_FALSE(x86_lbr_mispred); +static DEFINE_STATIC_KEY_FALSE(x86_lbr_cycles); +static DEFINE_STATIC_KEY_FALSE(x86_lbr_type); + static __always_inline int get_lbr_br_type(u64 info) { - if (!static_cpu_has(X86_FEATURE_ARCH_LBR) || !x86_pmu.lbr_br_type) - return 0; + int type = 0; - return (info & LBR_INFO_BR_TYPE) >> LBR_INFO_BR_TYPE_OFFSET; + if (static_branch_likely(&x86_lbr_type)) + type = (info & LBR_INFO_BR_TYPE) >> LBR_INFO_BR_TYPE_OFFSET; + + return type; } static __always_inline bool get_lbr_mispred(u64 info) { - if (static_cpu_has(X86_FEATURE_ARCH_LBR) && !x86_pmu.lbr_mispred) - return 0; + bool mispred = 0; - return !!(info & LBR_INFO_MISPRED); -} + if (static_branch_likely(&x86_lbr_mispred)) + mispred = !!(info & LBR_INFO_MISPRED); -static __always_inline bool get_lbr_predicted(u64 info) -{ - if (static_cpu_has(X86_FEATURE_ARCH_LBR) && !x86_pmu.lbr_mispred) - return 0; - - return !(info & LBR_INFO_MISPRED); + return mispred; } static __always_inline u16 get_lbr_cycles(u64 info) { - if (static_cpu_has(X86_FEATURE_ARCH_LBR) && - !(x86_pmu.lbr_timed_lbr && info & LBR_INFO_CYC_CNT_VALID)) - return 0; + u16 cycles = info & LBR_INFO_CYCLES; - return info & LBR_INFO_CYCLES; + if (static_cpu_has(X86_FEATURE_ARCH_LBR) && + (!static_branch_likely(&x86_lbr_cycles) || + !(info & LBR_INFO_CYC_CNT_VALID))) + cycles = 0; + + return cycles; } static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, @@ -969,7 +954,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, e->from = from; e->to = to; e->mispred = get_lbr_mispred(info); - e->predicted = get_lbr_predicted(info); + e->predicted = !e->mispred; e->in_tx = !!(info & LBR_INFO_IN_TX); e->abort = !!(info & LBR_INFO_ABORT); e->cycles = get_lbr_cycles(info); @@ -1128,7 +1113,7 @@ static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event) if ((br_type & PERF_SAMPLE_BRANCH_NO_CYCLES) && (br_type & PERF_SAMPLE_BRANCH_NO_FLAGS) && - (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)) + x86_pmu.lbr_has_info) reg->config |= LBR_NO_INFO; return 0; @@ -1714,6 +1699,38 @@ void intel_pmu_lbr_init_knl(void) x86_pmu.intel_cap.lbr_format = LBR_FORMAT_EIP_FLAGS; } +void intel_pmu_lbr_init(void) +{ + switch (x86_pmu.intel_cap.lbr_format) { + case LBR_FORMAT_EIP_FLAGS2: + x86_pmu.lbr_has_tsx = 1; + fallthrough; + case LBR_FORMAT_EIP_FLAGS: + x86_pmu.lbr_from_flags = 1; + break; + + case LBR_FORMAT_INFO: + x86_pmu.lbr_has_tsx = 1; + fallthrough; + case LBR_FORMAT_INFO2: + x86_pmu.lbr_has_info = 1; + break; + + case LBR_FORMAT_TIME: + x86_pmu.lbr_from_flags = 1; + x86_pmu.lbr_to_cycles = 1; + break; + } + + if (x86_pmu.lbr_has_info) { + /* + * Only used in combination with baseline pebs. + */ + static_branch_enable(&x86_lbr_mispred); + static_branch_enable(&x86_lbr_cycles); + } +} + /* * LBR state size is variable based on the max number of registers. * This calculates the expected state size, which should match @@ -1776,6 +1793,12 @@ void __init intel_pmu_arch_lbr_init(void) x86_pmu.lbr_br_type = ecx.split.lbr_br_type; x86_pmu.lbr_nr = lbr_nr; + if (x86_pmu.lbr_mispred) + static_branch_enable(&x86_lbr_mispred); + if (x86_pmu.lbr_timed_lbr) + static_branch_enable(&x86_lbr_cycles); + if (x86_pmu.lbr_br_type) + static_branch_enable(&x86_lbr_type); arch_lbr_xsave = is_arch_lbr_xsave_available(); if (arch_lbr_xsave) { diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index c72e368dd1..e497da9bf4 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -1187,7 +1187,7 @@ static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id * PCI slot and func to indicate the uncore box. */ if (id->driver_data & ~0xffff) { - struct pci_driver *pci_drv = pdev->driver; + struct pci_driver *pci_drv = to_pci_driver(pdev->dev.driver); pmu = uncore_pci_find_dev_pmu(pdev, pci_drv->id_table); if (pmu == NULL) @@ -1762,7 +1762,7 @@ static const struct intel_uncore_init_fun rkl_uncore_init __initconst = { static const struct intel_uncore_init_fun adl_uncore_init __initconst = { .cpu_init = adl_uncore_cpu_init, - .mmio_init = tgl_uncore_mmio_init, + .mmio_init = adl_uncore_mmio_init, }; static const struct intel_uncore_init_fun icx_uncore_init __initconst = { diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index b9687980aa..2adeaf4de4 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -584,10 +584,11 @@ void snb_uncore_cpu_init(void); void nhm_uncore_cpu_init(void); void skl_uncore_cpu_init(void); void icl_uncore_cpu_init(void); -void adl_uncore_cpu_init(void); void tgl_uncore_cpu_init(void); +void adl_uncore_cpu_init(void); void tgl_uncore_mmio_init(void); void tgl_l_uncore_mmio_init(void); +void adl_uncore_mmio_init(void); int snb_pci2phy_map_init(int devid); /* uncore_snbep.c */ diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 3049c646fa..6ddadb482f 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -494,8 +494,8 @@ void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box) writel(0, box->io_addr); } -static void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, - struct perf_event *event) +void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, + struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index 6d735611c2..cfaf558bdb 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -139,6 +139,8 @@ void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, struct perf_event *event); +void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, + struct perf_event *event); void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box); diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index 0f63706cda..f698a55bde 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Nehalem/SandBridge/Haswell/Broadwell/Skylake uncore support */ #include "uncore.h" +#include "uncore_discovery.h" /* Uncore IMC PCI IDs */ #define PCI_DEVICE_ID_INTEL_SNB_IMC 0x0100 @@ -64,6 +65,20 @@ #define PCI_DEVICE_ID_INTEL_RKL_2_IMC 0x4c53 #define PCI_DEVICE_ID_INTEL_ADL_1_IMC 0x4660 #define PCI_DEVICE_ID_INTEL_ADL_2_IMC 0x4641 +#define PCI_DEVICE_ID_INTEL_ADL_3_IMC 0x4601 +#define PCI_DEVICE_ID_INTEL_ADL_4_IMC 0x4602 +#define PCI_DEVICE_ID_INTEL_ADL_5_IMC 0x4609 +#define PCI_DEVICE_ID_INTEL_ADL_6_IMC 0x460a +#define PCI_DEVICE_ID_INTEL_ADL_7_IMC 0x4621 +#define PCI_DEVICE_ID_INTEL_ADL_8_IMC 0x4623 +#define PCI_DEVICE_ID_INTEL_ADL_9_IMC 0x4629 +#define PCI_DEVICE_ID_INTEL_ADL_10_IMC 0x4637 +#define PCI_DEVICE_ID_INTEL_ADL_11_IMC 0x463b +#define PCI_DEVICE_ID_INTEL_ADL_12_IMC 0x4648 +#define PCI_DEVICE_ID_INTEL_ADL_13_IMC 0x4649 +#define PCI_DEVICE_ID_INTEL_ADL_14_IMC 0x4650 +#define PCI_DEVICE_ID_INTEL_ADL_15_IMC 0x4668 +#define PCI_DEVICE_ID_INTEL_ADL_16_IMC 0x4670 /* SNB event control */ #define SNB_UNC_CTL_EV_SEL_MASK 0x000000ff @@ -155,6 +170,7 @@ DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7"); DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); +DEFINE_UNCORE_FORMAT_ATTR(chmask, chmask, "config:8-11"); DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18"); DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23"); DEFINE_UNCORE_FORMAT_ATTR(cmask5, cmask, "config:24-28"); @@ -1334,6 +1350,62 @@ static const struct pci_device_id tgl_uncore_pci_ids[] = { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_2_IMC), .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_3_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_4_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_5_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_6_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_7_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_8_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_9_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_10_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_11_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_12_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_13_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_14_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_15_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, + { /* IMC */ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_16_IMC), + .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), + }, { /* end: all zeroes */ } }; @@ -1390,7 +1462,8 @@ static struct pci_dev *tgl_uncore_get_mc_dev(void) #define TGL_UNCORE_MMIO_IMC_MEM_OFFSET 0x10000 #define TGL_UNCORE_PCI_IMC_MAP_SIZE 0xe000 -static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static void __uncore_imc_init_box(struct intel_uncore_box *box, + unsigned int base_offset) { struct pci_dev *pdev = tgl_uncore_get_mc_dev(); struct intel_uncore_pmu *pmu = box->pmu; @@ -1417,11 +1490,17 @@ static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) addr |= ((resource_size_t)mch_bar << 32); #endif + addr += base_offset; box->io_addr = ioremap(addr, type->mmio_map_size); if (!box->io_addr) pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); } +static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +{ + __uncore_imc_init_box(box, 0); +} + static struct intel_uncore_ops tgl_uncore_imc_freerunning_ops = { .init_box = tgl_uncore_imc_freerunning_init_box, .exit_box = uncore_mmio_exit_box, @@ -1469,3 +1548,136 @@ void tgl_uncore_mmio_init(void) } /* end of Tiger Lake MMIO uncore support */ + +/* Alder Lake MMIO uncore support */ +#define ADL_UNCORE_IMC_BASE 0xd900 +#define ADL_UNCORE_IMC_MAP_SIZE 0x200 +#define ADL_UNCORE_IMC_CTR 0xe8 +#define ADL_UNCORE_IMC_CTRL 0xd0 +#define ADL_UNCORE_IMC_GLOBAL_CTL 0xc0 +#define ADL_UNCORE_IMC_BOX_CTL 0xc4 +#define ADL_UNCORE_IMC_FREERUNNING_BASE 0xd800 +#define ADL_UNCORE_IMC_FREERUNNING_MAP_SIZE 0x100 + +#define ADL_UNCORE_IMC_CTL_FRZ (1 << 0) +#define ADL_UNCORE_IMC_CTL_RST_CTRL (1 << 1) +#define ADL_UNCORE_IMC_CTL_RST_CTRS (1 << 2) +#define ADL_UNCORE_IMC_CTL_INT (ADL_UNCORE_IMC_CTL_RST_CTRL | \ + ADL_UNCORE_IMC_CTL_RST_CTRS) + +static void adl_uncore_imc_init_box(struct intel_uncore_box *box) +{ + __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE); + + /* The global control in MC1 can control both MCs. */ + if (box->io_addr && (box->pmu->pmu_idx == 1)) + writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + ADL_UNCORE_IMC_GLOBAL_CTL); +} + +static void adl_uncore_mmio_disable_box(struct intel_uncore_box *box) +{ + if (!box->io_addr) + return; + + writel(ADL_UNCORE_IMC_CTL_FRZ, box->io_addr + uncore_mmio_box_ctl(box)); +} + +static void adl_uncore_mmio_enable_box(struct intel_uncore_box *box) +{ + if (!box->io_addr) + return; + + writel(0, box->io_addr + uncore_mmio_box_ctl(box)); +} + +static struct intel_uncore_ops adl_uncore_mmio_ops = { + .init_box = adl_uncore_imc_init_box, + .exit_box = uncore_mmio_exit_box, + .disable_box = adl_uncore_mmio_disable_box, + .enable_box = adl_uncore_mmio_enable_box, + .disable_event = intel_generic_uncore_mmio_disable_event, + .enable_event = intel_generic_uncore_mmio_enable_event, + .read_counter = uncore_mmio_read_counter, +}; + +#define ADL_UNC_CTL_CHMASK_MASK 0x00000f00 +#define ADL_UNC_IMC_EVENT_MASK (SNB_UNC_CTL_EV_SEL_MASK | \ + ADL_UNC_CTL_CHMASK_MASK | \ + SNB_UNC_CTL_EDGE_DET) + +static struct attribute *adl_uncore_imc_formats_attr[] = { + &format_attr_event.attr, + &format_attr_chmask.attr, + &format_attr_edge.attr, + NULL, +}; + +static const struct attribute_group adl_uncore_imc_format_group = { + .name = "format", + .attrs = adl_uncore_imc_formats_attr, +}; + +static struct intel_uncore_type adl_uncore_imc = { + .name = "imc", + .num_counters = 5, + .num_boxes = 2, + .perf_ctr_bits = 64, + .perf_ctr = ADL_UNCORE_IMC_CTR, + .event_ctl = ADL_UNCORE_IMC_CTRL, + .event_mask = ADL_UNC_IMC_EVENT_MASK, + .box_ctl = ADL_UNCORE_IMC_BOX_CTL, + .mmio_offset = 0, + .mmio_map_size = ADL_UNCORE_IMC_MAP_SIZE, + .ops = &adl_uncore_mmio_ops, + .format_group = &adl_uncore_imc_format_group, +}; + +enum perf_adl_uncore_imc_freerunning_types { + ADL_MMIO_UNCORE_IMC_DATA_TOTAL, + ADL_MMIO_UNCORE_IMC_DATA_READ, + ADL_MMIO_UNCORE_IMC_DATA_WRITE, + ADL_MMIO_UNCORE_IMC_FREERUNNING_TYPE_MAX +}; + +static struct freerunning_counters adl_uncore_imc_freerunning[] = { + [ADL_MMIO_UNCORE_IMC_DATA_TOTAL] = { 0x40, 0x0, 0x0, 1, 64 }, + [ADL_MMIO_UNCORE_IMC_DATA_READ] = { 0x58, 0x0, 0x0, 1, 64 }, + [ADL_MMIO_UNCORE_IMC_DATA_WRITE] = { 0xA0, 0x0, 0x0, 1, 64 }, +}; + +static void adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +{ + __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE); +} + +static struct intel_uncore_ops adl_uncore_imc_freerunning_ops = { + .init_box = adl_uncore_imc_freerunning_init_box, + .exit_box = uncore_mmio_exit_box, + .read_counter = uncore_mmio_read_counter, + .hw_config = uncore_freerunning_hw_config, +}; + +static struct intel_uncore_type adl_uncore_imc_free_running = { + .name = "imc_free_running", + .num_counters = 3, + .num_boxes = 2, + .num_freerunning_types = ADL_MMIO_UNCORE_IMC_FREERUNNING_TYPE_MAX, + .mmio_map_size = ADL_UNCORE_IMC_FREERUNNING_MAP_SIZE, + .freerunning = adl_uncore_imc_freerunning, + .ops = &adl_uncore_imc_freerunning_ops, + .event_descs = tgl_uncore_imc_events, + .format_group = &tgl_uncore_imc_format_group, +}; + +static struct intel_uncore_type *adl_mmio_uncores[] = { + &adl_uncore_imc, + &adl_uncore_imc_free_running, + NULL +}; + +void adl_uncore_mmio_init(void) +{ + uncore_mmio_uncores = adl_mmio_uncores; +} + +/* end of Alder Lake MMIO uncore support */ diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index e3ac05c97b..150261d929 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -14,6 +14,7 @@ #include +#include #include #include @@ -73,7 +74,7 @@ static inline bool constraint_match(struct event_constraint *c, u64 ecode) #define PERF_X86_EVENT_PEBS_NA_HSW 0x0010 /* haswell style datala, unknown */ #define PERF_X86_EVENT_EXCL 0x0020 /* HT exclusivity on counter */ #define PERF_X86_EVENT_DYNAMIC 0x0040 /* dynamic alloc'd constraint */ -#define PERF_X86_EVENT_RDPMC_ALLOWED 0x0080 /* grant rdpmc permission */ + #define PERF_X86_EVENT_EXCL_ACCT 0x0100 /* accounted EXCL event */ #define PERF_X86_EVENT_AUTO_RELOAD 0x0200 /* use PEBS auto-reload */ #define PERF_X86_EVENT_LARGE_PEBS 0x0400 /* use large PEBS */ @@ -214,7 +215,8 @@ enum { LBR_FORMAT_EIP_FLAGS2 = 0x04, LBR_FORMAT_INFO = 0x05, LBR_FORMAT_TIME = 0x06, - LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_TIME, + LBR_FORMAT_INFO2 = 0x07, + LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_INFO2, }; enum { @@ -726,6 +728,7 @@ struct x86_pmu { void (*enable_all)(int added); void (*enable)(struct perf_event *); void (*disable)(struct perf_event *); + void (*assign)(struct perf_event *event, int idx); void (*add)(struct perf_event *); void (*del)(struct perf_event *); void (*read)(struct perf_event *event); @@ -838,6 +841,11 @@ struct x86_pmu { bool lbr_double_abort; /* duplicated lbr aborts */ bool lbr_pt_coexist; /* (LBR|BTS) may coexist with PT */ + unsigned int lbr_has_info:1; + unsigned int lbr_has_tsx:1; + unsigned int lbr_from_flags:1; + unsigned int lbr_to_cycles:1; + /* * Intel Architectural LBR CPUID Enumeration */ @@ -1240,6 +1248,25 @@ static inline bool intel_pmu_has_bts(struct perf_event *event) return intel_pmu_has_bts_period(event, hwc->sample_period); } +static __always_inline void __intel_pmu_pebs_disable_all(void) +{ + wrmsrl(MSR_IA32_PEBS_ENABLE, 0); +} + +static __always_inline void __intel_pmu_arch_lbr_disable(void) +{ + wrmsrl(MSR_ARCH_LBR_CTL, 0); +} + +static __always_inline void __intel_pmu_lbr_disable(void) +{ + u64 debugctl; + + rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); + debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); + wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); +} + int intel_pmu_save_and_restart(struct perf_event *event); struct event_constraint * @@ -1371,6 +1398,8 @@ void intel_pmu_lbr_init_skl(void); void intel_pmu_lbr_init_knl(void); +void intel_pmu_lbr_init(void); + void intel_pmu_arch_lbr_init(void); void intel_pmu_pebs_data_source_nhm(void); diff --git a/arch/x86/hyperv/Makefile b/arch/x86/hyperv/Makefile index 48e2c51464..5d2de10809 100644 --- a/arch/x86/hyperv/Makefile +++ b/arch/x86/hyperv/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-y := hv_init.o mmu.o nested.o irqdomain.o +obj-y := hv_init.o mmu.o nested.o irqdomain.o ivm.o obj-$(CONFIG_X86_64) += hv_apic.o hv_proc.o ifdef CONFIG_X86_64 diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index b6d48ca5b0..8b392b6b7b 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -28,6 +28,7 @@ #include #include #include +#include int hyperv_init_cpuhp; u64 hv_current_partition_id = ~0ull; @@ -36,12 +37,42 @@ EXPORT_SYMBOL_GPL(hv_current_partition_id); void *hv_hypercall_pg; EXPORT_SYMBOL_GPL(hv_hypercall_pg); +union hv_ghcb * __percpu *hv_ghcb_pg; + /* Storage to save the hypercall page temporarily for hibernation */ static void *hv_hypercall_pg_saved; struct hv_vp_assist_page **hv_vp_assist_page; EXPORT_SYMBOL_GPL(hv_vp_assist_page); +static int hyperv_init_ghcb(void) +{ + u64 ghcb_gpa; + void *ghcb_va; + void **ghcb_base; + + if (!hv_isolation_type_snp()) + return 0; + + if (!hv_ghcb_pg) + return -EINVAL; + + /* + * GHCB page is allocated by paravisor. The address + * returned by MSR_AMD64_SEV_ES_GHCB is above shared + * memory boundary and map it here. + */ + rdmsrl(MSR_AMD64_SEV_ES_GHCB, ghcb_gpa); + ghcb_va = memremap(ghcb_gpa, HV_HYP_PAGE_SIZE, MEMREMAP_WB); + if (!ghcb_va) + return -ENOMEM; + + ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg); + *ghcb_base = ghcb_va; + + return 0; +} + static int hv_cpu_init(unsigned int cpu) { union hv_vp_assist_msr_contents msr = { 0 }; @@ -85,7 +116,7 @@ static int hv_cpu_init(unsigned int cpu) } } - return 0; + return hyperv_init_ghcb(); } static void (*hv_reenlightenment_cb)(void); @@ -183,6 +214,14 @@ static int hv_cpu_die(unsigned int cpu) { struct hv_reenlightenment_control re_ctrl; unsigned int new_cpu; + void **ghcb_va; + + if (hv_ghcb_pg) { + ghcb_va = (void **)this_cpu_ptr(hv_ghcb_pg); + if (*ghcb_va) + memunmap(*ghcb_va); + *ghcb_va = NULL; + } hv_common_cpu_die(cpu); @@ -365,10 +404,16 @@ void __init hyperv_init(void) goto common_free; } + if (hv_isolation_type_snp()) { + hv_ghcb_pg = alloc_percpu(union hv_ghcb *); + if (!hv_ghcb_pg) + goto free_vp_assist_page; + } + cpuhp = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online", hv_cpu_init, hv_cpu_die); if (cpuhp < 0) - goto free_vp_assist_page; + goto free_ghcb_page; /* * Setup the hypercall page and enable hypercalls. @@ -378,14 +423,15 @@ void __init hyperv_init(void) guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0); wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); + /* Hyper-V requires to write guest os id via ghcb in SNP IVM. */ + hv_ghcb_msr_write(HV_X64_MSR_GUEST_OS_ID, guest_id); + hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START, VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_ROX, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, __builtin_return_address(0)); - if (hv_hypercall_pg == NULL) { - wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); - goto remove_cpuhp_state; - } + if (hv_hypercall_pg == NULL) + goto clean_guest_os_id; rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); hypercall_msr.enable = 1; @@ -453,10 +499,25 @@ void __init hyperv_init(void) /* Query the VMs extended capability once, so that it can be cached. */ hv_query_ext_cap(0); + +#ifdef CONFIG_SWIOTLB + /* + * Swiotlb bounce buffer needs to be mapped in extra address + * space. Map function doesn't work in the early place and so + * call swiotlb_update_mem_attributes() here. + */ + if (hv_is_isolation_supported()) + swiotlb_update_mem_attributes(); +#endif + return; -remove_cpuhp_state: +clean_guest_os_id: + wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); + hv_ghcb_msr_write(HV_X64_MSR_GUEST_OS_ID, 0); cpuhp_remove_state(cpuhp); +free_ghcb_page: + free_percpu(hv_ghcb_pg); free_vp_assist_page: kfree(hv_vp_assist_page); hv_vp_assist_page = NULL; @@ -475,6 +536,7 @@ void hyperv_cleanup(void) /* Reset our OS id */ wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); + hv_ghcb_msr_write(HV_X64_MSR_GUEST_OS_ID, 0); /* * Reset hypercall page reference before reset the page, @@ -545,16 +607,3 @@ bool hv_is_hyperv_initialized(void) return hypercall_msr.enable; } EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized); - -enum hv_isolation_type hv_get_isolation_type(void) -{ - if (!(ms_hyperv.priv_high & HV_ISOLATION)) - return HV_ISOLATION_TYPE_NONE; - return FIELD_GET(HV_ISOLATION_TYPE, ms_hyperv.isolation_config_b); -} -EXPORT_SYMBOL_GPL(hv_get_isolation_type); - -bool hv_is_isolation_supported(void) -{ - return hv_get_isolation_type() != HV_ISOLATION_TYPE_NONE; -} diff --git a/arch/x86/hyperv/irqdomain.c b/arch/x86/hyperv/irqdomain.c index 514fc64e23..7e0f6bedc2 100644 --- a/arch/x86/hyperv/irqdomain.c +++ b/arch/x86/hyperv/irqdomain.c @@ -253,64 +253,43 @@ static int hv_unmap_msi_interrupt(struct pci_dev *dev, struct hv_interrupt_entry return hv_unmap_interrupt(hv_build_pci_dev_id(dev).as_uint64, old_entry); } -static void hv_teardown_msi_irq_common(struct pci_dev *dev, struct msi_desc *msidesc, int irq) +static void hv_teardown_msi_irq(struct pci_dev *dev, struct irq_data *irqd) { - u64 status; struct hv_interrupt_entry old_entry; - struct irq_desc *desc; - struct irq_data *data; struct msi_msg msg; + u64 status; - desc = irq_to_desc(irq); - if (!desc) { - pr_debug("%s: no irq desc\n", __func__); - return; - } - - data = &desc->irq_data; - if (!data) { - pr_debug("%s: no irq data\n", __func__); - return; - } - - if (!data->chip_data) { + if (!irqd->chip_data) { pr_debug("%s: no chip data\n!", __func__); return; } - old_entry = *(struct hv_interrupt_entry *)data->chip_data; + old_entry = *(struct hv_interrupt_entry *)irqd->chip_data; entry_to_msi_msg(&old_entry, &msg); - kfree(data->chip_data); - data->chip_data = NULL; + kfree(irqd->chip_data); + irqd->chip_data = NULL; status = hv_unmap_msi_interrupt(dev, &old_entry); - if (status != HV_STATUS_SUCCESS) { + if (status != HV_STATUS_SUCCESS) pr_err("%s: hypercall failed, status %lld\n", __func__, status); - return; - } } -static void hv_msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) +static void hv_msi_free_irq(struct irq_domain *domain, + struct msi_domain_info *info, unsigned int virq) { - int i; - struct msi_desc *entry; - struct pci_dev *pdev; + struct irq_data *irqd = irq_get_irq_data(virq); + struct msi_desc *desc; - if (WARN_ON_ONCE(!dev_is_pci(dev))) + if (!irqd) return; - pdev = to_pci_dev(dev); + desc = irq_data_get_msi_desc(irqd); + if (!desc || !desc->irq || WARN_ON_ONCE(!dev_is_pci(desc->dev))) + return; - for_each_pci_msi_entry(entry, pdev) { - if (entry->irq) { - for (i = 0; i < entry->nvec_used; i++) { - hv_teardown_msi_irq_common(pdev, entry, entry->irq + i); - irq_domain_free_irqs(entry->irq + i, 1); - } - } - } + hv_teardown_msi_irq(to_pci_dev(desc->dev), irqd); } /* @@ -329,7 +308,7 @@ static struct irq_chip hv_pci_msi_controller = { }; static struct msi_domain_ops pci_msi_domain_ops = { - .domain_free_irqs = hv_msi_domain_free_irqs, + .msi_free = hv_msi_free_irq, .msi_prepare = pci_msi_prepare, }; diff --git a/arch/x86/ia32/audit.c b/arch/x86/ia32/audit.c index 6efe6cb376..59e19549e7 100644 --- a/arch/x86/ia32/audit.c +++ b/arch/x86/ia32/audit.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include @@ -31,15 +32,17 @@ int ia32_classify_syscall(unsigned syscall) { switch (syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: case __NR_execveat: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c index 5e3d9b7fd5..c9c3859322 100644 --- a/arch/x86/ia32/ia32_signal.c +++ b/arch/x86/ia32/ia32_signal.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -57,8 +56,8 @@ static inline void reload_segments(struct sigcontext_32 *sc) /* * Do a signal return; undo the signal stack. */ -static int ia32_restore_sigcontext(struct pt_regs *regs, - struct sigcontext_32 __user *usc) +static bool ia32_restore_sigcontext(struct pt_regs *regs, + struct sigcontext_32 __user *usc) { struct sigcontext_32 sc; @@ -66,7 +65,7 @@ static int ia32_restore_sigcontext(struct pt_regs *regs, current->restart_block.fn = do_no_restart_syscall; if (unlikely(copy_from_user(&sc, usc, sizeof(sc)))) - return -EFAULT; + return false; /* Get only the ia32 registers. */ regs->bx = sc.bx; @@ -111,7 +110,7 @@ COMPAT_SYSCALL_DEFINE0(sigreturn) set_current_blocked(&set); - if (ia32_restore_sigcontext(regs, &frame->sc)) + if (!ia32_restore_sigcontext(regs, &frame->sc)) goto badframe; return regs->ax; @@ -135,7 +134,7 @@ COMPAT_SYSCALL_DEFINE0(rt_sigreturn) set_current_blocked(&set); - if (ia32_restore_sigcontext(regs, &frame->uc.uc_mcontext)) + if (!ia32_restore_sigcontext(regs, &frame->uc.uc_mcontext)) goto badframe; if (compat_restore_altstack(&frame->uc.uc_stack)) @@ -220,8 +219,8 @@ static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, sp = fpu__alloc_mathframe(sp, 1, &fx_aligned, &math_size); *fpstate = (struct _fpstate_32 __user *) sp; - if (copy_fpstate_to_sigframe(*fpstate, (void __user *)fx_aligned, - math_size) < 0) + if (!copy_fpstate_to_sigframe(*fpstate, (void __user *)fx_aligned, + math_size)) return (void __user *) -1L; sp -= frame_size; diff --git a/arch/x86/include/asm/GEN-for-each-reg.h b/arch/x86/include/asm/GEN-for-each-reg.h index 1b07fb102c..07949102a0 100644 --- a/arch/x86/include/asm/GEN-for-each-reg.h +++ b/arch/x86/include/asm/GEN-for-each-reg.h @@ -1,11 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * These are in machine order; things rely on that. + */ #ifdef CONFIG_64BIT GEN(rax) -GEN(rbx) GEN(rcx) GEN(rdx) +GEN(rbx) +GEN(rsp) +GEN(rbp) GEN(rsi) GEN(rdi) -GEN(rbp) GEN(r8) GEN(r9) GEN(r10) @@ -16,10 +21,11 @@ GEN(r14) GEN(r15) #else GEN(eax) -GEN(ebx) GEN(ecx) GEN(edx) +GEN(ebx) +GEN(esp) +GEN(ebp) GEN(esi) GEN(edi) -GEN(ebp) #endif diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index a3c2315aca..58eee64028 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -75,6 +75,7 @@ extern int alternatives_patched; extern void alternative_instructions(void); extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); +extern void apply_retpolines(s32 *start, s32 *end); struct module; diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd_nb.h index 455066a06f..00d1a400b7 100644 --- a/arch/x86/include/asm/amd_nb.h +++ b/arch/x86/include/asm/amd_nb.h @@ -24,7 +24,6 @@ extern int amd_set_subcaches(int, unsigned long); extern int amd_smn_read(u16 node, u32 address, u32 *value); extern int amd_smn_write(u16 node, u32 address, u32 value); -extern int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo); struct amd_l3_cache { unsigned indices; diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h index 4cb726c71e..8f80de627c 100644 --- a/arch/x86/include/asm/asm-prototypes.h +++ b/arch/x86/include/asm/asm-prototypes.h @@ -17,21 +17,3 @@ extern void cmpxchg8b_emu(void); #endif -#ifdef CONFIG_RETPOLINE - -#undef GEN -#define GEN(reg) \ - extern asmlinkage void __x86_indirect_thunk_ ## reg (void); -#include - -#undef GEN -#define GEN(reg) \ - extern asmlinkage void __x86_indirect_alt_call_ ## reg (void); -#include - -#undef GEN -#define GEN(reg) \ - extern asmlinkage void __x86_indirect_alt_jmp_ ## reg (void); -#include - -#endif /* CONFIG_RETPOLINE */ diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index 3ad3da9a7d..c878fed305 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -6,11 +6,13 @@ # define __ASM_FORM(x, ...) x,## __VA_ARGS__ # define __ASM_FORM_RAW(x, ...) x,## __VA_ARGS__ # define __ASM_FORM_COMMA(x, ...) x,## __VA_ARGS__, +# define __ASM_REGPFX % #else #include # define __ASM_FORM(x, ...) " " __stringify(x,##__VA_ARGS__) " " # define __ASM_FORM_RAW(x, ...) __stringify(x,##__VA_ARGS__) # define __ASM_FORM_COMMA(x, ...) " " __stringify(x,##__VA_ARGS__) "," +# define __ASM_REGPFX %% #endif #define _ASM_BYTES(x, ...) __ASM_FORM(.byte x,##__VA_ARGS__ ;) @@ -49,6 +51,9 @@ #define _ASM_SI __ASM_REG(si) #define _ASM_DI __ASM_REG(di) +/* Adds a (%rip) suffix on 64 bits only; for immediate memory references */ +#define _ASM_RIP(x) __ASM_SEL_RAW(x, x (__ASM_REGPFX rip)) + #ifndef __x86_64__ /* 32 bit */ @@ -122,28 +127,19 @@ #ifdef __KERNEL__ +# include + /* Exception table entry */ #ifdef __ASSEMBLY__ -# define _ASM_EXTABLE_HANDLE(from, to, handler) \ + +# define _ASM_EXTABLE_TYPE(from, to, type) \ .pushsection "__ex_table","a" ; \ .balign 4 ; \ .long (from) - . ; \ .long (to) - . ; \ - .long (handler) - . ; \ + .long type ; \ .popsection -# define _ASM_EXTABLE(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_default) - -# define _ASM_EXTABLE_UA(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_uaccess) - -# define _ASM_EXTABLE_CPY(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_copy) - -# define _ASM_EXTABLE_FAULT(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_fault) - # ifdef CONFIG_KPROBES # define _ASM_NOKPROBE(entry) \ .pushsection "_kprobe_blacklist","aw" ; \ @@ -155,26 +151,51 @@ # endif #else /* ! __ASSEMBLY__ */ -# define _EXPAND_EXTABLE_HANDLE(x) #x -# define _ASM_EXTABLE_HANDLE(from, to, handler) \ + +# define DEFINE_EXTABLE_TYPE_REG \ + ".macro extable_type_reg type:req reg:req\n" \ + ".set found, 0\n" \ + ".set regnr, 0\n" \ + ".irp rs,rax,rcx,rdx,rbx,rsp,rbp,rsi,rdi,r8,r9,r10,r11,r12,r13,r14,r15\n" \ + ".ifc \\reg, %%\\rs\n" \ + ".set found, found+1\n" \ + ".long \\type + (regnr << 8)\n" \ + ".endif\n" \ + ".set regnr, regnr+1\n" \ + ".endr\n" \ + ".set regnr, 0\n" \ + ".irp rs,eax,ecx,edx,ebx,esp,ebp,esi,edi,r8d,r9d,r10d,r11d,r12d,r13d,r14d,r15d\n" \ + ".ifc \\reg, %%\\rs\n" \ + ".set found, found+1\n" \ + ".long \\type + (regnr << 8)\n" \ + ".endif\n" \ + ".set regnr, regnr+1\n" \ + ".endr\n" \ + ".if (found != 1)\n" \ + ".error \"extable_type_reg: bad register argument\"\n" \ + ".endif\n" \ + ".endm\n" + +# define UNDEFINE_EXTABLE_TYPE_REG \ + ".purgem extable_type_reg\n" + +# define _ASM_EXTABLE_TYPE(from, to, type) \ " .pushsection \"__ex_table\",\"a\"\n" \ " .balign 4\n" \ " .long (" #from ") - .\n" \ " .long (" #to ") - .\n" \ - " .long (" _EXPAND_EXTABLE_HANDLE(handler) ") - .\n" \ + " .long " __stringify(type) " \n" \ " .popsection\n" -# define _ASM_EXTABLE(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_default) - -# define _ASM_EXTABLE_UA(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_uaccess) - -# define _ASM_EXTABLE_CPY(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_copy) - -# define _ASM_EXTABLE_FAULT(from, to) \ - _ASM_EXTABLE_HANDLE(from, to, ex_handler_fault) +# define _ASM_EXTABLE_TYPE_REG(from, to, type, reg) \ + " .pushsection \"__ex_table\",\"a\"\n" \ + " .balign 4\n" \ + " .long (" #from ") - .\n" \ + " .long (" #to ") - .\n" \ + DEFINE_EXTABLE_TYPE_REG \ + "extable_type_reg reg=" __stringify(reg) ", type=" __stringify(type) " \n"\ + UNDEFINE_EXTABLE_TYPE_REG \ + " .popsection\n" /* For C file, we already have NOKPROBE_SYMBOL macro */ @@ -188,6 +209,17 @@ register unsigned long current_stack_pointer asm(_ASM_SP); #define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer) #endif /* __ASSEMBLY__ */ -#endif /* __KERNEL__ */ +#define _ASM_EXTABLE(from, to) \ + _ASM_EXTABLE_TYPE(from, to, EX_TYPE_DEFAULT) +#define _ASM_EXTABLE_UA(from, to) \ + _ASM_EXTABLE_TYPE(from, to, EX_TYPE_UACCESS) + +#define _ASM_EXTABLE_CPY(from, to) \ + _ASM_EXTABLE_TYPE(from, to, EX_TYPE_COPY) + +#define _ASM_EXTABLE_FAULT(from, to) \ + _ASM_EXTABLE_TYPE(from, to, EX_TYPE_FAULT) + +#endif /* __KERNEL__ */ #endif /* _ASM_X86_ASM_H */ diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h index 3ba772a69c..35389b2af8 100644 --- a/arch/x86/include/asm/barrier.h +++ b/arch/x86/include/asm/barrier.h @@ -19,9 +19,9 @@ #define wmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "sfence", \ X86_FEATURE_XMM2) ::: "memory", "cc") #else -#define mb() asm volatile("mfence":::"memory") -#define rmb() asm volatile("lfence":::"memory") -#define wmb() asm volatile("sfence" ::: "memory") +#define __mb() asm volatile("mfence":::"memory") +#define __rmb() asm volatile("lfence":::"memory") +#define __wmb() asm volatile("sfence" ::: "memory") #endif /** @@ -51,8 +51,8 @@ static inline unsigned long array_index_mask_nospec(unsigned long index, /* Prevent speculative execution past this barrier. */ #define barrier_nospec() alternative("", "lfence", X86_FEATURE_LFENCE_RDTSC) -#define dma_rmb() barrier() -#define dma_wmb() barrier() +#define __dma_rmb() barrier() +#define __dma_wmb() barrier() #define __smp_mb() asm volatile("lock; addl $0,-4(%%" _ASM_SP ")" ::: "memory", "cc") diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index 0367efdc5b..a288ecd230 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -380,8 +380,6 @@ static __always_inline int fls64(__u64 x) #include #endif -#include - #include #include diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 16a51e7288..1261842d00 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -173,20 +173,25 @@ extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit); * means that the boot_cpu_has() variant is already fast enough for the * majority of cases and you should stick to using it as it is generally * only two instructions: a RIP-relative MOV and a TEST. + * + * Do not use an "m" constraint for [cap_byte] here: gcc doesn't know + * that this is only used on a fallback path and will sometimes cause + * it to manifest the address of boot_cpu_data in a register, fouling + * the mainline (post-initialization) code. */ static __always_inline bool _static_cpu_has(u16 bit) { asm_volatile_goto( ALTERNATIVE_TERNARY("jmp 6f", %P[feature], "", "jmp %l[t_no]") - ".section .altinstr_aux,\"ax\"\n" + ".pushsection .altinstr_aux,\"ax\"\n" "6:\n" - " testb %[bitnum],%[cap_byte]\n" + " testb %[bitnum]," _ASM_RIP(%P[cap_byte]) "\n" " jnz %l[t_yes]\n" " jmp %l[t_no]\n" - ".previous\n" + ".popsection\n" : : [feature] "i" (bit), [bitnum] "i" (1 << (bit & 7)), - [cap_byte] "m" (((const char *)boot_cpu_data.x86_capability)[bit >> 3]) + [cap_byte] "i" (&((const char *)boot_cpu_data.x86_capability)[bit >> 3]) : : t_yes, t_no); t_yes: return true; diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 4423759f61..65d147974f 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -277,6 +277,7 @@ #define X86_FEATURE_XSAVEC (10*32+ 1) /* XSAVEC instruction */ #define X86_FEATURE_XGETBV1 (10*32+ 2) /* XGETBV with ECX = 1 instruction */ #define X86_FEATURE_XSAVES (10*32+ 3) /* XSAVES/XRSTORS instructions */ +#define X86_FEATURE_XFD (10*32+ 4) /* "" eXtended Feature Disabling */ /* * Extended auxiliary flags: Linux defined - for features scattered in various @@ -298,6 +299,9 @@ /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX_VNNI (12*32+ 4) /* AVX VNNI instructions */ #define X86_FEATURE_AVX512_BF16 (12*32+ 5) /* AVX512 BFLOAT16 instructions */ +#define X86_FEATURE_AMX_BF16 (18*32+22) /* AMX bf16 Support */ +#define X86_FEATURE_AMX_TILE (18*32+24) /* AMX tile Support */ +#define X86_FEATURE_AMX_INT8 (18*32+25) /* AMX int8 Support */ /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */ #define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */ @@ -313,6 +317,7 @@ #define X86_FEATURE_AMD_SSBD (13*32+24) /* "" Speculative Store Bypass Disable */ #define X86_FEATURE_VIRT_SSBD (13*32+25) /* Virtualized Speculative Store Bypass Disable */ #define X86_FEATURE_AMD_SSB_NO (13*32+26) /* "" Speculative Store Bypass is fixed in hardware. */ +#define X86_FEATURE_CPPC (13*32+27) /* Collaborative Processor Performance Control */ /* Thermal and Power Management Leaf, CPUID level 0x00000006 (EAX), word 14 */ #define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */ diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index 63158fd558..03cb127750 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -46,13 +46,14 @@ extern unsigned long efi_mixed_mode_stack_pa; #define __efi_nargs(...) __efi_nargs_(__VA_ARGS__) #define __efi_nargs_(...) __efi_nargs__(0, ##__VA_ARGS__, \ + __efi_arg_sentinel(9), __efi_arg_sentinel(8), \ __efi_arg_sentinel(7), __efi_arg_sentinel(6), \ __efi_arg_sentinel(5), __efi_arg_sentinel(4), \ __efi_arg_sentinel(3), __efi_arg_sentinel(2), \ __efi_arg_sentinel(1), __efi_arg_sentinel(0)) -#define __efi_nargs__(_0, _1, _2, _3, _4, _5, _6, _7, n, ...) \ +#define __efi_nargs__(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, n, ...) \ __take_second_arg(n, \ - ({ BUILD_BUG_ON_MSG(1, "__efi_nargs limit exceeded"); 8; })) + ({ BUILD_BUG_ON_MSG(1, "__efi_nargs limit exceeded"); 10; })) #define __efi_arg_sentinel(n) , n /* @@ -176,8 +177,9 @@ extern u64 efi_setup; extern efi_status_t __efi64_thunk(u32, ...); #define efi64_thunk(...) ({ \ - __efi_nargs_check(efi64_thunk, 6, __VA_ARGS__); \ - __efi64_thunk(__VA_ARGS__); \ + u64 __pad[3]; /* must have space for 3 args on the stack */ \ + __efi_nargs_check(efi64_thunk, 9, __VA_ARGS__); \ + __efi64_thunk(__VA_ARGS__, __pad); \ }) static inline bool efi_is_mixed(void) @@ -306,6 +308,10 @@ static inline u32 efi64_convert_status(efi_status_t status) #define __efi64_argmap_query_mode(gop, mode, size, info) \ ((gop), (mode), efi64_zero_upper(size), efi64_zero_upper(info)) +/* TCG2 protocol */ +#define __efi64_argmap_hash_log_extend_event(prot, fl, addr, size, ev) \ + ((prot), (fl), 0ULL, (u64)(addr), 0ULL, (u64)(size), 0ULL, ev) + /* * The macros below handle the plumbing for the argument mapping. To add a * mapping for a specific EFI method, simply define a macro diff --git a/arch/x86/include/asm/extable.h b/arch/x86/include/asm/extable.h index 1f0cbc5293..155c991ba9 100644 --- a/arch/x86/include/asm/extable.h +++ b/arch/x86/include/asm/extable.h @@ -1,12 +1,18 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_X86_EXTABLE_H #define _ASM_X86_EXTABLE_H + +#include + /* - * The exception table consists of triples of addresses relative to the - * exception table entry itself. The first address is of an instruction - * that is allowed to fault, the second is the target at which the program - * should continue. The third is a handler function to deal with the fault - * caused by the instruction in the first field. + * The exception table consists of two addresses relative to the + * exception table entry itself and a type selector field. + * + * The first address is of an instruction that is allowed to fault, the + * second is the target at which the program should continue. + * + * The type entry is used by fixup_exception() to select the handler to + * deal with the fault caused by the instruction in the first field. * * All the routines below use bits of fixup code that are out of line * with the main instruction path. This means when everything is well, @@ -15,7 +21,7 @@ */ struct exception_table_entry { - int insn, fixup, handler; + int insn, fixup, data; }; struct pt_regs; @@ -25,21 +31,27 @@ struct pt_regs; do { \ (a)->fixup = (b)->fixup + (delta); \ (b)->fixup = (tmp).fixup - (delta); \ - (a)->handler = (b)->handler + (delta); \ - (b)->handler = (tmp).handler - (delta); \ + (a)->data = (b)->data; \ + (b)->data = (tmp).data; \ } while (0) -enum handler_type { - EX_HANDLER_NONE, - EX_HANDLER_FAULT, - EX_HANDLER_UACCESS, - EX_HANDLER_OTHER -}; - extern int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, unsigned long fault_addr); extern int fixup_bug(struct pt_regs *regs, int trapnr); -extern enum handler_type ex_get_fault_handler_type(unsigned long ip); +extern int ex_get_fixup_type(unsigned long ip); extern void early_fixup_exception(struct pt_regs *regs, int trapnr); +#ifdef CONFIG_X86_MCE +extern void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr); +#else +static inline void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr) { } +#endif + +#if defined(CONFIG_BPF_JIT) && defined(CONFIG_X86_64) +bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs); +#else +static inline bool ex_handler_bpf(const struct exception_table_entry *x, + struct pt_regs *regs) { return false; } +#endif + #endif diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h index 23bef08a83..c83b302035 100644 --- a/arch/x86/include/asm/fpu/api.h +++ b/arch/x86/include/asm/fpu/api.h @@ -12,6 +12,8 @@ #define _ASM_X86_FPU_API_H #include +#include + /* * Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It * disables preemption so be careful if you intend to use it for long periods @@ -48,9 +50,9 @@ static inline void kernel_fpu_begin(void) } /* - * Use fpregs_lock() while editing CPU's FPU registers or fpu->state. + * Use fpregs_lock() while editing CPU's FPU registers or fpu->fpstate. * A context switch will (and softirq might) save CPU's FPU registers to - * fpu->state and set TIF_NEED_FPU_LOAD leaving CPU's FPU registers in + * fpu->fpstate.regs and set TIF_NEED_FPU_LOAD leaving CPU's FPU registers in * a random state. * * local_bh_disable() protects against both preemption and soft interrupts @@ -100,12 +102,67 @@ extern void switch_fpu_return(void); */ extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name); -/* - * Tasks that are not using SVA have mm->pasid set to zero to note that they - * will not have the valid bit set in MSR_IA32_PASID while they are running. - */ -#define PASID_DISABLED 0 +/* Trap handling */ +extern int fpu__exception_code(struct fpu *fpu, int trap_nr); +extern void fpu_sync_fpstate(struct fpu *fpu); +extern void fpu_reset_from_exception_fixup(void); -static inline void update_pasid(void) { } +/* Boot, hotplug and resume */ +extern void fpu__init_cpu(void); +extern void fpu__init_system(struct cpuinfo_x86 *c); +extern void fpu__init_check_bugs(void); +extern void fpu__resume_cpu(void); + +#ifdef CONFIG_MATH_EMULATION +extern void fpstate_init_soft(struct swregs_state *soft); +#else +static inline void fpstate_init_soft(struct swregs_state *soft) {} +#endif + +/* State tracking */ +DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); + +/* Process cleanup */ +#ifdef CONFIG_X86_64 +extern void fpstate_free(struct fpu *fpu); +#else +static inline void fpstate_free(struct fpu *fpu) { } +#endif + +/* fpstate-related functions which are exported to KVM */ +extern void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature); + +extern u64 xstate_get_guest_group_perm(void); + +/* KVM specific functions */ +extern bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu); +extern void fpu_free_guest_fpstate(struct fpu_guest *gfpu); +extern int fpu_swap_kvm_fpstate(struct fpu_guest *gfpu, bool enter_guest); +extern int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures); + +#ifdef CONFIG_X86_64 +extern void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd); +extern void fpu_sync_guest_vmexit_xfd_state(void); +#else +static inline void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) { } +static inline void fpu_sync_guest_vmexit_xfd_state(void) { } +#endif + +extern void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf, unsigned int size, u32 pkru); +extern int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, u64 xcr0, u32 *vpkru); + +static inline void fpstate_set_confidential(struct fpu_guest *gfpu) +{ + gfpu->fpstate->is_confidential = true; +} + +static inline bool fpstate_is_confidential(struct fpu_guest *gfpu) +{ + return gfpu->fpstate->is_confidential; +} + +/* prctl */ +struct task_struct; +extern long fpu_xstate_prctl(struct task_struct *tsk, int option, unsigned long arg2); #endif /* _ASM_X86_FPU_API_H */ diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h index 5a18694a89..e69de29bb2 100644 --- a/arch/x86/include/asm/fpu/internal.h +++ b/arch/x86/include/asm/fpu/internal.h @@ -1,540 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 1994 Linus Torvalds - * - * Pentium III FXSR, SSE support - * General FPU state handling cleanups - * Gareth Hughes , May 2000 - * x86-64 work by Andi Kleen 2002 - */ - -#ifndef _ASM_X86_FPU_INTERNAL_H -#define _ASM_X86_FPU_INTERNAL_H - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -/* - * High level FPU state handling functions: - */ -extern int fpu__restore_sig(void __user *buf, int ia32_frame); -extern void fpu__drop(struct fpu *fpu); -extern void fpu__clear_user_states(struct fpu *fpu); -extern int fpu__exception_code(struct fpu *fpu, int trap_nr); - -extern void fpu_sync_fpstate(struct fpu *fpu); - -/* Clone and exit operations */ -extern int fpu_clone(struct task_struct *dst); -extern void fpu_flush_thread(void); - -/* - * Boot time FPU initialization functions: - */ -extern void fpu__init_cpu(void); -extern void fpu__init_system_xstate(void); -extern void fpu__init_cpu_xstate(void); -extern void fpu__init_system(struct cpuinfo_x86 *c); -extern void fpu__init_check_bugs(void); -extern void fpu__resume_cpu(void); - -/* - * Debugging facility: - */ -#ifdef CONFIG_X86_DEBUG_FPU -# define WARN_ON_FPU(x) WARN_ON_ONCE(x) -#else -# define WARN_ON_FPU(x) ({ (void)(x); 0; }) -#endif - -/* - * FPU related CPU feature flag helper routines: - */ -static __always_inline __pure bool use_xsaveopt(void) -{ - return static_cpu_has(X86_FEATURE_XSAVEOPT); -} - -static __always_inline __pure bool use_xsave(void) -{ - return static_cpu_has(X86_FEATURE_XSAVE); -} - -static __always_inline __pure bool use_fxsr(void) -{ - return static_cpu_has(X86_FEATURE_FXSR); -} - -/* - * fpstate handling functions: - */ - -extern union fpregs_state init_fpstate; - -extern void fpstate_init(union fpregs_state *state); -#ifdef CONFIG_MATH_EMULATION -extern void fpstate_init_soft(struct swregs_state *soft); -#else -static inline void fpstate_init_soft(struct swregs_state *soft) {} -#endif -extern void save_fpregs_to_fpstate(struct fpu *fpu); - -/* Returns 0 or the negated trap number, which results in -EFAULT for #PF */ -#define user_insn(insn, output, input...) \ -({ \ - int err; \ - \ - might_fault(); \ - \ - asm volatile(ASM_STAC "\n" \ - "1: " #insn "\n" \ - "2: " ASM_CLAC "\n" \ - ".section .fixup,\"ax\"\n" \ - "3: negl %%eax\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE_FAULT(1b, 3b) \ - : [err] "=a" (err), output \ - : "0"(0), input); \ - err; \ -}) - -#define kernel_insn_err(insn, output, input...) \ -({ \ - int err; \ - asm volatile("1:" #insn "\n\t" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl $-1,%[err]\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : [err] "=r" (err), output \ - : "0"(0), input); \ - err; \ -}) - -#define kernel_insn(insn, output, input...) \ - asm volatile("1:" #insn "\n\t" \ - "2:\n" \ - _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_fprestore) \ - : output : input) - -static inline int fnsave_to_user_sigframe(struct fregs_state __user *fx) -{ - return user_insn(fnsave %[fx]; fwait, [fx] "=m" (*fx), "m" (*fx)); -} - -static inline int fxsave_to_user_sigframe(struct fxregs_state __user *fx) -{ - if (IS_ENABLED(CONFIG_X86_32)) - return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx)); - else - return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx)); - -} - -static inline void fxrstor(struct fxregs_state *fx) -{ - if (IS_ENABLED(CONFIG_X86_32)) - kernel_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); - else - kernel_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int fxrstor_safe(struct fxregs_state *fx) -{ - if (IS_ENABLED(CONFIG_X86_32)) - return kernel_insn_err(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); - else - return kernel_insn_err(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int fxrstor_from_user_sigframe(struct fxregs_state __user *fx) -{ - if (IS_ENABLED(CONFIG_X86_32)) - return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); - else - return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline void frstor(struct fregs_state *fx) -{ - kernel_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int frstor_safe(struct fregs_state *fx) -{ - return kernel_insn_err(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int frstor_from_user_sigframe(struct fregs_state __user *fx) -{ - return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline void fxsave(struct fxregs_state *fx) -{ - if (IS_ENABLED(CONFIG_X86_32)) - asm volatile( "fxsave %[fx]" : [fx] "=m" (*fx)); - else - asm volatile("fxsaveq %[fx]" : [fx] "=m" (*fx)); -} - -/* These macros all use (%edi)/(%rdi) as the single memory argument. */ -#define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27" -#define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37" -#define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f" -#define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f" -#define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f" - -/* - * After this @err contains 0 on success or the negated trap number when - * the operation raises an exception. For faults this results in -EFAULT. - */ -#define XSTATE_OP(op, st, lmask, hmask, err) \ - asm volatile("1:" op "\n\t" \ - "xor %[err], %[err]\n" \ - "2:\n\t" \ - ".pushsection .fixup,\"ax\"\n\t" \ - "3: negl %%eax\n\t" \ - "jmp 2b\n\t" \ - ".popsection\n\t" \ - _ASM_EXTABLE_FAULT(1b, 3b) \ - : [err] "=a" (err) \ - : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ - : "memory") - -/* - * If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact - * format and supervisor states in addition to modified optimization in - * XSAVEOPT. - * - * Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT - * supports modified optimization which is not supported by XSAVE. - * - * We use XSAVE as a fallback. - * - * The 661 label is defined in the ALTERNATIVE* macros as the address of the - * original instruction which gets replaced. We need to use it here as the - * address of the instruction where we might get an exception at. - */ -#define XSTATE_XSAVE(st, lmask, hmask, err) \ - asm volatile(ALTERNATIVE_2(XSAVE, \ - XSAVEOPT, X86_FEATURE_XSAVEOPT, \ - XSAVES, X86_FEATURE_XSAVES) \ - "\n" \ - "xor %[err], %[err]\n" \ - "3:\n" \ - ".pushsection .fixup,\"ax\"\n" \ - "4: movl $-2, %[err]\n" \ - "jmp 3b\n" \ - ".popsection\n" \ - _ASM_EXTABLE(661b, 4b) \ - : [err] "=r" (err) \ - : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ - : "memory") - -/* - * Use XRSTORS to restore context if it is enabled. XRSTORS supports compact - * XSAVE area format. - */ -#define XSTATE_XRESTORE(st, lmask, hmask) \ - asm volatile(ALTERNATIVE(XRSTOR, \ - XRSTORS, X86_FEATURE_XSAVES) \ - "\n" \ - "3:\n" \ - _ASM_EXTABLE_HANDLE(661b, 3b, ex_handler_fprestore)\ - : \ - : "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \ - : "memory") - -/* - * This function is called only during boot time when x86 caps are not set - * up and alternative can not be used yet. - */ -static inline void os_xrstor_booting(struct xregs_state *xstate) -{ - u64 mask = xfeatures_mask_fpstate(); - u32 lmask = mask; - u32 hmask = mask >> 32; - int err; - - WARN_ON(system_state != SYSTEM_BOOTING); - - if (boot_cpu_has(X86_FEATURE_XSAVES)) - XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); - else - XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); - - /* - * We should never fault when copying from a kernel buffer, and the FPU - * state we set at boot time should be valid. - */ - WARN_ON_FPU(err); -} - -/* - * Save processor xstate to xsave area. - * - * Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features - * and command line options. The choice is permanent until the next reboot. - */ -static inline void os_xsave(struct xregs_state *xstate) -{ - u64 mask = xfeatures_mask_all; - u32 lmask = mask; - u32 hmask = mask >> 32; - int err; - - WARN_ON_FPU(!alternatives_patched); - - XSTATE_XSAVE(xstate, lmask, hmask, err); - - /* We should never fault when copying to a kernel buffer: */ - WARN_ON_FPU(err); -} - -/* - * Restore processor xstate from xsave area. - * - * Uses XRSTORS when XSAVES is used, XRSTOR otherwise. - */ -static inline void os_xrstor(struct xregs_state *xstate, u64 mask) -{ - u32 lmask = mask; - u32 hmask = mask >> 32; - - XSTATE_XRESTORE(xstate, lmask, hmask); -} - -/* - * Save xstate to user space xsave area. - * - * We don't use modified optimization because xrstor/xrstors might track - * a different application. - * - * We don't use compacted format xsave area for - * backward compatibility for old applications which don't understand - * compacted format of xsave area. - */ -static inline int xsave_to_user_sigframe(struct xregs_state __user *buf) -{ - /* - * Include the features which are not xsaved/rstored by the kernel - * internally, e.g. PKRU. That's user space ABI and also required - * to allow the signal handler to modify PKRU. - */ - u64 mask = xfeatures_mask_uabi(); - u32 lmask = mask; - u32 hmask = mask >> 32; - int err; - - /* - * Clear the xsave header first, so that reserved fields are - * initialized to zero. - */ - err = __clear_user(&buf->header, sizeof(buf->header)); - if (unlikely(err)) - return -EFAULT; - - stac(); - XSTATE_OP(XSAVE, buf, lmask, hmask, err); - clac(); - - return err; -} - -/* - * Restore xstate from user space xsave area. - */ -static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 mask) -{ - struct xregs_state *xstate = ((__force struct xregs_state *)buf); - u32 lmask = mask; - u32 hmask = mask >> 32; - int err; - - stac(); - XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); - clac(); - - return err; -} - -/* - * Restore xstate from kernel space xsave area, return an error code instead of - * an exception. - */ -static inline int os_xrstor_safe(struct xregs_state *xstate, u64 mask) -{ - u32 lmask = mask; - u32 hmask = mask >> 32; - int err; - - if (cpu_feature_enabled(X86_FEATURE_XSAVES)) - XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); - else - XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); - - return err; -} - -extern void __restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask); - -static inline void restore_fpregs_from_fpstate(union fpregs_state *fpstate) -{ - __restore_fpregs_from_fpstate(fpstate, xfeatures_mask_fpstate()); -} - -extern int copy_fpstate_to_sigframe(void __user *buf, void __user *fp, int size); - -/* - * FPU context switch related helper methods: - */ - -DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); - -/* - * The in-register FPU state for an FPU context on a CPU is assumed to be - * valid if the fpu->last_cpu matches the CPU, and the fpu_fpregs_owner_ctx - * matches the FPU. - * - * If the FPU register state is valid, the kernel can skip restoring the - * FPU state from memory. - * - * Any code that clobbers the FPU registers or updates the in-memory - * FPU state for a task MUST let the rest of the kernel know that the - * FPU registers are no longer valid for this task. - * - * Either one of these invalidation functions is enough. Invalidate - * a resource you control: CPU if using the CPU for something else - * (with preemption disabled), FPU for the current task, or a task that - * is prevented from running by the current task. - */ -static inline void __cpu_invalidate_fpregs_state(void) -{ - __this_cpu_write(fpu_fpregs_owner_ctx, NULL); -} - -static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu) -{ - fpu->last_cpu = -1; -} - -static inline int fpregs_state_valid(struct fpu *fpu, unsigned int cpu) -{ - return fpu == this_cpu_read(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu; -} - -/* - * These generally need preemption protection to work, - * do try to avoid using these on their own: - */ -static inline void fpregs_deactivate(struct fpu *fpu) -{ - this_cpu_write(fpu_fpregs_owner_ctx, NULL); - trace_x86_fpu_regs_deactivated(fpu); -} - -static inline void fpregs_activate(struct fpu *fpu) -{ - this_cpu_write(fpu_fpregs_owner_ctx, fpu); - trace_x86_fpu_regs_activated(fpu); -} - -/* Internal helper for switch_fpu_return() and signal frame setup */ -static inline void fpregs_restore_userregs(void) -{ - struct fpu *fpu = ¤t->thread.fpu; - int cpu = smp_processor_id(); - - if (WARN_ON_ONCE(current->flags & PF_KTHREAD)) - return; - - if (!fpregs_state_valid(fpu, cpu)) { - u64 mask; - - /* - * This restores _all_ xstate which has not been - * established yet. - * - * If PKRU is enabled, then the PKRU value is already - * correct because it was either set in switch_to() or in - * flush_thread(). So it is excluded because it might be - * not up to date in current->thread.fpu.xsave state. - */ - mask = xfeatures_mask_restore_user() | - xfeatures_mask_supervisor(); - __restore_fpregs_from_fpstate(&fpu->state, mask); - - fpregs_activate(fpu); - fpu->last_cpu = cpu; - } - clear_thread_flag(TIF_NEED_FPU_LOAD); -} - -/* - * FPU state switching for scheduling. - * - * This is a two-stage process: - * - * - switch_fpu_prepare() saves the old state. - * This is done within the context of the old process. - * - * - switch_fpu_finish() sets TIF_NEED_FPU_LOAD; the floating point state - * will get loaded on return to userspace, or when the kernel needs it. - * - * If TIF_NEED_FPU_LOAD is cleared then the CPU's FPU registers - * are saved in the current thread's FPU register state. - * - * If TIF_NEED_FPU_LOAD is set then CPU's FPU registers may not - * hold current()'s FPU registers. It is required to load the - * registers before returning to userland or using the content - * otherwise. - * - * The FPU context is only stored/restored for a user task and - * PF_KTHREAD is used to distinguish between kernel and user threads. - */ -static inline void switch_fpu_prepare(struct fpu *old_fpu, int cpu) -{ - if (static_cpu_has(X86_FEATURE_FPU) && !(current->flags & PF_KTHREAD)) { - save_fpregs_to_fpstate(old_fpu); - /* - * The save operation preserved register state, so the - * fpu_fpregs_owner_ctx is still @old_fpu. Store the - * current CPU number in @old_fpu, so the next return - * to user space can avoid the FPU register restore - * when is returns on the same CPU and still owns the - * context. - */ - old_fpu->last_cpu = cpu; - - trace_x86_fpu_regs_deactivated(old_fpu); - } -} - -/* - * Misc helper functions: - */ - -/* - * Delay loading of the complete FPU state until the return to userland. - * PKRU is handled separately. - */ -static inline void switch_fpu_finish(struct fpu *new_fpu) -{ - if (cpu_feature_enabled(X86_FEATURE_FPU)) - set_thread_flag(TIF_NEED_FPU_LOAD); -} - -#endif /* _ASM_X86_FPU_INTERNAL_H */ diff --git a/arch/x86/include/asm/fpu/signal.h b/arch/x86/include/asm/fpu/signal.h index 8b6631dffe..e1c9df9102 100644 --- a/arch/x86/include/asm/fpu/signal.h +++ b/arch/x86/include/asm/fpu/signal.h @@ -5,6 +5,11 @@ #ifndef _ASM_X86_FPU_SIGNAL_H #define _ASM_X86_FPU_SIGNAL_H +#include +#include + +#include + #ifdef CONFIG_X86_64 # include # include @@ -31,6 +36,9 @@ fpu__alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long fpu__get_fpstate_size(void); -extern void fpu__init_prepare_fx_sw_frame(void); +extern bool copy_fpstate_to_sigframe(void __user *buf, void __user *fp, int size); +extern void fpu__clear_user_states(struct fpu *fpu); +extern bool fpu__restore_sig(void __user *buf, int ia32_frame); +extern void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask); #endif /* _ASM_X86_FPU_SIGNAL_H */ diff --git a/arch/x86/include/asm/fpu/types.h b/arch/x86/include/asm/fpu/types.h index f5a38a5f3a..eb7cd1139d 100644 --- a/arch/x86/include/asm/fpu/types.h +++ b/arch/x86/include/asm/fpu/types.h @@ -120,6 +120,9 @@ enum xfeature { XFEATURE_RSRVD_COMP_13, XFEATURE_RSRVD_COMP_14, XFEATURE_LBR, + XFEATURE_RSRVD_COMP_16, + XFEATURE_XTILE_CFG, + XFEATURE_XTILE_DATA, XFEATURE_MAX, }; @@ -136,12 +139,21 @@ enum xfeature { #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) #define XFEATURE_MASK_PASID (1 << XFEATURE_PASID) #define XFEATURE_MASK_LBR (1 << XFEATURE_LBR) +#define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG) +#define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA) #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ | XFEATURE_MASK_ZMM_Hi256 \ | XFEATURE_MASK_Hi16_ZMM) +#ifdef CONFIG_X86_64 +# define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA \ + | XFEATURE_MASK_XTILE_CFG) +#else +# define XFEATURE_MASK_XTILE (0) +#endif + #define FIRST_EXTENDED_XFEATURE XFEATURE_YMM struct reg_128_bit { @@ -153,6 +165,9 @@ struct reg_256_bit { struct reg_512_bit { u8 regbytes[512/8]; }; +struct reg_1024_byte { + u8 regbytes[1024]; +}; /* * State component 2: @@ -255,6 +270,23 @@ struct arch_lbr_state { u64 ler_to; u64 ler_info; struct lbr_entry entries[]; +}; + +/* + * State component 17: 64-byte tile configuration register. + */ +struct xtile_cfg { + u64 tcfg[8]; +} __packed; + +/* + * State component 18: 1KB tile data register. + * Each register represents 16 64-byte rows of the matrix + * data. But the number of registers depends on the actual + * implementation. + */ +struct xtile_data { + struct reg_1024_byte tmm; } __packed; /* @@ -309,6 +341,93 @@ union fpregs_state { u8 __padding[PAGE_SIZE]; }; +struct fpstate { + /* @kernel_size: The size of the kernel register image */ + unsigned int size; + + /* @user_size: The size in non-compacted UABI format */ + unsigned int user_size; + + /* @xfeatures: xfeatures for which the storage is sized */ + u64 xfeatures; + + /* @user_xfeatures: xfeatures valid in UABI buffers */ + u64 user_xfeatures; + + /* @xfd: xfeatures disabled to trap userspace use. */ + u64 xfd; + + /* @is_valloc: Indicator for dynamically allocated state */ + unsigned int is_valloc : 1; + + /* @is_guest: Indicator for guest state (KVM) */ + unsigned int is_guest : 1; + + /* + * @is_confidential: Indicator for KVM confidential mode. + * The FPU registers are restored by the + * vmentry firmware from encrypted guest + * memory. On vmexit the FPU registers are + * saved by firmware to encrypted guest memory + * and the registers are scrubbed before + * returning to the host. So there is no + * content which is worth saving and restoring. + * The fpstate has to be there so that + * preemption and softirq FPU usage works + * without special casing. + */ + unsigned int is_confidential : 1; + + /* @in_use: State is in use */ + unsigned int in_use : 1; + + /* @regs: The register state union for all supported formats */ + union fpregs_state regs; + + /* @regs is dynamically sized! Don't add anything after @regs! */ +} __aligned(64); + +#define FPU_GUEST_PERM_LOCKED BIT_ULL(63) + +struct fpu_state_perm { + /* + * @__state_perm: + * + * This bitmap indicates the permission for state components, which + * are available to a thread group. The permission prctl() sets the + * enabled state bits in thread_group_leader()->thread.fpu. + * + * All run time operations use the per thread information in the + * currently active fpu.fpstate which contains the xfeature masks + * and sizes for kernel and user space. + * + * This master permission field is only to be used when + * task.fpu.fpstate based checks fail to validate whether the task + * is allowed to expand it's xfeatures set which requires to + * allocate a larger sized fpstate buffer. + * + * Do not access this field directly. Use the provided helper + * function. Unlocked access is possible for quick checks. + */ + u64 __state_perm; + + /* + * @__state_size: + * + * The size required for @__state_perm. Only valid to access + * with sighand locked. + */ + unsigned int __state_size; + + /* + * @__user_state_size: + * + * The size required for @__state_perm user part. Only valid to + * access with sighand locked. + */ + unsigned int __user_state_size; +}; + /* * Highest level per task FPU state data structure that * contains the FPU register state plus various FPU @@ -337,19 +456,130 @@ struct fpu { unsigned long avx512_timestamp; /* - * @state: + * @fpstate: * - * In-memory copy of all FPU registers that we save/restore - * over context switches. If the task is using the FPU then - * the registers in the FPU are more recent than this state - * copy. If the task context-switches away then they get - * saved here and represent the FPU state. + * Pointer to the active struct fpstate. Initialized to + * point at @__fpstate below. */ - union fpregs_state state; + struct fpstate *fpstate; + /* - * WARNING: 'state' is dynamically-sized. Do not put + * @__task_fpstate: + * + * Pointer to an inactive struct fpstate. Initialized to NULL. Is + * used only for KVM support to swap out the regular task fpstate. + */ + struct fpstate *__task_fpstate; + + /* + * @perm: + * + * Permission related information + */ + struct fpu_state_perm perm; + + /* + * @guest_perm: + * + * Permission related information for guest pseudo FPUs + */ + struct fpu_state_perm guest_perm; + + /* + * @__fpstate: + * + * Initial in-memory storage for FPU registers which are saved in + * context switch and when the kernel uses the FPU. The registers + * are restored from this storage on return to user space if they + * are not longer containing the tasks FPU register state. + */ + struct fpstate __fpstate; + /* + * WARNING: '__fpstate' is dynamically-sized. Do not put * anything after it here. */ }; +/* + * Guest pseudo FPU container + */ +struct fpu_guest { + /* + * @xfeatures: xfeature bitmap of features which are + * currently enabled for the guest vCPU. + */ + u64 xfeatures; + + /* + * @perm: xfeature bitmap of features which are + * permitted to be enabled for the guest + * vCPU. + */ + u64 perm; + + /* + * @xfd_err: Save the guest value. + */ + u64 xfd_err; + + /* + * @uabi_size: Size required for save/restore + */ + unsigned int uabi_size; + + /* + * @fpstate: Pointer to the allocated guest fpstate + */ + struct fpstate *fpstate; +}; + +/* + * FPU state configuration data. Initialized at boot time. Read only after init. + */ +struct fpu_state_config { + /* + * @max_size: + * + * The maximum size of the register state buffer. Includes all + * supported features except independent managed features. + */ + unsigned int max_size; + + /* + * @default_size: + * + * The default size of the register state buffer. Includes all + * supported features except independent managed features and + * features which have to be requested by user space before usage. + */ + unsigned int default_size; + + /* + * @max_features: + * + * The maximum supported features bitmap. Does not include + * independent managed features. + */ + u64 max_features; + + /* + * @default_features: + * + * The default supported features bitmap. Does not include + * independent managed features and features which have to + * be requested by user space before usage. + */ + u64 default_features; + /* + * @legacy_features: + * + * Features which can be reported back to user space + * even without XSAVE support, i.e. legacy features FP + SSE + */ + u64 legacy_features; +}; + +/* FPU state configuration information */ +extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg; + #endif /* _ASM_X86_FPU_H */ diff --git a/arch/x86/include/asm/fpu/xcr.h b/arch/x86/include/asm/fpu/xcr.h index 1c7ab8d95d..9656a5bc6f 100644 --- a/arch/x86/include/asm/fpu/xcr.h +++ b/arch/x86/include/asm/fpu/xcr.h @@ -2,18 +2,8 @@ #ifndef _ASM_X86_FPU_XCR_H #define _ASM_X86_FPU_XCR_H -/* - * MXCSR and XCR definitions: - */ - -static inline void ldmxcsr(u32 mxcsr) -{ - asm volatile("ldmxcsr %0" :: "m" (mxcsr)); -} - -extern unsigned int mxcsr_feature_mask; - #define XCR_XFEATURE_ENABLED_MASK 0x00000000 +#define XCR_XFEATURE_IN_USE_MASK 0x00000001 static inline u64 xgetbv(u32 index) { @@ -31,4 +21,15 @@ static inline void xsetbv(u32 index, u64 value) asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index)); } +/* + * Return a mask of xfeatures which are currently being tracked + * by the processor as being in the initial configuration. + * + * Callers should check X86_FEATURE_XGETBV1. + */ +static inline u64 xfeatures_in_use(void) +{ + return xgetbv(XCR_XFEATURE_IN_USE_MASK); +} + #endif /* _ASM_X86_FPU_XCR_H */ diff --git a/arch/x86/include/asm/fpu/xstate.h b/arch/x86/include/asm/fpu/xstate.h index 109dfcc752..cd3dd170e2 100644 --- a/arch/x86/include/asm/fpu/xstate.h +++ b/arch/x86/include/asm/fpu/xstate.h @@ -14,6 +14,8 @@ #define XSTATE_CPUID 0x0000000d +#define TILE_CPUID 0x0000001d + #define FXSAVE_SIZE 512 #define XSAVE_HDR_SIZE 64 @@ -33,7 +35,8 @@ XFEATURE_MASK_Hi16_ZMM | \ XFEATURE_MASK_PKRU | \ XFEATURE_MASK_BNDREGS | \ - XFEATURE_MASK_BNDCSR) + XFEATURE_MASK_BNDCSR | \ + XFEATURE_MASK_XTILE) /* * Features which are restored when returning to user space. @@ -43,6 +46,9 @@ #define XFEATURE_MASK_USER_RESTORE \ (XFEATURE_MASK_USER_SUPPORTED & ~XFEATURE_MASK_PKRU) +/* Features which are dynamically enabled for a process on request */ +#define XFEATURE_MASK_USER_DYNAMIC XFEATURE_MASK_XTILE_DATA + /* All currently supported supervisor features */ #define XFEATURE_MASK_SUPERVISOR_SUPPORTED (XFEATURE_MASK_PASID) @@ -78,78 +84,49 @@ XFEATURE_MASK_INDEPENDENT | \ XFEATURE_MASK_SUPERVISOR_UNSUPPORTED) -#ifdef CONFIG_X86_64 -#define REX_PREFIX "0x48, " -#else -#define REX_PREFIX -#endif - -extern u64 xfeatures_mask_all; - -static inline u64 xfeatures_mask_supervisor(void) -{ - return xfeatures_mask_all & XFEATURE_MASK_SUPERVISOR_SUPPORTED; -} +/* + * The feature mask required to restore FPU state: + * - All user states which are not eagerly switched in switch_to()/exec() + * - The suporvisor states + */ +#define XFEATURE_MASK_FPSTATE (XFEATURE_MASK_USER_RESTORE | \ + XFEATURE_MASK_SUPERVISOR_SUPPORTED) /* - * The xfeatures which are enabled in XCR0 and expected to be in ptrace - * buffers and signal frames. + * Features in this mask have space allocated in the signal frame, but may not + * have that space initialized when the feature is in its init state. */ -static inline u64 xfeatures_mask_uabi(void) -{ - return xfeatures_mask_all & XFEATURE_MASK_USER_SUPPORTED; -} - -/* - * The xfeatures which are restored by the kernel when returning to user - * mode. This is not necessarily the same as xfeatures_mask_uabi() as the - * kernel does not manage all XCR0 enabled features via xsave/xrstor as - * some of them have to be switched eagerly on context switch and exec(). - */ -static inline u64 xfeatures_mask_restore_user(void) -{ - return xfeatures_mask_all & XFEATURE_MASK_USER_RESTORE; -} - -/* - * Like xfeatures_mask_restore_user() but additionally restors the - * supported supervisor states. - */ -static inline u64 xfeatures_mask_fpstate(void) -{ - return xfeatures_mask_all & \ - (XFEATURE_MASK_USER_RESTORE | XFEATURE_MASK_SUPERVISOR_SUPPORTED); -} - -static inline u64 xfeatures_mask_independent(void) -{ - if (!boot_cpu_has(X86_FEATURE_ARCH_LBR)) - return XFEATURE_MASK_INDEPENDENT & ~XFEATURE_MASK_LBR; - - return XFEATURE_MASK_INDEPENDENT; -} +#define XFEATURE_MASK_SIGFRAME_INITOPT (XFEATURE_MASK_XTILE | \ + XFEATURE_MASK_USER_DYNAMIC) extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS]; extern void __init update_regset_xstate_info(unsigned int size, u64 xstate_mask); -void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr); int xfeature_size(int xfeature_nr); -int copy_uabi_from_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf); -int copy_sigframe_from_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf); void xsaves(struct xregs_state *xsave, u64 mask); void xrstors(struct xregs_state *xsave, u64 mask); -enum xstate_copy_mode { - XSTATE_COPY_FP, - XSTATE_COPY_FX, - XSTATE_COPY_XSAVE, -}; +int xfd_enable_feature(u64 xfd_err); -struct membuf; -void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, - enum xstate_copy_mode mode); +#ifdef CONFIG_X86_64 +DECLARE_STATIC_KEY_FALSE(__fpu_state_size_dynamic); +#endif + +#ifdef CONFIG_X86_64 +DECLARE_STATIC_KEY_FALSE(__fpu_state_size_dynamic); + +static __always_inline __pure bool fpu_state_size_dynamic(void) +{ + return static_branch_unlikely(&__fpu_state_size_dynamic); +} +#else +static __always_inline __pure bool fpu_state_size_dynamic(void) +{ + return false; +} +#endif #endif diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index 9f3130f408..024d979764 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h @@ -57,6 +57,13 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs) #define ftrace_instruction_pointer_set(fregs, _ip) \ do { (fregs)->regs.ip = (_ip); } while (0) + +struct ftrace_ops; +#define ftrace_graph_func ftrace_graph_func +void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *op, struct ftrace_regs *fregs); +#else +#define FTRACE_GRAPH_TRAMP_ADDR FTRACE_GRAPH_ADDR #endif #ifdef CONFIG_DYNAMIC_FTRACE @@ -65,8 +72,6 @@ struct dyn_arch_ftrace { /* No extra data needed for x86 */ }; -#define FTRACE_GRAPH_TRAMP_ADDR FTRACE_GRAPH_ADDR - #endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* __ASSEMBLY__ */ #endif /* CONFIG_FUNCTION_TRACER */ diff --git a/arch/x86/include/asm/futex.h b/arch/x86/include/asm/futex.h index f9c00110a6..99d345b686 100644 --- a/arch/x86/include/asm/futex.h +++ b/arch/x86/include/asm/futex.h @@ -17,13 +17,9 @@ do { \ int oldval = 0, ret; \ asm volatile("1:\t" insn "\n" \ "2:\n" \ - "\t.section .fixup,\"ax\"\n" \ - "3:\tmov\t%3, %1\n" \ - "\tjmp\t2b\n" \ - "\t.previous\n" \ - _ASM_EXTABLE_UA(1b, 3b) \ + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %1) \ : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \ - : "i" (-EFAULT), "0" (oparg), "1" (0)); \ + : "0" (oparg), "1" (0)); \ if (ret) \ goto label; \ *oval = oldval; \ @@ -39,15 +35,11 @@ do { \ "3:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ "\tjnz\t2b\n" \ "4:\n" \ - "\t.section .fixup,\"ax\"\n" \ - "5:\tmov\t%5, %1\n" \ - "\tjmp\t4b\n" \ - "\t.previous\n" \ - _ASM_EXTABLE_UA(1b, 5b) \ - _ASM_EXTABLE_UA(3b, 5b) \ + _ASM_EXTABLE_TYPE_REG(1b, 4b, EX_TYPE_EFAULT_REG, %1) \ + _ASM_EXTABLE_TYPE_REG(3b, 4b, EX_TYPE_EFAULT_REG, %1) \ : "=&a" (oldval), "=&r" (ret), \ "+m" (*uaddr), "=&r" (tem) \ - : "r" (oparg), "i" (-EFAULT), "1" (0)); \ + : "r" (oparg), "1" (0)); \ if (ret) \ goto label; \ *oval = oldval; \ @@ -95,15 +87,11 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, if (!user_access_begin(uaddr, sizeof(u32))) return -EFAULT; asm volatile("\n" - "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" + "1:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" "2:\n" - "\t.section .fixup, \"ax\"\n" - "3:\tmov %3, %0\n" - "\tjmp 2b\n" - "\t.previous\n" - _ASM_EXTABLE_UA(1b, 3b) + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %0) \ : "+r" (ret), "=a" (oldval), "+m" (*uaddr) - : "i" (-EFAULT), "r" (newval), "1" (oldval) + : "r" (newval), "1" (oldval) : "memory" ); user_access_end(); diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 2322d6bd58..0a9407dc08 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -276,6 +276,23 @@ enum hv_isolation_type { #define HV_X64_MSR_TIME_REF_COUNT HV_REGISTER_TIME_REF_COUNT #define HV_X64_MSR_REFERENCE_TSC HV_REGISTER_REFERENCE_TSC +/* Hyper-V memory host visibility */ +enum hv_mem_host_visibility { + VMBUS_PAGE_NOT_VISIBLE = 0, + VMBUS_PAGE_VISIBLE_READ_ONLY = 1, + VMBUS_PAGE_VISIBLE_READ_WRITE = 3 +}; + +/* HvCallModifySparseGpaPageHostVisibility hypercall */ +#define HV_MAX_MODIFY_GPA_REP_COUNT ((PAGE_SIZE / sizeof(u64)) - 2) +struct hv_gpa_range_for_visibility { + u64 partition_id; + u32 host_visibility:2; + u32 reserved0:30; + u32 reserved1; + u64 gpa_page_list[HV_MAX_MODIFY_GPA_REP_COUNT]; +} __packed; + /* * Declare the MSR used to setup pages used to communicate with the hypervisor. */ @@ -585,6 +602,39 @@ enum hv_interrupt_type { HV_X64_INTERRUPT_TYPE_MAXIMUM = 0x000A, }; +union hv_msi_address_register { + u32 as_uint32; + struct { + u32 reserved1:2; + u32 destination_mode:1; + u32 redirection_hint:1; + u32 reserved2:8; + u32 destination_id:8; + u32 msi_base:12; + }; +} __packed; + +union hv_msi_data_register { + u32 as_uint32; + struct { + u32 vector:8; + u32 delivery_mode:3; + u32 reserved1:3; + u32 level_assert:1; + u32 trigger_mode:1; + u32 reserved2:16; + }; +} __packed; + +/* HvRetargetDeviceInterrupt hypercall */ +union hv_msi_entry { + u64 as_uint64; + struct { + union hv_msi_address_register address; + union hv_msi_data_register data; + } __packed; +}; + #include #endif diff --git a/arch/x86/include/asm/ia32.h b/arch/x86/include/asm/ia32.h index 2c5f7861d3..fada857f0a 100644 --- a/arch/x86/include/asm/ia32.h +++ b/arch/x86/include/asm/ia32.h @@ -68,6 +68,6 @@ extern void ia32_pick_mmap_layout(struct mm_struct *mm); #endif -#endif /* !CONFIG_IA32_SUPPORT */ +#endif /* CONFIG_IA32_EMULATION */ #endif /* _ASM_X86_IA32_H */ diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index 4ec3613551..f07faa61c7 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -15,10 +15,13 @@ #define INSN_CODE_SEG_OPND_SZ(params) (params & 0xf) #define INSN_CODE_SEG_PARAMS(oper_sz, addr_sz) (oper_sz | (addr_sz << 4)) +int pt_regs_offset(struct pt_regs *regs, int regno); + bool insn_has_rep_prefix(struct insn *insn); void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_rm_off(struct insn *insn, struct pt_regs *regs); int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs); +unsigned long *insn_get_modrm_reg_ptr(struct insn *insn, struct pt_regs *regs); unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx); int insn_get_code_seg_params(struct pt_regs *regs); int insn_get_effective_ip(struct pt_regs *regs, unsigned long *ip); @@ -29,4 +32,16 @@ int insn_fetch_from_user_inatomic(struct pt_regs *regs, bool insn_decode_from_regs(struct insn *insn, struct pt_regs *regs, unsigned char buf[MAX_INSN_SIZE], int buf_size); +enum mmio_type { + MMIO_DECODE_FAILED, + MMIO_WRITE, + MMIO_WRITE_IMM, + MMIO_READ, + MMIO_READ_ZERO_EXTEND, + MMIO_READ_SIGN_EXTEND, + MMIO_MOVS, +}; + +enum mmio_type insn_decode_mmio(struct insn *insn, int *bytes); + #endif /* _ASM_X86_INSN_EVAL_H */ diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h index 27158436f3..048b6d5aff 100644 --- a/arch/x86/include/asm/intel-family.h +++ b/arch/x86/include/asm/intel-family.h @@ -108,6 +108,8 @@ #define INTEL_FAM6_ALDERLAKE 0x97 /* Golden Cove / Gracemont */ #define INTEL_FAM6_ALDERLAKE_L 0x9A /* Golden Cove / Gracemont */ +#define INTEL_FAM6_RAPTORLAKE 0xB7 + /* "Small Core" Processors (Atom) */ #define INTEL_FAM6_ATOM_BONNELL 0x1C /* Diamondville, Pineview */ diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index 841a5d104a..f6d91ecb80 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -40,6 +40,7 @@ #include #include +#include #include #include #include @@ -256,21 +257,6 @@ static inline void slow_down_io(void) #endif -#ifdef CONFIG_AMD_MEM_ENCRYPT -#include - -extern struct static_key_false sev_enable_key; -static inline bool sev_key_active(void) -{ - return static_branch_unlikely(&sev_enable_key); -} - -#else /* !CONFIG_AMD_MEM_ENCRYPT */ - -static inline bool sev_key_active(void) { return false; } - -#endif /* CONFIG_AMD_MEM_ENCRYPT */ - #define BUILDIO(bwl, bw, type) \ static inline void out##bwl(unsigned type value, int port) \ { \ @@ -301,7 +287,7 @@ static inline unsigned type in##bwl##_p(int port) \ \ static inline void outs##bwl(int port, const void *addr, unsigned long count) \ { \ - if (sev_key_active()) { \ + if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) { \ unsigned type *value = (unsigned type *)addr; \ while (count) { \ out##bwl(*value, port); \ @@ -317,7 +303,7 @@ static inline void outs##bwl(int port, const void *addr, unsigned long count) \ \ static inline void ins##bwl(int port, void *addr, unsigned long count) \ { \ - if (sev_key_active()) { \ + if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) { \ unsigned type *value = (unsigned type *)addr; \ while (count) { \ *value = in##bwl(port); \ @@ -391,6 +377,7 @@ extern void arch_io_free_memtype_wc(resource_size_t start, resource_size_t size) #define arch_io_reserve_memtype_wc arch_io_reserve_memtype_wc #endif +#ifdef CONFIG_AMD_MEM_ENCRYPT extern bool arch_memremap_can_ram_remap(resource_size_t offset, unsigned long size, unsigned long flags); @@ -398,6 +385,13 @@ extern bool arch_memremap_can_ram_remap(resource_size_t offset, extern bool phys_mem_access_encrypted(unsigned long phys_addr, unsigned long size); +#else +static inline bool phys_mem_access_encrypted(unsigned long phys_addr, + unsigned long size) +{ + return true; +} +#endif /** * iosubmit_cmds512 - copy data to single MMIO location, in 512-bit units diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 8d55bd1184..ae9d40f6c7 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -58,7 +58,7 @@ * the output constraints to make the compiler aware that R11 cannot be * reused after the asm() statement. * - * For builds with CONFIG_UNWIND_FRAME_POINTER ASM_CALL_CONSTRAINT is + * For builds with CONFIG_UNWINDER_FRAME_POINTER, ASM_CALL_CONSTRAINT is * required as well as this prevents certain creative GCC variants from * misplacing the ASM code. * @@ -201,6 +201,7 @@ IRQ_CONSTRAINTS, regs, vector); \ } +#ifndef CONFIG_PREEMPT_RT /* * Macro to invoke __do_softirq on the irq stack. This is only called from * task context when bottom halves are about to be reenabled and soft @@ -214,6 +215,8 @@ __this_cpu_write(hardirq_stack_inuse, false); \ } +#endif + #else /* CONFIG_X86_64 */ /* System vector handlers always run on the stack they interrupted. */ #define run_sysvec_on_irqstack_cond(func, regs) \ diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h index c5ce9845c9..87761396e8 100644 --- a/arch/x86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -114,8 +114,6 @@ static __always_inline unsigned long arch_local_irq_save(void) #define SAVE_FLAGS pushfq; popq %rax #endif -#define INTERRUPT_RETURN jmp native_iret - #endif #endif /* __ASSEMBLY__ */ @@ -143,8 +141,13 @@ static __always_inline void arch_local_irq_restore(unsigned long flags) #ifdef CONFIG_X86_64 #ifdef CONFIG_XEN_PV #define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV +#define INTERRUPT_RETURN \ + ANNOTATE_RETPOLINE_SAFE; \ + ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);", \ + X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;") #else #define SWAPGS swapgs +#define INTERRUPT_RETURN jmp native_iret #endif #endif #endif /* !__ASSEMBLY__ */ diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h index 0a6e34b070..11b7c06e28 100644 --- a/arch/x86/include/asm/kexec.h +++ b/arch/x86/include/asm/kexec.h @@ -129,7 +129,7 @@ relocate_kernel(unsigned long indirection_page, unsigned long page_list, unsigned long start_address, unsigned int preserve_context, - unsigned int sme_active); + unsigned int host_mem_enc_active); #endif #define ARCH_HAS_KIMAGE_ARCH diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h index bd7f5886a7..71ea2eab43 100644 --- a/arch/x86/include/asm/kprobes.h +++ b/arch/x86/include/asm/kprobes.h @@ -49,7 +49,6 @@ extern __visible kprobe_opcode_t optprobe_template_end[]; extern const int kretprobe_blacklist_size; void arch_remove_kprobe(struct kprobe *p); -asmlinkage void kretprobe_trampoline(void); extern void arch_kprobe_override_function(struct pt_regs *regs); diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 9e50da3ed0..d39e0de06b 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -35,6 +35,7 @@ KVM_X86_OP(get_cpl) KVM_X86_OP(set_segment) KVM_X86_OP_NULL(get_cs_db_l_bits) KVM_X86_OP(set_cr0) +KVM_X86_OP_NULL(post_set_cr3) KVM_X86_OP(is_valid_cr4) KVM_X86_OP(set_cr4) KVM_X86_OP(set_efer) @@ -54,6 +55,7 @@ KVM_X86_OP_NULL(tlb_remote_flush) KVM_X86_OP_NULL(tlb_remote_flush_with_range) KVM_X86_OP(tlb_flush_gva) KVM_X86_OP(tlb_flush_guest) +KVM_X86_OP(vcpu_pre_run) KVM_X86_OP(run) KVM_X86_OP_NULL(handle_exit) KVM_X86_OP_NULL(skip_emulated_instruction) @@ -80,7 +82,7 @@ KVM_X86_OP_NULL(guest_apic_has_interrupt) KVM_X86_OP(load_eoi_exitmap) KVM_X86_OP(set_virtual_apic_mode) KVM_X86_OP_NULL(set_apic_access_page_addr) -KVM_X86_OP(deliver_posted_interrupt) +KVM_X86_OP(deliver_interrupt) KVM_X86_OP_NULL(sync_pir_to_irr) KVM_X86_OP(set_tss_addr) KVM_X86_OP(set_identity_map_addr) @@ -97,8 +99,6 @@ KVM_X86_OP(handle_exit_irqoff) KVM_X86_OP_NULL(request_immediate_exit) KVM_X86_OP(sched_in) KVM_X86_OP_NULL(update_cpu_dirty_logging) -KVM_X86_OP_NULL(pre_block) -KVM_X86_OP_NULL(post_block) KVM_X86_OP_NULL(vcpu_blocking) KVM_X86_OP_NULL(vcpu_unblocking) KVM_X86_OP_NULL(update_pi_irte) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 01759199d7..ec9830d2aa 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -38,7 +38,6 @@ #define __KVM_HAVE_ARCH_VCPU_DEBUGFS #define KVM_MAX_VCPUS 1024 -#define KVM_SOFT_MAX_VCPUS 710 /* * In x86, the VCPU ID corresponds to the APIC ID, and APIC IDs @@ -50,7 +49,7 @@ * so ratio of 4 should be enough. */ #define KVM_VCPU_ID_RATIO 4 -#define KVM_MAX_VCPU_ID (KVM_MAX_VCPUS * KVM_VCPU_ID_RATIO) +#define KVM_MAX_VCPU_IDS (KVM_MAX_VCPUS * KVM_VCPU_ID_RATIO) /* memory slots that are not exposed to userspace */ #define KVM_PRIVATE_MEM_SLOTS 3 @@ -136,7 +135,7 @@ #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) -#define KVM_PERMILLE_MMU_PAGES 20 +#define KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO 50 #define KVM_MIN_ALLOC_MMU_PAGES 64UL #define KVM_MMU_HASH_SHIFT 12 #define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT) @@ -292,25 +291,31 @@ struct kvm_kernel_irq_routing_entry; * the number of unique SPs that can theoretically be created is 2^n, where n * is the number of bits that are used to compute the role. * - * But, even though there are 18 bits in the mask below, not all combinations - * of modes and flags are possible. The maximum number of possible upper-level - * shadow pages for a single gfn is in the neighborhood of 2^13. + * But, even though there are 19 bits in the mask below, not all combinations + * of modes and flags are possible: * - * - invalid shadow pages are not accounted. - * - level is effectively limited to four combinations, not 16 as the number - * bits would imply, as 4k SPs are not tracked (allowed to go unsync). - * - level is effectively unused for non-PAE paging because there is exactly - * one upper level (see 4k SP exception above). - * - quadrant is used only for non-PAE paging and is exclusive with - * gpte_is_8_bytes. - * - execonly and ad_disabled are used only for nested EPT, which makes it - * exclusive with quadrant. + * - invalid shadow pages are not accounted, so the bits are effectively 18 + * + * - quadrant will only be used if has_4_byte_gpte=1 (non-PAE paging); + * execonly and ad_disabled are only used for nested EPT which has + * has_4_byte_gpte=0. Therefore, 2 bits are always unused. + * + * - the 4 bits of level are effectively limited to the values 2/3/4/5, + * as 4k SPs are not tracked (allowed to go unsync). In addition non-PAE + * paging has exactly one upper level, making level completely redundant + * when has_4_byte_gpte=1. + * + * - on top of this, smep_andnot_wp and smap_andnot_wp are only set if + * cr0_wp=0, therefore these three bits only give rise to 5 possibilities. + * + * Therefore, the maximum number of possible upper-level shadow pages for a + * single gfn is a bit less than 2^13. */ union kvm_mmu_page_role { u32 word; struct { unsigned level:4; - unsigned gpte_is_8_bytes:1; + unsigned has_4_byte_gpte:1; unsigned quadrant:2; unsigned direct:1; unsigned access:3; @@ -408,6 +413,7 @@ struct kvm_mmu_root_info { #define KVM_HAVE_MMU_RWLOCK struct kvm_mmu_page; +struct kvm_page_fault; /* * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit, @@ -417,14 +423,12 @@ struct kvm_mmu_page; struct kvm_mmu { unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu); u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index); - int (*page_fault)(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u32 err, - bool prefault); + int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); void (*inject_page_fault)(struct kvm_vcpu *vcpu, struct x86_exception *fault); - gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gpa_t gva_or_gpa, - u32 access, struct x86_exception *exception); - gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, - struct x86_exception *exception); + gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, + gpa_t gva_or_gpa, u32 access, + struct x86_exception *exception); int (*sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp); void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa); @@ -491,6 +495,7 @@ struct kvm_pmc { */ u64 current_config; bool is_paused; + bool intr; }; struct kvm_pmu { @@ -500,7 +505,6 @@ struct kvm_pmu { u64 fixed_ctr_ctrl; u64 global_ctrl; u64 global_status; - u64 global_ovf_ctrl; u64 counter_bitmask[2]; u64 global_ctrl_mask; u64 global_ovf_ctrl_mask; @@ -582,7 +586,6 @@ struct kvm_vcpu_hv { struct kvm_hyperv_exit exit; struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT]; DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); - cpumask_t tlb_flush; bool enforce_cpuid; struct { u32 features_eax; /* HYPERV_CPUID_FEATURES.EAX */ @@ -607,6 +610,7 @@ struct kvm_vcpu_xen { u64 last_steal; u64 runstate_entry_time; u64 runstate_times[4]; + unsigned long evtchn_pending_sel; }; struct kvm_vcpu_arch { @@ -643,6 +647,7 @@ struct kvm_vcpu_arch { u64 smi_count; bool tpr_access_reporting; bool xsaves_enabled; + bool xfd_no_write_intercept; u64 ia32_xss; u64 microcode_version; u64 arch_capabilities; @@ -692,14 +697,12 @@ struct kvm_vcpu_arch { * * Note that while the PKRU state lives inside the fpu registers, * it is switched out separately at VMENTER and VMEXIT time. The - * "guest_fpu" state here contains the guest FPU context, with the + * "guest_fpstate" state here contains the guest FPU context, with the * host PRKU bits. */ - struct fpu *user_fpu; - struct fpu *guest_fpu; + struct fpu_guest guest_fpu; u64 xcr0; - u64 guest_supported_xcr0; struct kvm_pio_request pio; void *pio_data; @@ -729,6 +732,7 @@ struct kvm_vcpu_arch { int cpuid_nent; struct kvm_cpuid_entry2 *cpuid_entries; + u32 kvm_cpuid_base; u64 reserved_gpa_bits; int maxphyaddr; @@ -777,6 +781,7 @@ struct kvm_vcpu_arch { unsigned nmi_pending; /* NMI queued after currently running handler */ bool nmi_injected; /* Trying to inject an NMI this entry */ bool smi_pending; /* SMI queued after currently running handler */ + u8 handling_intr_from_guest; struct kvm_mtrr mtrr_state; u64 pat; @@ -1017,7 +1022,7 @@ struct msr_bitmap_range { struct kvm_xen { bool long_mode; u8 upcall_vector; - gfn_t shinfo_gfn; + struct gfn_to_pfn_cache shinfo_cache; }; enum kvm_irqchip_mode { @@ -1038,6 +1043,8 @@ struct kvm_x86_msr_filter { #define APICV_INHIBIT_REASON_IRQWIN 3 #define APICV_INHIBIT_REASON_PIT_REINJ 4 #define APICV_INHIBIT_REASON_X2APIC 5 +#define APICV_INHIBIT_REASON_BLOCKIRQ 6 +#define APICV_INHIBIT_REASON_ABSENT 7 struct kvm_arch { unsigned long n_used_mmu_pages; @@ -1075,7 +1082,7 @@ struct kvm_arch { atomic_t apic_map_dirty; /* Protects apic_access_memslot_enabled and apicv_inhibit_reasons */ - struct mutex apicv_update_lock; + struct rw_semaphore apicv_update_lock; bool apic_access_memslot_enabled; unsigned long apicv_inhibit_reasons; @@ -1089,17 +1096,23 @@ struct kvm_arch { unsigned long irq_sources_bitmap; s64 kvmclock_offset; + + /* + * This also protects nr_vcpus_matched_tsc which is read from a + * preemption-disabled region, so it must be a raw spinlock. + */ raw_spinlock_t tsc_write_lock; u64 last_tsc_nsec; u64 last_tsc_write; u32 last_tsc_khz; + u64 last_tsc_offset; u64 cur_tsc_nsec; u64 cur_tsc_write; u64 cur_tsc_offset; u64 cur_tsc_generation; int nr_vcpus_matched_tsc; - raw_spinlock_t pvclock_gtod_sync_lock; + seqcount_raw_spinlock_t pvclock_sc; bool use_master_clock; u64 master_kernel_ns; u64 master_cycle_now; @@ -1209,10 +1222,11 @@ struct kvm_arch { #endif /* CONFIG_X86_64 */ /* - * If set, rmaps have been allocated for all memslots and should be - * allocated for any newly created or modified memslots. + * If set, at least one shadow root has been allocated. This flag + * is used as one input when determining whether certain memslot + * related allocations are necessary. */ - bool memslots_have_rmaps; + bool shadow_root_allocated; #if IS_ENABLED(CONFIG_HYPERV) hpa_t hv_root_tdp; @@ -1298,6 +1312,8 @@ static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical) } struct kvm_x86_ops { + const char *name; + int (*hardware_enable)(void); void (*hardware_disable)(void); void (*hardware_unsetup)(void); @@ -1329,6 +1345,7 @@ struct kvm_x86_ops { struct kvm_segment *var, int seg); void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); + void (*post_set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); bool (*is_valid_cr4)(struct kvm_vcpu *vcpu, unsigned long cr0); void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); int (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); @@ -1363,6 +1380,7 @@ struct kvm_x86_ops { */ void (*tlb_flush_guest)(struct kvm_vcpu *vcpu); + int (*vcpu_pre_run)(struct kvm_vcpu *vcpu); enum exit_fastpath_completion (*run)(struct kvm_vcpu *vcpu); int (*handle_exit)(struct kvm_vcpu *vcpu, enum exit_fastpath_completion exit_fastpath); @@ -1391,7 +1409,8 @@ struct kvm_x86_ops { void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu); - int (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector); + void (*deliver_interrupt)(struct kvm_lapic *apic, int delivery_mode, + int trig_mode, int vector); int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu); int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr); @@ -1408,10 +1427,11 @@ struct kvm_x86_ops { void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu, u64 multiplier); /* - * Retrieve somewhat arbitrary exit information. Intended to be used - * only from within tracepoints to avoid VMREADs when tracing is off. + * Retrieve somewhat arbitrary exit information. Intended to + * be used only from within tracepoints or error paths. */ - void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2, + void (*get_exit_info)(struct kvm_vcpu *vcpu, u32 *reason, + u64 *info1, u64 *info2, u32 *exit_int_info, u32 *exit_int_info_err_code); int (*check_intercept)(struct kvm_vcpu *vcpu, @@ -1435,18 +1455,6 @@ struct kvm_x86_ops { const struct kvm_pmu_ops *pmu_ops; const struct kvm_x86_nested_ops *nested_ops; - /* - * Architecture specific hooks for vCPU blocking due to - * HLT instruction. - * Returns for .pre_block(): - * - 0 means continue to block the vCPU. - * - 1 means we cannot block the vCPU since some event - * happens during this period, such as, 'ON' bit in - * posted-interrupts descriptor is set. - */ - int (*pre_block)(struct kvm_vcpu *vcpu); - void (*post_block)(struct kvm_vcpu *vcpu); - void (*vcpu_blocking)(struct kvm_vcpu *vcpu); void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); @@ -1471,10 +1479,12 @@ struct kvm_x86_ops { int (*mem_enc_reg_region)(struct kvm *kvm, struct kvm_enc_region *argp); int (*mem_enc_unreg_region)(struct kvm *kvm, struct kvm_enc_region *argp); int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); + int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); int (*get_msr_feature)(struct kvm_msr_entry *entry); - bool (*can_emulate_instruction)(struct kvm_vcpu *vcpu, void *insn, int insn_len); + bool (*can_emulate_instruction)(struct kvm_vcpu *vcpu, int emul_type, + void *insn, int insn_len); bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu); int (*enable_direct_tlbflush)(struct kvm_vcpu *vcpu); @@ -1510,7 +1520,7 @@ struct kvm_x86_init_ops { int (*disabled_by_bios)(void); int (*check_processor_compatibility)(void); int (*hardware_setup)(void); - bool (*intel_pt_intr_in_guest)(void); + unsigned int (*handle_intel_pt_intr)(void); struct kvm_x86_ops *runtime_ops; }; @@ -1546,6 +1556,8 @@ static inline struct kvm *kvm_arch_alloc_vm(void) { return __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO); } + +#define __KVM_HAVE_ARCH_VM_FREE void kvm_arch_free_vm(struct kvm *kvm); #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB @@ -1558,6 +1570,9 @@ static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm) return -ENOTSUPP; } +#define kvm_arch_pmi_in_guest(vcpu) \ + ((vcpu) && (vcpu)->arch.handling_intr_from_guest) + int kvm_mmu_module_init(void); void kvm_mmu_module_exit(void); @@ -1577,10 +1592,9 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, const struct kvm_memory_slot *memslot); void kvm_mmu_zap_all(struct kvm *kvm); void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen); -unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm); void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages); -int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3); +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, const void *val, int bytes); @@ -1630,7 +1644,8 @@ extern u64 kvm_mce_cap_supported; * * EMULTYPE_SKIP - Set when emulating solely to skip an instruction, i.e. to * decode the instruction length. For use *only* by - * kvm_x86_ops.skip_emulated_instruction() implementations. + * kvm_x86_ops.skip_emulated_instruction() implementations if + * EMULTYPE_COMPLETE_USER_EXIT is not set. * * EMULTYPE_ALLOW_RETRY_PF - Set when the emulator should resume the guest to * retry native execution under certain conditions, @@ -1650,6 +1665,10 @@ extern u64 kvm_mce_cap_supported; * * EMULTYPE_PF - Set when emulating MMIO by way of an intercepted #PF, in which * case the CR2/GPA value pass on the stack is valid. + * + * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility + * state and inject single-step #DBs after skipping + * an instruction (after completing userspace I/O). */ #define EMULTYPE_NO_DECODE (1 << 0) #define EMULTYPE_TRAP_UD (1 << 1) @@ -1658,10 +1677,14 @@ extern u64 kvm_mce_cap_supported; #define EMULTYPE_TRAP_UD_FORCED (1 << 4) #define EMULTYPE_VMWARE_GP (1 << 5) #define EMULTYPE_PF (1 << 6) +#define EMULTYPE_COMPLETE_USER_EXIT (1 << 7) int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type); int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, void *insn, int insn_len); +void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, + u64 *data, u8 ndata); +void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu); void kvm_enable_efer_bits(u64); bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer); @@ -1679,7 +1702,7 @@ int kvm_emulate_monitor(struct kvm_vcpu *vcpu); int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in); int kvm_emulate_cpuid(struct kvm_vcpu *vcpu); int kvm_emulate_halt(struct kvm_vcpu *vcpu); -int kvm_vcpu_halt(struct kvm_vcpu *vcpu); +int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu); int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu); int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); @@ -1690,8 +1713,6 @@ void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector); int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, int reason, bool has_error_code, u32 error_code); -void kvm_free_guest_fpu(struct kvm_vcpu *vcpu); - void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0); void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4); int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); @@ -1720,9 +1741,6 @@ void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); -int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, - gfn_t gfn, void *data, int offset, int len, - u32 access); bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl); bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr); @@ -1750,12 +1768,9 @@ void kvm_inject_nmi(struct kvm_vcpu *vcpu); void kvm_update_dr7(struct kvm_vcpu *vcpu); int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn); -void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, ulong roots_to_free); void kvm_mmu_free_guest_mode_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu); -gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, - struct x86_exception *exception); gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception); gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, @@ -1848,7 +1863,6 @@ int kvm_cpu_has_extint(struct kvm_vcpu *v); int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu); int kvm_cpu_get_interrupt(struct kvm_vcpu *v); void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); -void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu); int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, unsigned long ipi_bitmap_high, u32 min, @@ -1871,7 +1885,6 @@ u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier); unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu); bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip); -void kvm_make_mclock_inprogress_request(struct kvm *kvm); void kvm_make_scan_ioapic_request(struct kvm *kvm); void kvm_make_scan_ioapic_request_mask(struct kvm *kvm, unsigned long *vcpu_bitmap); @@ -1890,8 +1903,6 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu); -int kvm_is_in_guest(void); - void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size); bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu); @@ -1920,8 +1931,6 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) static_call_cond(kvm_x86_vcpu_unblocking)(vcpu); } -static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} - static inline int kvm_cpu_get_apicid(int mps_cpu) { #ifdef CONFIG_X86_LOCAL_APIC @@ -1940,6 +1949,9 @@ static inline int kvm_cpu_get_apicid(int mps_cpu) int kvm_cpu_dirty_log_size(void); -int alloc_all_memslots_rmaps(struct kvm *kvm); +int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages); + +#define KVM_CLOCK_VALID_FLAGS \ + (KVM_CLOCK_TSC_STABLE | KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC) #endif /* _ASM_X86_KVM_HOST_H */ diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h index 6a5f3acf2b..eb186bc57f 100644 --- a/arch/x86/include/asm/kvm_page_track.h +++ b/arch/x86/include/asm/kvm_page_track.h @@ -49,8 +49,12 @@ struct kvm_page_track_notifier_node { int kvm_page_track_init(struct kvm *kvm); void kvm_page_track_cleanup(struct kvm *kvm); +bool kvm_page_track_write_tracking_enabled(struct kvm *kvm); +int kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot); + void kvm_page_track_free_memslot(struct kvm_memory_slot *slot); -int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, +int kvm_page_track_create_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot, unsigned long npages); void kvm_slot_page_track_add_page(struct kvm *kvm, @@ -59,8 +63,9 @@ void kvm_slot_page_track_add_page(struct kvm *kvm, void kvm_slot_page_track_remove_page(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, enum kvm_page_track_mode mode); -bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, - enum kvm_page_track_mode mode); +bool kvm_slot_page_track_is_active(struct kvm *kvm, + const struct kvm_memory_slot *slot, + gfn_t gfn, enum kvm_page_track_mode mode); void kvm_page_track_register_notifier(struct kvm *kvm, diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index 69299878b2..56935ebb1d 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h @@ -83,6 +83,18 @@ static inline long kvm_hypercall4(unsigned int nr, unsigned long p1, return ret; } +static inline long kvm_sev_hypercall3(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3) +{ + long ret; + + asm volatile("vmmcall" + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2), "d"(p3) + : "memory"); + return ret; +} + #ifdef CONFIG_KVM_GUEST void kvmclock_init(void); void kvmclock_disable(void); diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index 365111789c..030907922b 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -18,6 +18,20 @@ #define __ALIGN_STR __stringify(__ALIGN) #endif +#ifdef CONFIG_SLS +#define RET ret; int3 +#else +#define RET ret +#endif + +#else /* __ASSEMBLY__ */ + +#ifdef CONFIG_SLS +#define ASM_RET "ret; int3\n\t" +#else +#define ASM_RET "ret\n\t" +#endif + #endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_LINKAGE_H */ diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index da9321548f..cc73061e72 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -205,28 +205,16 @@ struct cper_ia_proc_ctx; int mcheck_init(void); void mcheck_cpu_init(struct cpuinfo_x86 *c); void mcheck_cpu_clear(struct cpuinfo_x86 *c); -void mcheck_vendor_init_severity(void); int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info, u64 lapic_id); #else static inline int mcheck_init(void) { return 0; } static inline void mcheck_cpu_init(struct cpuinfo_x86 *c) {} static inline void mcheck_cpu_clear(struct cpuinfo_x86 *c) {} -static inline void mcheck_vendor_init_severity(void) {} static inline int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info, u64 lapic_id) { return -EINVAL; } #endif -#ifdef CONFIG_X86_ANCIENT_MCE -void intel_p5_mcheck_init(struct cpuinfo_x86 *c); -void winchip_mcheck_init(struct cpuinfo_x86 *c); -static inline void enable_p5_mce(void) { mce_p5_enabled = 1; } -#else -static inline void intel_p5_mcheck_init(struct cpuinfo_x86 *c) {} -static inline void winchip_mcheck_init(struct cpuinfo_x86 *c) {} -static inline void enable_p5_mce(void) {} -#endif - void mce_setup(struct mce *m); void mce_log(struct mce *m); DECLARE_PER_CPU(struct device *, mce_device); @@ -325,31 +313,22 @@ enum smca_bank_types { SMCA_SMU, /* System Management Unit */ SMCA_SMU_V2, SMCA_MP5, /* Microprocessor 5 Unit */ + SMCA_MPDMA, /* MPDMA Unit */ SMCA_NBIO, /* Northbridge IO Unit */ SMCA_PCIE, /* PCI Express Unit */ SMCA_PCIE_V2, SMCA_XGMI_PCS, /* xGMI PCS Unit */ + SMCA_NBIF, /* NBIF Unit */ + SMCA_SHUB, /* System HUB Unit */ + SMCA_SATA, /* SATA Unit */ + SMCA_USB, /* USB Unit */ + SMCA_GMI_PCS, /* GMI PCS Unit */ SMCA_XGMI_PHY, /* xGMI PHY Unit */ SMCA_WAFL_PHY, /* WAFL PHY Unit */ + SMCA_GMI_PHY, /* GMI PHY Unit */ N_SMCA_BANK_TYPES }; -#define HWID_MCATYPE(hwid, mcatype) (((hwid) << 16) | (mcatype)) - -struct smca_hwid { - unsigned int bank_type; /* Use with smca_bank_types for easy indexing. */ - u32 hwid_mcatype; /* (hwid,mcatype) tuple */ - u8 count; /* Number of instances. */ -}; - -struct smca_bank { - struct smca_hwid *hwid; - u32 id; /* Value of MCA_IPID[InstanceId]. */ - u8 sysfs_id; /* Value used for sysfs name. */ -}; - -extern struct smca_bank smca_banks[MAX_NR_BANKS]; - extern const char *smca_get_long_name(enum smca_bank_types t); extern bool amd_mce_is_memory_error(struct mce *m); @@ -357,16 +336,13 @@ extern int mce_threshold_create_device(unsigned int cpu); extern int mce_threshold_remove_device(unsigned int cpu); void mce_amd_feature_init(struct cpuinfo_x86 *c); -int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr); - +enum smca_bank_types smca_get_bank_type(unsigned int cpu, unsigned int bank); #else static inline int mce_threshold_create_device(unsigned int cpu) { return 0; }; static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; }; static inline bool amd_mce_is_memory_error(struct mce *m) { return false; }; static inline void mce_amd_feature_init(struct cpuinfo_x86 *c) { } -static inline int -umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) { return -EINVAL; }; #endif static inline void mce_hygon_feature_init(struct cpuinfo_x86 *c) { return mce_amd_feature_init(c); } diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h index 3fb9f5ebef..e2c6f433ed 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -44,6 +44,8 @@ void __init sme_enable(struct boot_params *bp); int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size); int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size); +void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, + bool enc); void __init mem_encrypt_free_decrypted_mem(void); @@ -51,9 +53,6 @@ void __init mem_encrypt_free_decrypted_mem(void); void __init mem_encrypt_init(void); void __init sev_es_init_vc_handling(void); -bool sme_active(void); -bool sev_active(void); -bool sev_es_active(void); #define __bss_decrypted __section(".bss..decrypted") @@ -76,14 +75,13 @@ static inline void __init sme_encrypt_kernel(struct boot_params *bp) { } static inline void __init sme_enable(struct boot_params *bp) { } static inline void sev_es_init_vc_handling(void) { } -static inline bool sme_active(void) { return false; } -static inline bool sev_active(void) { return false; } -static inline bool sev_es_active(void) { return false; } static inline int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0; } static inline int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return 0; } +static inline void __init +early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, bool enc) {} static inline void mem_encrypt_free_decrypted_mem(void) { } @@ -102,11 +100,6 @@ static inline void mem_encrypt_free_decrypted_mem(void) { } extern char __start_bss_decrypted[], __end_bss_decrypted[], __start_bss_decrypted_unused[]; -static inline bool mem_encrypt_active(void) -{ - return sme_me_mask; -} - static inline u64 sme_get_me_mask(void) { return sme_me_mask; diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index ab45a220fa..d6bfdfb0f0 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -130,14 +130,11 @@ static inline unsigned int x86_cpuid_family(void) extern void __init load_ucode_bsp(void); extern void load_ucode_ap(void); void reload_early_microcode(void); -extern bool get_builtin_firmware(struct cpio_data *cd, const char *name); extern bool initrd_gone; #else static inline void __init load_ucode_bsp(void) { } static inline void load_ucode_ap(void) { } static inline void reload_early_microcode(void) { } -static inline bool -get_builtin_firmware(struct cpio_data *cd, const char *name) { return false; } #endif #endif /* _ASM_X86_MICROCODE_H */ diff --git a/arch/x86/include/asm/mmx.h b/arch/x86/include/asm/mmx.h index f572d0f944..e69de29bb2 100644 --- a/arch/x86/include/asm/mmx.h +++ b/arch/x86/include/asm/mmx.h @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_X86_MMX_H -#define _ASM_X86_MMX_H - -/* - * MMX 3Dnow! helper operations - */ - -#include - -extern void *_mmx_memcpy(void *to, const void *from, size_t size); -extern void mmx_clear_page(void *page); -extern void mmx_copy_page(void *to, void *from); - -#endif /* _ASM_X86_MMX_H */ diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index adccbc2091..a82f603d43 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -11,23 +11,14 @@ #include #include +union hv_ghcb; + +DECLARE_STATIC_KEY_FALSE(isolation_type_snp); + typedef int (*hyperv_fill_flush_list_func)( struct hv_guest_mapping_flush_list *flush, void *data); -static inline void hv_set_register(unsigned int reg, u64 value) -{ - wrmsrl(reg, value); -} - -static inline u64 hv_get_register(unsigned int reg) -{ - u64 value; - - rdmsrl(reg, value); - return value; -} - #define hv_get_raw_timer() rdtsc_ordered() void hyperv_vector_handler(struct pt_regs *regs); @@ -39,6 +30,8 @@ extern void *hv_hypercall_pg; extern u64 hv_current_partition_id; +extern union hv_ghcb * __percpu *hv_ghcb_pg; + int hv_call_deposit_pages(int node, u64 partition_id, u32 num_pages); int hv_call_add_logical_proc(int node, u32 lp_index, u32 acpi_id); int hv_call_create_vp(int node, u64 partition_id, u32 vp_index, u32 flags); @@ -176,18 +169,55 @@ bool hv_vcpu_is_preempted(int vcpu); static inline void hv_apic_init(void) {} #endif -static inline void hv_set_msi_entry_from_desc(union hv_msi_entry *msi_entry, - struct msi_desc *msi_desc) -{ - msi_entry->address.as_uint32 = msi_desc->msg.address_lo; - msi_entry->data.as_uint32 = msi_desc->msg.data; -} - struct irq_domain *hv_create_pci_msi_domain(void); int hv_map_ioapic_interrupt(int ioapic_id, bool level, int vcpu, int vector, struct hv_interrupt_entry *entry); int hv_unmap_ioapic_interrupt(int ioapic_id, struct hv_interrupt_entry *entry); +int hv_set_mem_host_visibility(unsigned long addr, int numpages, bool visible); + +#ifdef CONFIG_AMD_MEM_ENCRYPT +void hv_ghcb_msr_write(u64 msr, u64 value); +void hv_ghcb_msr_read(u64 msr, u64 *value); +#else +static inline void hv_ghcb_msr_write(u64 msr, u64 value) {} +static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {} +#endif + +extern bool hv_isolation_type_snp(void); + +static inline bool hv_is_synic_reg(unsigned int reg) +{ + if ((reg >= HV_REGISTER_SCONTROL) && + (reg <= HV_REGISTER_SINT15)) + return true; + return false; +} + +static inline u64 hv_get_register(unsigned int reg) +{ + u64 value; + + if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) + hv_ghcb_msr_read(reg, &value); + else + rdmsrl(reg, value); + return value; +} + +static inline void hv_set_register(unsigned int reg, u64 value) +{ + if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) { + hv_ghcb_msr_write(reg, value); + + /* Write proxy bit via wrmsl instruction */ + if (reg >= HV_REGISTER_SINT0 && + reg <= HV_REGISTER_SINT15) + wrmsrl(reg, value | 1 << 20); + } else { + wrmsrl(reg, value); + } +} #else /* CONFIG_HYPERV */ static inline void hyperv_init(void) {} @@ -205,6 +235,13 @@ static inline int hyperv_flush_guest_mapping_range(u64 as, { return -1; } +static inline void hv_set_register(unsigned int reg, u64 value) { } +static inline u64 hv_get_register(unsigned int reg) { return 0; } +static inline int hv_set_mem_host_visibility(unsigned long addr, int numpages, + bool visible) +{ + return -1; +} #endif /* CONFIG_HYPERV */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index a7c413432b..a4a39c3e0f 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -476,6 +476,7 @@ #define MSR_AMD64_ICIBSEXTDCTL 0xc001103c #define MSR_AMD64_IBSOPDATA4 0xc001103d #define MSR_AMD64_IBS_REG_COUNT_MAX 8 /* includes MSR_AMD64_IBSBRTARGET */ +#define MSR_AMD64_SVM_AVIC_DOORBELL 0xc001011b #define MSR_AMD64_VM_PAGE_FLUSH 0xc001011e #define MSR_AMD64_SEV_ES_GHCB 0xc0010130 #define MSR_AMD64_SEV 0xc0010131 @@ -486,6 +487,23 @@ #define MSR_AMD64_VIRT_SPEC_CTRL 0xc001011f +/* AMD Collaborative Processor Performance Control MSRs */ +#define MSR_AMD_CPPC_CAP1 0xc00102b0 +#define MSR_AMD_CPPC_ENABLE 0xc00102b1 +#define MSR_AMD_CPPC_CAP2 0xc00102b2 +#define MSR_AMD_CPPC_REQ 0xc00102b3 +#define MSR_AMD_CPPC_STATUS 0xc00102b4 + +#define AMD_CPPC_LOWEST_PERF(x) (((x) >> 0) & 0xff) +#define AMD_CPPC_LOWNONLIN_PERF(x) (((x) >> 8) & 0xff) +#define AMD_CPPC_NOMINAL_PERF(x) (((x) >> 16) & 0xff) +#define AMD_CPPC_HIGHEST_PERF(x) (((x) >> 24) & 0xff) + +#define AMD_CPPC_MAX_PERF(x) (((x) & 0xff) << 0) +#define AMD_CPPC_MIN_PERF(x) (((x) & 0xff) << 8) +#define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16) +#define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24) + /* Fam 17h MSRs */ #define MSR_F17H_IRPERF 0xc00000e9 @@ -625,6 +643,8 @@ #define MSR_IA32_BNDCFGS_RSVD 0x00000ffc +#define MSR_IA32_XFD 0x000001c4 +#define MSR_IA32_XFD_ERR 0x000001c5 #define MSR_IA32_XSS 0x00000da0 #define MSR_IA32_APICBASE 0x0000001b diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index a3f87f1015..d42e6c6b47 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -92,7 +92,7 @@ static __always_inline unsigned long long __rdmsr(unsigned int msr) asm volatile("1: rdmsr\n" "2:\n" - _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_rdmsr_unsafe) + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR) : EAX_EDX_RET(val, low, high) : "c" (msr)); return EAX_EDX_VAL(val, low, high); @@ -102,7 +102,7 @@ static __always_inline void __wrmsr(unsigned int msr, u32 low, u32 high) { asm volatile("1: wrmsr\n" "2:\n" - _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_wrmsr_unsafe) + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR) : : "c" (msr), "a"(low), "d" (high) : "memory"); } @@ -137,17 +137,11 @@ static inline unsigned long long native_read_msr_safe(unsigned int msr, { DECLARE_ARGS(val, low, high); - asm volatile("2: rdmsr ; xor %[err],%[err]\n" - "1:\n\t" - ".section .fixup,\"ax\"\n\t" - "3: mov %[fault],%[err]\n\t" - "xorl %%eax, %%eax\n\t" - "xorl %%edx, %%edx\n\t" - "jmp 1b\n\t" - ".previous\n\t" - _ASM_EXTABLE(2b, 3b) + asm volatile("1: rdmsr ; xor %[err],%[err]\n" + "2:\n\t" + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_RDMSR_SAFE, %[err]) : [err] "=r" (*err), EAX_EDX_RET(val, low, high) - : "c" (msr), [fault] "i" (-EIO)); + : "c" (msr)); if (tracepoint_enabled(read_msr)) do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err); return EAX_EDX_VAL(val, low, high); @@ -169,15 +163,11 @@ native_write_msr_safe(unsigned int msr, u32 low, u32 high) { int err; - asm volatile("2: wrmsr ; xor %[err],%[err]\n" - "1:\n\t" - ".section .fixup,\"ax\"\n\t" - "3: mov %[fault],%[err] ; jmp 1b\n\t" - ".previous\n\t" - _ASM_EXTABLE(2b, 3b) + asm volatile("1: wrmsr ; xor %[err],%[err]\n" + "2:\n\t" + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_WRMSR_SAFE, %[err]) : [err] "=a" (err) - : "c" (msr), "0" (low), "d" (high), - [fault] "i" (-EIO) + : "c" (msr), "0" (low), "d" (high) : "memory"); if (tracepoint_enabled(write_msr)) do_trace_write_msr(msr, ((u64)high << 32 | low), err); diff --git a/arch/x86/include/asm/mtrr.h b/arch/x86/include/asm/mtrr.h index 829df26fd7..76d726074c 100644 --- a/arch/x86/include/asm/mtrr.h +++ b/arch/x86/include/asm/mtrr.h @@ -24,8 +24,8 @@ #define _ASM_X86_MTRR_H #include -#include +void mtrr_bp_init(void); /* * The following functions are for use by other drivers that cannot use @@ -43,7 +43,6 @@ extern int mtrr_del(int reg, unsigned long base, unsigned long size); extern int mtrr_del_page(int reg, unsigned long base, unsigned long size); extern void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi); extern void mtrr_ap_init(void); -extern void mtrr_bp_init(void); extern void set_mtrr_aps_delayed_init(void); extern void mtrr_aps_init(void); extern void mtrr_bp_restore(void); @@ -84,11 +83,6 @@ static inline int mtrr_trim_uncached_memory(unsigned long end_pfn) static inline void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi) { } -static inline void mtrr_bp_init(void) -{ - pat_disable("PAT support disabled because CONFIG_MTRR is disabled in the kernel."); -} - #define mtrr_ap_init() do {} while (0) #define set_mtrr_aps_delayed_init() do {} while (0) #define mtrr_aps_init() do {} while (0) diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index b3dd514f80..acbaeaf83b 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -5,12 +5,15 @@ #include #include +#include #include #include #include #include +#define RETPOLINE_THUNK_SIZE 32 + /* * Fill the CPU return stack buffer. * @@ -118,6 +121,16 @@ ".popsection\n\t" #ifdef CONFIG_RETPOLINE + +typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; + +#define GEN(reg) \ + extern retpoline_thunk_t __x86_indirect_thunk_ ## reg; +#include +#undef GEN + +extern retpoline_thunk_t __x86_indirect_thunk_array[]; + #ifdef CONFIG_X86_64 /* @@ -305,63 +318,4 @@ static inline void mds_idle_clear_cpu_buffers(void) #endif /* __ASSEMBLY__ */ -/* - * Below is used in the eBPF JIT compiler and emits the byte sequence - * for the following assembly: - * - * With retpolines configured: - * - * callq do_rop - * spec_trap: - * pause - * lfence - * jmp spec_trap - * do_rop: - * mov %rcx,(%rsp) for x86_64 - * mov %edx,(%esp) for x86_32 - * retq - * - * Without retpolines configured: - * - * jmp *%rcx for x86_64 - * jmp *%edx for x86_32 - */ -#ifdef CONFIG_RETPOLINE -# ifdef CONFIG_X86_64 -# define RETPOLINE_RCX_BPF_JIT_SIZE 17 -# define RETPOLINE_RCX_BPF_JIT() \ -do { \ - EMIT1_off32(0xE8, 7); /* callq do_rop */ \ - /* spec_trap: */ \ - EMIT2(0xF3, 0x90); /* pause */ \ - EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ - EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ - /* do_rop: */ \ - EMIT4(0x48, 0x89, 0x0C, 0x24); /* mov %rcx,(%rsp) */ \ - EMIT1(0xC3); /* retq */ \ -} while (0) -# else /* !CONFIG_X86_64 */ -# define RETPOLINE_EDX_BPF_JIT() \ -do { \ - EMIT1_off32(0xE8, 7); /* call do_rop */ \ - /* spec_trap: */ \ - EMIT2(0xF3, 0x90); /* pause */ \ - EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ - EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ - /* do_rop: */ \ - EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */ \ - EMIT1(0xC3); /* ret */ \ -} while (0) -# endif -#else /* !CONFIG_RETPOLINE */ -# ifdef CONFIG_X86_64 -# define RETPOLINE_RCX_BPF_JIT_SIZE 2 -# define RETPOLINE_RCX_BPF_JIT() \ - EMIT2(0xFF, 0xE1); /* jmp *%rcx */ -# else /* !CONFIG_X86_64 */ -# define RETPOLINE_EDX_BPF_JIT() \ - EMIT2(0xFF, 0xE2) /* jmp *%edx */ -# endif -#endif - #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ diff --git a/arch/x86/include/asm/page_32.h b/arch/x86/include/asm/page_32.h index 94dbd51df5..df42f8aa99 100644 --- a/arch/x86/include/asm/page_32.h +++ b/arch/x86/include/asm/page_32.h @@ -19,19 +19,6 @@ extern unsigned long __phys_addr(unsigned long); #define pfn_valid(pfn) ((pfn) < max_mapnr) #endif /* CONFIG_FLATMEM */ -#ifdef CONFIG_X86_USE_3DNOW -#include - -static inline void clear_page(void *page) -{ - mmx_clear_page(page); -} - -static inline void copy_page(void *to, void *from) -{ - mmx_copy_page(to, from); -} -#else /* !CONFIG_X86_USE_3DNOW */ #include static inline void clear_page(void *page) @@ -43,7 +30,6 @@ static inline void copy_page(void *to, void *from) { memcpy(to, from, PAGE_SIZE); } -#endif /* CONFIG_X86_3DNOW */ #endif /* !__ASSEMBLY__ */ #endif /* _ASM_X86_PAGE_32_H */ diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h index 4bde0dc661..e9c86299b8 100644 --- a/arch/x86/include/asm/page_64.h +++ b/arch/x86/include/asm/page_64.h @@ -5,6 +5,7 @@ #include #ifndef __ASSEMBLY__ +#include #include /* duplicated to the one in bootmem.h */ diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index da3a1ac82b..0d76502cc6 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -52,11 +52,11 @@ void __init paravirt_set_cap(void); /* The paravirtualized I/O functions */ static inline void slow_down_io(void) { - pv_ops.cpu.io_delay(); + PVOP_VCALL0(cpu.io_delay); #ifdef REALLY_SLOW_IO - pv_ops.cpu.io_delay(); - pv_ops.cpu.io_delay(); - pv_ops.cpu.io_delay(); + PVOP_VCALL0(cpu.io_delay); + PVOP_VCALL0(cpu.io_delay); + PVOP_VCALL0(cpu.io_delay); #endif } @@ -97,6 +97,12 @@ static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) PVOP_VCALL1(mmu.exit_mmap, mm); } +static inline void notify_page_enc_status_changed(unsigned long pfn, + int npages, bool enc) +{ + PVOP_VCALL3(mmu.notify_page_enc_status_changed, pfn, npages, enc); +} + #ifdef CONFIG_PARAVIRT_XXL static inline void load_sp0(unsigned long sp0) { @@ -113,12 +119,12 @@ static inline void __cpuid(unsigned int *eax, unsigned int *ebx, /* * These special macros can be used to get or set a debugging register */ -static inline unsigned long paravirt_get_debugreg(int reg) +static __always_inline unsigned long paravirt_get_debugreg(int reg) { return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg); } #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) -static inline void set_debugreg(unsigned long val, int reg) +static __always_inline void set_debugreg(unsigned long val, int reg) { PVOP_VCALL2(cpu.set_debugreg, reg, val); } @@ -133,14 +139,14 @@ static inline void write_cr0(unsigned long x) PVOP_VCALL1(cpu.write_cr0, x); } -static inline unsigned long read_cr2(void) +static __always_inline unsigned long read_cr2(void) { return PVOP_ALT_CALLEE0(unsigned long, mmu.read_cr2, "mov %%cr2, %%rax;", ALT_NOT(X86_FEATURE_XENPV)); } -static inline void write_cr2(unsigned long x) +static __always_inline void write_cr2(unsigned long x) { PVOP_VCALL1(mmu.write_cr2, x); } @@ -653,10 +659,10 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); * functions. */ #define PV_THUNK_NAME(func) "__raw_callee_save_" #func -#define PV_CALLEE_SAVE_REGS_THUNK(func) \ +#define __PV_CALLEE_SAVE_REGS_THUNK(func, section) \ extern typeof(func) __raw_callee_save_##func; \ \ - asm(".pushsection .text;" \ + asm(".pushsection " section ", \"ax\";" \ ".globl " PV_THUNK_NAME(func) ";" \ ".type " PV_THUNK_NAME(func) ", @function;" \ PV_THUNK_NAME(func) ":" \ @@ -665,10 +671,13 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); "call " #func ";" \ PV_RESTORE_ALL_CALLER_REGS \ FRAME_END \ - "ret;" \ + ASM_RET \ ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \ ".popsection") +#define PV_CALLEE_SAVE_REGS_THUNK(func) \ + __PV_CALLEE_SAVE_REGS_THUNK(func, ".text") + /* Get a reference to a callee-save function */ #define PV_CALLEE_SAVE(func) \ ((struct paravirt_callee_save) { __raw_callee_save_##func }) @@ -678,23 +687,23 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); ((struct paravirt_callee_save) { func }) #ifdef CONFIG_PARAVIRT_XXL -static inline notrace unsigned long arch_local_save_flags(void) +static __always_inline unsigned long arch_local_save_flags(void) { return PVOP_ALT_CALLEE0(unsigned long, irq.save_fl, "pushf; pop %%rax;", ALT_NOT(X86_FEATURE_XENPV)); } -static inline notrace void arch_local_irq_disable(void) +static __always_inline void arch_local_irq_disable(void) { PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT(X86_FEATURE_XENPV)); } -static inline notrace void arch_local_irq_enable(void) +static __always_inline void arch_local_irq_enable(void) { PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT(X86_FEATURE_XENPV)); } -static inline notrace unsigned long arch_local_irq_save(void) +static __always_inline unsigned long arch_local_irq_save(void) { unsigned long f; @@ -743,11 +752,6 @@ extern void default_banner(void); #define PARA_SITE(ptype, ops) _PVSITE(ptype, ops, .quad, 8) #define PARA_INDIRECT(addr) *addr(%rip) -#define INTERRUPT_RETURN \ - ANNOTATE_RETPOLINE_SAFE; \ - ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);", \ - X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;") - #ifdef CONFIG_DEBUG_ENTRY .macro PARA_IRQ_save_fl PARA_SITE(PARA_PATCH(PV_IRQ_save_fl), diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index d9d6b0203e..a69012e190 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -168,6 +168,7 @@ struct pv_mmu_ops { /* Hook for intercepting the destruction of an mm_struct. */ void (*exit_mmap)(struct mm_struct *mm); + void (*notify_page_enc_status_changed)(unsigned long pfn, int npages, bool enc); #ifdef CONFIG_PARAVIRT_XXL struct paravirt_callee_save read_cr2; @@ -577,7 +578,9 @@ void paravirt_leave_lazy_mmu(void); void paravirt_flush_lazy_mmu(void); void _paravirt_nop(void); +void paravirt_BUG(void); u64 _paravirt_ident_64(u64); +unsigned long paravirt_ret0(void); #define paravirt_nop ((void *)_paravirt_nop) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 448cd01eb3..8a9432fb38 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -22,10 +22,12 @@ #define pgprot_decrypted(prot) __pgprot(__sme_clr(pgprot_val(prot))) #ifndef __ASSEMBLY__ +#include #include #include #include #include +#include extern pgd_t early_top_pgt[PTRS_PER_PGD]; bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd); @@ -752,7 +754,7 @@ static inline bool pte_accessible(struct mm_struct *mm, pte_t a) return true; if ((pte_flags(a) & _PAGE_PROTNONE) && - mm_tlb_flush_pending(mm)) + atomic_read(&mm->tlb_flush_pending)) return true; return false; @@ -1006,18 +1008,21 @@ static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp) static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { + page_table_check_pte_set(mm, addr, ptep, pte); set_pte(ptep, pte); } static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { + page_table_check_pmd_set(mm, addr, pmdp, pmd); set_pmd(pmdp, pmd); } static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, pud_t *pudp, pud_t pud) { + page_table_check_pud_set(mm, addr, pudp, pud); native_set_pud(pudp, pud); } @@ -1048,6 +1053,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { pte_t pte = native_ptep_get_and_clear(ptep); + page_table_check_pte_clear(mm, addr, pte); return pte; } @@ -1063,12 +1069,23 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, * care about updates and native needs no locking */ pte = native_local_ptep_get_and_clear(ptep); + page_table_check_pte_clear(mm, addr, pte); } else { pte = ptep_get_and_clear(mm, addr, ptep); } return pte; } +#define __HAVE_ARCH_PTEP_CLEAR +static inline void ptep_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + if (IS_ENABLED(CONFIG_PAGE_TABLE_CHECK)) + ptep_get_and_clear(mm, addr, ptep); + else + pte_clear(mm, addr, ptep); +} + #define __HAVE_ARCH_PTEP_SET_WRPROTECT static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) @@ -1109,14 +1126,22 @@ static inline int pmd_write(pmd_t pmd) static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { - return native_pmdp_get_and_clear(pmdp); + pmd_t pmd = native_pmdp_get_and_clear(pmdp); + + page_table_check_pmd_clear(mm, addr, pmd); + + return pmd; } #define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pud_t *pudp) { - return native_pudp_get_and_clear(pudp); + pud_t pud = native_pudp_get_and_clear(pudp); + + page_table_check_pud_clear(mm, addr, pud); + + return pud; } #define __HAVE_ARCH_PMDP_SET_WRPROTECT @@ -1137,6 +1162,7 @@ static inline int pud_write(pud_t pud) static inline pmd_t pmdp_establish(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd) { + page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd); if (IS_ENABLED(CONFIG_SMP)) { return xchg(pmdp, pmd); } else { diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 577f342dbf..2c5f12ae7d 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -164,7 +164,8 @@ enum cpuid_regs_idx { #define X86_VENDOR_NSC 8 #define X86_VENDOR_HYGON 9 #define X86_VENDOR_ZHAOXIN 10 -#define X86_VENDOR_NUM 11 +#define X86_VENDOR_VORTEX 11 +#define X86_VENDOR_NUM 12 #define X86_VENDOR_UNKNOWN 0xff @@ -461,9 +462,6 @@ DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr); #endif /* !X86_64 */ -extern unsigned int fpu_kernel_xstate_size; -extern unsigned int fpu_user_xstate_size; - struct perf_event; struct thread_struct { @@ -538,12 +536,12 @@ struct thread_struct { */ }; -/* Whitelist the FPU state from the task_struct for hardened usercopy. */ +extern void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size); + static inline void arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size) { - *offset = offsetof(struct thread_struct, fpu.state); - *size = fpu_kernel_xstate_size; + fpu_thread_struct_whitelist(offset, size); } static inline void @@ -590,7 +588,7 @@ static inline void load_sp0(unsigned long sp0) /* Free all resources held by a thread. */ extern void release_thread(struct task_struct *); -unsigned long get_wchan(struct task_struct *p); +unsigned long __get_wchan(struct task_struct *p); /* * Generic CPUID function @@ -808,11 +806,14 @@ static inline u32 amd_get_nodes_per_socket(void) { return 0; } static inline u32 amd_get_highest_perf(void) { return 0; } #endif +#define for_each_possible_hypervisor_cpuid_base(function) \ + for (function = 0x40000000; function < 0x40010000; function += 0x100) + static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves) { uint32_t base, eax, signature[3]; - for (base = 0x40000000; base < 0x40010000; base += 0x100) { + for_each_possible_hypervisor_cpuid_base(base) { cpuid(base, &eax, &signature[0], &signature[1], &signature[2]); if (!memcmp(sig, signature, 12) && @@ -854,4 +855,12 @@ enum mds_mitigations { MDS_MITIGATION_VMWERV, }; +#ifdef CONFIG_X86_SGX +int arch_memory_failure(unsigned long pfn, int flags); +#define arch_memory_failure arch_memory_failure + +bool arch_is_platform_page(u64 paddr); +#define arch_is_platform_page arch_is_platform_page +#endif + #endif /* _ASM_X86_PROCESSOR_H */ diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h index 8c5d1910a8..feed36d44d 100644 --- a/arch/x86/include/asm/proto.h +++ b/arch/x86/include/asm/proto.h @@ -40,6 +40,6 @@ void x86_report_nx(void); extern int reboot_force; long do_arch_prctl_common(struct task_struct *task, int option, - unsigned long cpuid_enabled); + unsigned long arg2); #endif /* _ASM_X86_PROTO_H */ diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h index b94f615600..703663175a 100644 --- a/arch/x86/include/asm/ptrace.h +++ b/arch/x86/include/asm/ptrace.h @@ -181,7 +181,7 @@ static inline bool any_64bit_mode(struct pt_regs *regs) #define current_user_stack_pointer() current_pt_regs()->sp #define compat_user_stack_pointer() current_pt_regs()->sp -static inline bool ip_within_syscall_gap(struct pt_regs *regs) +static __always_inline bool ip_within_syscall_gap(struct pt_regs *regs) { bool ret = (regs->ip >= (unsigned long)entry_SYSCALL_64 && regs->ip < (unsigned long)entry_SYSCALL_64_safe_stack); diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h index d86ab94221..d87451df48 100644 --- a/arch/x86/include/asm/qspinlock.h +++ b/arch/x86/include/asm/qspinlock.h @@ -53,6 +53,7 @@ static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) static inline void queued_spin_unlock(struct qspinlock *lock) { + kcsan_release(); pv_queued_spin_unlock(lock); } diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h index 159622ee06..1474cf9625 100644 --- a/arch/x86/include/asm/qspinlock_paravirt.h +++ b/arch/x86/include/asm/qspinlock_paravirt.h @@ -48,7 +48,7 @@ asm (".pushsection .text;" "jne .slowpath;" "pop %rdx;" FRAME_END - "ret;" + ASM_RET ".slowpath: " "push %rsi;" "movzbl %al,%esi;" @@ -56,7 +56,7 @@ asm (".pushsection .text;" "pop %rsi;" "pop %rdx;" FRAME_END - "ret;" + ASM_RET ".size " PV_UNLOCK ", .-" PV_UNLOCK ";" ".popsection"); diff --git a/arch/x86/include/asm/required-features.h b/arch/x86/include/asm/required-features.h index b2d504f119..aff774775c 100644 --- a/arch/x86/include/asm/required-features.h +++ b/arch/x86/include/asm/required-features.h @@ -35,11 +35,7 @@ # define NEED_CMOV 0 #endif -#ifdef CONFIG_X86_USE_3DNOW -# define NEED_3DNOW (1<<(X86_FEATURE_3DNOW & 31)) -#else # define NEED_3DNOW 0 -#endif #if defined(CONFIG_X86_P6_NOP) || defined(CONFIG_X86_64) # define NEED_NOPL (1<<(X86_FEATURE_NOPL & 31)) diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h index 72044026eb..b228c9d44e 100644 --- a/arch/x86/include/asm/segment.h +++ b/arch/x86/include/asm/segment.h @@ -307,14 +307,7 @@ do { \ \ asm volatile(" \n" \ "1: movl %k0,%%" #seg " \n" \ - \ - ".section .fixup,\"ax\" \n" \ - "2: xorl %k0,%k0 \n" \ - " jmp 1b \n" \ - ".previous \n" \ - \ - _ASM_EXTABLE(1b, 2b) \ - \ + _ASM_EXTABLE_TYPE_REG(1b, 1b, EX_TYPE_ZERO_REG, %k0)\ : "+r" (__val) : : "memory"); \ } while (0) @@ -339,7 +332,7 @@ static inline void __loadsegment_fs(unsigned short value) "1: movw %0, %%fs \n" "2: \n" - _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_clear_fs) + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_CLEAR_FS) : : "rm" (value) : "memory"); } diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index 43fa081a1a..ff0f2d9033 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -2,6 +2,7 @@ #ifndef _ASM_X86_SET_MEMORY_H #define _ASM_X86_SET_MEMORY_H +#include #include #include @@ -83,6 +84,7 @@ int set_pages_rw(struct page *page, int numpages); int set_direct_map_invalid_noflush(struct page *page); int set_direct_map_default_noflush(struct page *page); bool kernel_page_present(struct page *page); +void notify_range_enc_status_changed(unsigned long vaddr, int npages, bool enc); extern int kernel_set_to_readonly; @@ -98,6 +100,9 @@ static inline int set_mce_nospec(unsigned long pfn, bool unmap) unsigned long decoy_addr; int rc; + /* SGX pages are not in the 1:1 map */ + if (arch_is_platform_page(pfn << PAGE_SHIFT)) + return 0; /* * We would like to just call: * set_memory_XX((unsigned long)pfn_to_kaddr(pfn), 1); diff --git a/arch/x86/include/asm/sev-common.h b/arch/x86/include/asm/sev-common.h index 2cef6c5a52..1b2fd32b42 100644 --- a/arch/x86/include/asm/sev-common.h +++ b/arch/x86/include/asm/sev-common.h @@ -18,20 +18,19 @@ /* SEV Information Request/Response */ #define GHCB_MSR_SEV_INFO_RESP 0x001 #define GHCB_MSR_SEV_INFO_REQ 0x002 -#define GHCB_MSR_VER_MAX_POS 48 -#define GHCB_MSR_VER_MAX_MASK 0xffff -#define GHCB_MSR_VER_MIN_POS 32 -#define GHCB_MSR_VER_MIN_MASK 0xffff -#define GHCB_MSR_CBIT_POS 24 -#define GHCB_MSR_CBIT_MASK 0xff -#define GHCB_MSR_SEV_INFO(_max, _min, _cbit) \ - ((((_max) & GHCB_MSR_VER_MAX_MASK) << GHCB_MSR_VER_MAX_POS) | \ - (((_min) & GHCB_MSR_VER_MIN_MASK) << GHCB_MSR_VER_MIN_POS) | \ - (((_cbit) & GHCB_MSR_CBIT_MASK) << GHCB_MSR_CBIT_POS) | \ + +#define GHCB_MSR_SEV_INFO(_max, _min, _cbit) \ + /* GHCBData[63:48] */ \ + ((((_max) & 0xffff) << 48) | \ + /* GHCBData[47:32] */ \ + (((_min) & 0xffff) << 32) | \ + /* GHCBData[31:24] */ \ + (((_cbit) & 0xff) << 24) | \ GHCB_MSR_SEV_INFO_RESP) + #define GHCB_MSR_INFO(v) ((v) & 0xfffUL) -#define GHCB_MSR_PROTO_MAX(v) (((v) >> GHCB_MSR_VER_MAX_POS) & GHCB_MSR_VER_MAX_MASK) -#define GHCB_MSR_PROTO_MIN(v) (((v) >> GHCB_MSR_VER_MIN_POS) & GHCB_MSR_VER_MIN_MASK) +#define GHCB_MSR_PROTO_MAX(v) (((v) >> 48) & 0xffff) +#define GHCB_MSR_PROTO_MIN(v) (((v) >> 32) & 0xffff) /* CPUID Request/Response */ #define GHCB_MSR_CPUID_REQ 0x004 @@ -46,31 +45,48 @@ #define GHCB_CPUID_REQ_EBX 1 #define GHCB_CPUID_REQ_ECX 2 #define GHCB_CPUID_REQ_EDX 3 -#define GHCB_CPUID_REQ(fn, reg) \ - (GHCB_MSR_CPUID_REQ | \ - (((unsigned long)reg & GHCB_MSR_CPUID_REG_MASK) << GHCB_MSR_CPUID_REG_POS) | \ - (((unsigned long)fn) << GHCB_MSR_CPUID_FUNC_POS)) +#define GHCB_CPUID_REQ(fn, reg) \ + /* GHCBData[11:0] */ \ + (GHCB_MSR_CPUID_REQ | \ + /* GHCBData[31:12] */ \ + (((unsigned long)(reg) & 0x3) << 30) | \ + /* GHCBData[63:32] */ \ + (((unsigned long)fn) << 32)) /* AP Reset Hold */ -#define GHCB_MSR_AP_RESET_HOLD_REQ 0x006 -#define GHCB_MSR_AP_RESET_HOLD_RESP 0x007 +#define GHCB_MSR_AP_RESET_HOLD_REQ 0x006 +#define GHCB_MSR_AP_RESET_HOLD_RESP 0x007 /* GHCB Hypervisor Feature Request/Response */ -#define GHCB_MSR_HV_FT_REQ 0x080 -#define GHCB_MSR_HV_FT_RESP 0x081 +#define GHCB_MSR_HV_FT_REQ 0x080 +#define GHCB_MSR_HV_FT_RESP 0x081 #define GHCB_MSR_TERM_REQ 0x100 #define GHCB_MSR_TERM_REASON_SET_POS 12 #define GHCB_MSR_TERM_REASON_SET_MASK 0xf #define GHCB_MSR_TERM_REASON_POS 16 #define GHCB_MSR_TERM_REASON_MASK 0xff -#define GHCB_SEV_TERM_REASON(reason_set, reason_val) \ - (((((u64)reason_set) & GHCB_MSR_TERM_REASON_SET_MASK) << GHCB_MSR_TERM_REASON_SET_POS) | \ - ((((u64)reason_val) & GHCB_MSR_TERM_REASON_MASK) << GHCB_MSR_TERM_REASON_POS)) -#define GHCB_SEV_ES_REASON_GENERAL_REQUEST 0 -#define GHCB_SEV_ES_REASON_PROTOCOL_UNSUPPORTED 1 +#define GHCB_SEV_TERM_REASON(reason_set, reason_val) \ + /* GHCBData[15:12] */ \ + (((((u64)reason_set) & 0xf) << 12) | \ + /* GHCBData[23:16] */ \ + ((((u64)reason_val) & 0xff) << 16)) + +#define GHCB_SEV_ES_GEN_REQ 0 +#define GHCB_SEV_ES_PROT_UNSUPPORTED 1 #define GHCB_RESP_CODE(v) ((v) & GHCB_MSR_INFO_MASK) +/* + * Error codes related to GHCB input that can be communicated back to the guest + * by setting the lower 32-bits of the GHCB SW_EXITINFO1 field to 2. + */ +#define GHCB_ERR_NOT_REGISTERED 1 +#define GHCB_ERR_INVALID_USAGE 2 +#define GHCB_ERR_INVALID_SCRATCH_AREA 3 +#define GHCB_ERR_MISSING_INPUT 4 +#define GHCB_ERR_INVALID_INPUT 5 +#define GHCB_ERR_INVALID_EVENT 6 + #endif diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index fa5cd05d3b..ec060c4335 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -53,6 +53,7 @@ static inline u64 lower_bits(u64 val, unsigned int bits) struct real_mode_header; enum stack_type; +struct ghcb; /* Early IDT entry points for #VC handler */ extern void vc_no_ghcb(void); @@ -81,6 +82,11 @@ static __always_inline void sev_es_nmi_complete(void) __sev_es_nmi_complete(); } extern int __init sev_es_efi_map_ghcbs(pgd_t *pgd); +extern enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, + bool set_ghcb_msr, + struct es_em_ctxt *ctxt, + u64 exit_code, u64 exit_info_1, + u64 exit_info_2); #else static inline void sev_es_ist_enter(struct pt_regs *regs) { } static inline void sev_es_ist_exit(void) { } diff --git a/arch/x86/include/asm/sgx.h b/arch/x86/include/asm/sgx.h index 05f3e21f01..3f9334ef67 100644 --- a/arch/x86/include/asm/sgx.h +++ b/arch/x86/include/asm/sgx.h @@ -45,6 +45,24 @@ enum sgx_encls_function { EMODT = 0x0F, }; +/** + * SGX_ENCLS_FAULT_FLAG - flag signifying an ENCLS return code is a trapnr + * + * ENCLS has its own (positive value) error codes and also generates + * ENCLS specific #GP and #PF faults. And the ENCLS values get munged + * with system error codes as everything percolates back up the stack. + * Unfortunately (for us), we need to precisely identify each unique + * error code, e.g. the action taken if EWB fails varies based on the + * type of fault and on the exact SGX error code, i.e. we can't simply + * convert all faults to -EFAULT. + * + * To make all three error types coexist, we set bit 30 to identify an + * ENCLS fault. Bit 31 (technically bits N:31) is used to differentiate + * between positive (faults and SGX error codes) and negative (system + * error codes) values. + */ +#define SGX_ENCLS_FAULT_FLAG 0x40000000 + /** * enum sgx_return_code - The return code type for ENCLS, ENCLU and ENCLV * %SGX_NOT_TRACKED: Previous ETRACK's shootdown sequence has not diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 630ff08532..81a0211a37 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -16,7 +16,9 @@ DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map); DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_die_map); /* cpus sharing the last level cache: */ DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map); +DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_l2c_shared_map); DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id); +DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id); DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number); static inline struct cpumask *cpu_llc_shared_mask(int cpu) @@ -24,6 +26,11 @@ static inline struct cpumask *cpu_llc_shared_mask(int cpu) return per_cpu(cpu_llc_shared_map, cpu); } +static inline struct cpumask *cpu_l2c_shared_mask(int cpu) +{ + return per_cpu(cpu_l2c_shared_map, cpu); +} + DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid); DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid); DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid); @@ -119,6 +126,7 @@ static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask) void cpu_disable_common(void); void native_smp_prepare_boot_cpu(void); +void smp_prepare_cpus_common(void); void native_smp_prepare_cpus(unsigned int max_cpus); void calculate_max_logical_packages(void); void native_smp_cpus_done(unsigned int max_cpus); diff --git a/arch/x86/include/asm/static_call.h b/arch/x86/include/asm/static_call.h index cbb67b6030..ed4f8bb6c2 100644 --- a/arch/x86/include/asm/static_call.h +++ b/arch/x86/include/asm/static_call.h @@ -27,6 +27,7 @@ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ STATIC_CALL_TRAMP_STR(name) ": \n" \ insns " \n" \ + ".byte 0x53, 0x43, 0x54 \n" \ ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \ ".popsection \n") @@ -35,7 +36,7 @@ __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)") #define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \ - __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop") + __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; int3; nop; nop; nop") #define ARCH_ADD_TRAMP_KEY(name) \ diff --git a/arch/x86/include/asm/string_32.h b/arch/x86/include/asm/string_32.h index f74362b056..32c0d981a8 100644 --- a/arch/x86/include/asm/string_32.h +++ b/arch/x86/include/asm/string_32.h @@ -146,42 +146,9 @@ static __always_inline void *__constant_memcpy(void *to, const void *from, extern void *memcpy(void *, const void *, size_t); #ifndef CONFIG_FORTIFY_SOURCE -#ifdef CONFIG_X86_USE_3DNOW - -#include - -/* - * This CPU favours 3DNow strongly (eg AMD Athlon) - */ - -static inline void *__constant_memcpy3d(void *to, const void *from, size_t len) -{ - if (len < 512) - return __constant_memcpy(to, from, len); - return _mmx_memcpy(to, from, len); -} - -static inline void *__memcpy3d(void *to, const void *from, size_t len) -{ - if (len < 512) - return __memcpy(to, from, len); - return _mmx_memcpy(to, from, len); -} - -#define memcpy(t, f, n) \ - (__builtin_constant_p((n)) \ - ? __constant_memcpy3d((t), (f), (n)) \ - : __memcpy3d((t), (f), (n))) - -#else - -/* - * No 3D Now! - */ #define memcpy(t, f, n) __builtin_memcpy(t, f, n) -#endif #endif /* !CONFIG_FORTIFY_SOURCE */ #define __HAVE_ARCH_MEMMOVE diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index b00dbc5fac..bb2fb78523 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -220,6 +220,42 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_NESTED_CTL_SEV_ENABLE BIT(1) #define SVM_NESTED_CTL_SEV_ES_ENABLE BIT(2) + +/* AVIC */ +#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF) +#define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31 +#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31) + +#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK (0xFFULL) +#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12) +#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62) +#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63) +#define AVIC_PHYSICAL_ID_TABLE_SIZE_MASK (0xFF) + +#define AVIC_DOORBELL_PHYSICAL_ID_MASK (0xFF) + +#define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 +#define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 +#define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF + +enum avic_ipi_failure_cause { + AVIC_IPI_FAILURE_INVALID_INT_TYPE, + AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, + AVIC_IPI_FAILURE_INVALID_TARGET, + AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, +}; + + +/* + * 0xff is broadcast, so the max index allowed for physical APIC ID + * table is 0xfe. APIC IDs above 0xff are reserved. + */ +#define AVIC_MAX_PHYSICAL_ID_COUNT 0xff + +#define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) +#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL + + struct vmcb_seg { u16 selector; u16 attrib; diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h index f7e2d82d24..5b85987a5e 100644 --- a/arch/x86/include/asm/syscall.h +++ b/arch/x86/include/asm/syscall.h @@ -87,15 +87,6 @@ static inline void syscall_get_arguments(struct task_struct *task, memcpy(args, ®s->bx, 6 * sizeof(args[0])); } -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - unsigned int i, unsigned int n, - const unsigned long *args) -{ - BUG_ON(i + n > 6); - memcpy(®s->bx + i, args, n * sizeof(args[0])); -} - static inline int syscall_get_arch(struct task_struct *task) { return AUDIT_ARCH_I386; @@ -127,30 +118,6 @@ static inline void syscall_get_arguments(struct task_struct *task, } } -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - const unsigned long *args) -{ -# ifdef CONFIG_IA32_EMULATION - if (task->thread_info.status & TS_COMPAT) { - regs->bx = *args++; - regs->cx = *args++; - regs->dx = *args++; - regs->si = *args++; - regs->di = *args++; - regs->bp = *args; - } else -# endif - { - regs->di = *args++; - regs->si = *args++; - regs->dx = *args++; - regs->r10 = *args++; - regs->r8 = *args++; - regs->r9 = *args; - } -} - static inline int syscall_get_arch(struct task_struct *task) { /* x32 tasks should be considered AUDIT_ARCH_X86_64. */ diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index cf132663c2..ebec69c35e 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -57,6 +57,9 @@ struct thread_info { unsigned long flags; /* low level flags */ unsigned long syscall_work; /* SYSCALL_WORK_ flags */ u32 status; /* thread synchronous flags */ +#ifdef CONFIG_SMP + u32 cpu; /* current CPU */ +#endif }; #define INIT_THREAD_INFO(tsk) \ diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index b587a9ee9c..98fa0a1140 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -261,4 +261,9 @@ extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch); #endif /* !MODULE */ +static inline void __native_tlb_flush_global(unsigned long cr4) +{ + native_write_cr4(cr4 ^ X86_CR4_PGE); + native_write_cr4(cr4); +} #endif /* _ASM_X86_TLBFLUSH_H */ diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 55160445ea..2f0b6be8ea 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -103,6 +103,7 @@ static inline void setup_node_to_cpumask_map(void) { } #include extern const struct cpumask *cpu_coregroup_mask(int cpu); +extern const struct cpumask *cpu_clustergroup_mask(int cpu); #define topology_logical_package_id(cpu) (cpu_data(cpu).logical_proc_id) #define topology_physical_package_id(cpu) (cpu_data(cpu).phys_proc_id) @@ -113,7 +114,9 @@ extern const struct cpumask *cpu_coregroup_mask(int cpu); extern unsigned int __max_die_per_package; #ifdef CONFIG_SMP +#define topology_cluster_id(cpu) (per_cpu(cpu_l2c_id, cpu)) #define topology_die_cpumask(cpu) (per_cpu(cpu_die_map, cpu)) +#define topology_cluster_cpumask(cpu) (cpu_clustergroup_mask(cpu)) #define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu)) #define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu)) diff --git a/arch/x86/include/asm/trace/fpu.h b/arch/x86/include/asm/trace/fpu.h index 879b77792f..4645a63340 100644 --- a/arch/x86/include/asm/trace/fpu.h +++ b/arch/x86/include/asm/trace/fpu.h @@ -22,8 +22,8 @@ DECLARE_EVENT_CLASS(x86_fpu, __entry->fpu = fpu; __entry->load_fpu = test_thread_flag(TIF_NEED_FPU_LOAD); if (boot_cpu_has(X86_FEATURE_OSXSAVE)) { - __entry->xfeatures = fpu->state.xsave.header.xfeatures; - __entry->xcomp_bv = fpu->state.xsave.header.xcomp_bv; + __entry->xfeatures = fpu->fpstate->regs.xsave.header.xfeatures; + __entry->xcomp_bv = fpu->fpstate->regs.xsave.header.xcomp_bv; } ), TP_printk("x86/fpu: %p load: %d xfeatures: %llx xcomp_bv: %llx", diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index bb1430283c..ac96f9b2d6 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -352,24 +352,22 @@ do { \ "1: movl %[lowbits],%%eax\n" \ "2: movl %[highbits],%%edx\n" \ "3:\n" \ - ".section .fixup,\"ax\"\n" \ - "4: mov %[efault],%[errout]\n" \ - " xorl %%eax,%%eax\n" \ - " xorl %%edx,%%edx\n" \ - " jmp 3b\n" \ - ".previous\n" \ - _ASM_EXTABLE_UA(1b, 4b) \ - _ASM_EXTABLE_UA(2b, 4b) \ + _ASM_EXTABLE_TYPE_REG(1b, 3b, EX_TYPE_EFAULT_REG | \ + EX_FLAG_CLEAR_AX_DX, \ + %[errout]) \ + _ASM_EXTABLE_TYPE_REG(2b, 3b, EX_TYPE_EFAULT_REG | \ + EX_FLAG_CLEAR_AX_DX, \ + %[errout]) \ : [errout] "=r" (retval), \ [output] "=&A"(x) \ : [lowbits] "m" (__m(__ptr)), \ [highbits] "m" __m(((u32 __user *)(__ptr)) + 1), \ - [efault] "i" (-EFAULT), "0" (retval)); \ + "0" (retval)); \ }) #else #define __get_user_asm_u64(x, ptr, retval) \ - __get_user_asm(x, ptr, retval, "q", "=r") + __get_user_asm(x, ptr, retval, "q") #endif #define __get_user_size(x, ptr, size, retval) \ @@ -380,14 +378,14 @@ do { \ __chk_user_ptr(ptr); \ switch (size) { \ case 1: \ - __get_user_asm(x_u8__, ptr, retval, "b", "=q"); \ + __get_user_asm(x_u8__, ptr, retval, "b"); \ (x) = x_u8__; \ break; \ case 2: \ - __get_user_asm(x, ptr, retval, "w", "=r"); \ + __get_user_asm(x, ptr, retval, "w"); \ break; \ case 4: \ - __get_user_asm(x, ptr, retval, "l", "=r"); \ + __get_user_asm(x, ptr, retval, "l"); \ break; \ case 8: \ __get_user_asm_u64(x, ptr, retval); \ @@ -397,22 +395,19 @@ do { \ } \ } while (0) -#define __get_user_asm(x, addr, err, itype, ltype) \ +#define __get_user_asm(x, addr, err, itype) \ asm volatile("\n" \ "1: mov"itype" %[umem],%[output]\n" \ "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %[efault],%[errout]\n" \ - " xorl %k[output],%k[output]\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE_UA(1b, 3b) \ + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG | \ + EX_FLAG_CLEAR_AX, \ + %[errout]) \ : [errout] "=r" (err), \ - [output] ltype(x) \ + [output] "=a" (x) \ : [umem] "m" (__m(addr)), \ - [efault] "i" (-EFAULT), "0" (err)) + "0" (err)) -#endif // CONFIG_CC_ASM_GOTO_OUTPUT +#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT /* FIXME: this hack is definitely wrong -AK */ struct __large_struct { unsigned long buf[100]; }; diff --git a/arch/x86/include/asm/unwind.h b/arch/x86/include/asm/unwind.h index 70fc159ebe..2a1f873441 100644 --- a/arch/x86/include/asm/unwind.h +++ b/arch/x86/include/asm/unwind.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -15,6 +16,9 @@ struct unwind_state { unsigned long stack_mask; struct task_struct *task; int graph_idx; +#ifdef CONFIG_KRETPROBES + struct llist_node *kr_cur; +#endif bool error; #if defined(CONFIG_UNWINDER_ORC) bool signal, full_regs; @@ -99,6 +103,31 @@ void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size) {} #endif +static inline +unsigned long unwind_recover_kretprobe(struct unwind_state *state, + unsigned long addr, unsigned long *addr_p) +{ +#ifdef CONFIG_KRETPROBES + return is_kretprobe_trampoline(addr) ? + kretprobe_find_ret_addr(state->task, addr_p, &state->kr_cur) : + addr; +#else + return addr; +#endif +} + +/* Recover the return address modified by kretprobe and ftrace_graph. */ +static inline +unsigned long unwind_recover_ret_addr(struct unwind_state *state, + unsigned long addr, unsigned long *addr_p) +{ + unsigned long ret; + + ret = ftrace_graph_ret_addr(state->task, &state->graph_idx, + addr, addr_p); + return unwind_recover_kretprobe(state, ret, addr_p); +} + /* * This disables KASAN checking when reading a value from another task's stack, * since the other task could be running on another CPU and could have poisoned diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index 8e574c0afe..8b33674288 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -52,6 +52,11 @@ UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=8 type=UNWIND_HINT_TYPE_FUNC .endm +#else + +#define UNWIND_HINT_FUNC \ + UNWIND_HINT(ORC_REG_SP, 8, UNWIND_HINT_TYPE_FUNC, 0) + #endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_UNWIND_HINTS_H */ diff --git a/arch/x86/include/asm/word-at-a-time.h b/arch/x86/include/asm/word-at-a-time.h index 06006b0351..8338b0432b 100644 --- a/arch/x86/include/asm/word-at-a-time.h +++ b/arch/x86/include/asm/word-at-a-time.h @@ -77,30 +77,58 @@ static inline unsigned long find_zero(unsigned long mask) * and the next page not being mapped, take the exception and * return zeroes in the non-existing part. */ +#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT + static inline unsigned long load_unaligned_zeropad(const void *addr) { - unsigned long ret, dummy; + unsigned long offset, data; + unsigned long ret; + + asm_volatile_goto( + "1: mov %[mem], %[ret]\n" + + _ASM_EXTABLE(1b, %l[do_exception]) + + : [ret] "=r" (ret) + : [mem] "m" (*(unsigned long *)addr) + : : do_exception); + + return ret; + +do_exception: + offset = (unsigned long)addr & (sizeof(long) - 1); + addr = (void *)((unsigned long)addr & ~(sizeof(long) - 1)); + data = *(unsigned long *)addr; + ret = data >> offset * 8; - asm( - "1:\tmov %2,%0\n" - "2:\n" - ".section .fixup,\"ax\"\n" - "3:\t" - "lea %2,%1\n\t" - "and %3,%1\n\t" - "mov (%1),%0\n\t" - "leal %2,%%ecx\n\t" - "andl %4,%%ecx\n\t" - "shll $3,%%ecx\n\t" - "shr %%cl,%0\n\t" - "jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) - :"=&r" (ret),"=&c" (dummy) - :"m" (*(unsigned long *)addr), - "i" (-sizeof(unsigned long)), - "i" (sizeof(unsigned long)-1)); return ret; } +#else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ + +static inline unsigned long load_unaligned_zeropad(const void *addr) +{ + unsigned long offset, data; + unsigned long ret, err = 0; + + asm( "1: mov %[mem], %[ret]\n" + "2:\n" + + _ASM_EXTABLE_FAULT(1b, 2b) + + : [ret] "=&r" (ret), "+a" (err) + : [mem] "m" (*(unsigned long *)addr)); + + if (unlikely(err)) { + offset = (unsigned long)addr & (sizeof(long) - 1); + addr = (void *)((unsigned long)addr & ~(sizeof(long) - 1)); + data = *(unsigned long *)addr; + ret = data >> offset * 8; + } + + return ret; +} + +#endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ + #endif /* _ASM_WORD_AT_A_TIME_H */ diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 5c69f7eb5d..22b7412c08 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -289,12 +289,6 @@ struct x86_platform_ops { struct x86_hyper_runtime hyper; }; -struct pci_dev; - -struct x86_msi_ops { - void (*restore_msi_irqs)(struct pci_dev *dev); -}; - struct x86_apic_ops { unsigned int (*io_apic_read) (unsigned int apic, unsigned int reg); void (*restore)(void); diff --git a/arch/x86/include/asm/xen/cpuid.h b/arch/x86/include/asm/xen/cpuid.h index a9630104f1..78e667a31d 100644 --- a/arch/x86/include/asm/xen/cpuid.h +++ b/arch/x86/include/asm/xen/cpuid.h @@ -100,6 +100,13 @@ /* Memory mapped from other domains has valid IOMMU entries */ #define XEN_HVM_CPUID_IOMMU_MAPPINGS (1u << 2) #define XEN_HVM_CPUID_VCPU_ID_PRESENT (1u << 3) /* vcpu id is present in EBX */ +#define XEN_HVM_CPUID_DOMID_PRESENT (1u << 4) /* domid is present in ECX */ +/* + * Bits 55:49 from the IO-APIC RTE and bits 11:5 from the MSI address can be + * used to store high bits for the Destination ID. This expands the Destination + * ID field from 8 to 15 bits, allowing to target APIC IDs up 32768. + */ +#define XEN_HVM_CPUID_EXT_DEST_ID (1u << 5) /* * Leaf 6 (0x40000x05) diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h index 454b20815f..e5e0fe10c6 100644 --- a/arch/x86/include/asm/xen/hypercall.h +++ b/arch/x86/include/asm/xen/hypercall.h @@ -248,6 +248,7 @@ privcmd_call(unsigned int call, return res; } +#ifdef CONFIG_XEN_PV static inline int HYPERVISOR_set_trap_table(struct trap_info *table) { @@ -280,41 +281,13 @@ HYPERVISOR_callback_op(int cmd, void *arg) return _hypercall2(int, callback_op, cmd, arg); } -static inline int -HYPERVISOR_sched_op(int cmd, void *arg) -{ - return _hypercall2(int, sched_op, cmd, arg); -} - -static inline long -HYPERVISOR_set_timer_op(u64 timeout) -{ - unsigned long timeout_hi = (unsigned long)(timeout>>32); - unsigned long timeout_lo = (unsigned long)timeout; - return _hypercall2(long, set_timer_op, timeout_lo, timeout_hi); -} - -static inline int -HYPERVISOR_mca(struct xen_mc *mc_op) -{ - mc_op->interface_version = XEN_MCA_INTERFACE_VERSION; - return _hypercall1(int, mca, mc_op); -} - -static inline int -HYPERVISOR_platform_op(struct xen_platform_op *op) -{ - op->interface_version = XENPF_INTERFACE_VERSION; - return _hypercall1(int, platform_op, op); -} - -static inline int +static __always_inline int HYPERVISOR_set_debugreg(int reg, unsigned long value) { return _hypercall2(int, set_debugreg, reg, value); } -static inline unsigned long +static __always_inline unsigned long HYPERVISOR_get_debugreg(int reg) { return _hypercall1(unsigned long, get_debugreg, reg); @@ -323,128 +296,21 @@ HYPERVISOR_get_debugreg(int reg) static inline int HYPERVISOR_update_descriptor(u64 ma, u64 desc) { - if (sizeof(u64) == sizeof(long)) - return _hypercall2(int, update_descriptor, ma, desc); - return _hypercall4(int, update_descriptor, ma, ma>>32, desc, desc>>32); -} - -static inline long -HYPERVISOR_memory_op(unsigned int cmd, void *arg) -{ - return _hypercall2(long, memory_op, cmd, arg); -} - -static inline int -HYPERVISOR_multicall(void *call_list, uint32_t nr_calls) -{ - return _hypercall2(int, multicall, call_list, nr_calls); + return _hypercall2(int, update_descriptor, ma, desc); } static inline int HYPERVISOR_update_va_mapping(unsigned long va, pte_t new_val, unsigned long flags) { - if (sizeof(new_val) == sizeof(long)) - return _hypercall3(int, update_va_mapping, va, - new_val.pte, flags); - else - return _hypercall4(int, update_va_mapping, va, - new_val.pte, new_val.pte >> 32, flags); + return _hypercall3(int, update_va_mapping, va, new_val.pte, flags); } -static inline int -HYPERVISOR_event_channel_op(int cmd, void *arg) -{ - return _hypercall2(int, event_channel_op, cmd, arg); -} - -static inline int -HYPERVISOR_xen_version(int cmd, void *arg) -{ - return _hypercall2(int, xen_version, cmd, arg); -} - -static inline int -HYPERVISOR_console_io(int cmd, int count, char *str) -{ - return _hypercall3(int, console_io, cmd, count, str); -} - -static inline int -HYPERVISOR_physdev_op(int cmd, void *arg) -{ - return _hypercall2(int, physdev_op, cmd, arg); -} - -static inline int -HYPERVISOR_grant_table_op(unsigned int cmd, void *uop, unsigned int count) -{ - return _hypercall3(int, grant_table_op, cmd, uop, count); -} - -static inline int -HYPERVISOR_vm_assist(unsigned int cmd, unsigned int type) -{ - return _hypercall2(int, vm_assist, cmd, type); -} - -static inline int -HYPERVISOR_vcpu_op(int cmd, int vcpuid, void *extra_args) -{ - return _hypercall3(int, vcpu_op, cmd, vcpuid, extra_args); -} - -#ifdef CONFIG_X86_64 static inline int HYPERVISOR_set_segment_base(int reg, unsigned long value) { return _hypercall2(int, set_segment_base, reg, value); } -#endif - -static inline int -HYPERVISOR_suspend(unsigned long start_info_mfn) -{ - struct sched_shutdown r = { .reason = SHUTDOWN_suspend }; - - /* - * For a PV guest the tools require that the start_info mfn be - * present in rdx/edx when the hypercall is made. Per the - * hypercall calling convention this is the third hypercall - * argument, which is start_info_mfn here. - */ - return _hypercall3(int, sched_op, SCHEDOP_shutdown, &r, start_info_mfn); -} - -static inline unsigned long __must_check -HYPERVISOR_hvm_op(int op, void *arg) -{ - return _hypercall2(unsigned long, hvm_op, op, arg); -} - -static inline int -HYPERVISOR_tmem_op( - struct tmem_op *op) -{ - return _hypercall1(int, tmem_op, op); -} - -static inline int -HYPERVISOR_xenpmu_op(unsigned int op, void *arg) -{ - return _hypercall2(int, xenpmu_op, op, arg); -} - -static inline int -HYPERVISOR_dm_op( - domid_t dom, unsigned int nr_bufs, struct xen_dm_op_buf *bufs) -{ - int ret; - __xen_stac(); - ret = _hypercall3(int, dm_op, dom, nr_bufs, bufs); - __xen_clac(); - return ret; -} static inline void MULTI_fpu_taskswitch(struct multicall_entry *mcl, int set) @@ -461,16 +327,10 @@ MULTI_update_va_mapping(struct multicall_entry *mcl, unsigned long va, { mcl->op = __HYPERVISOR_update_va_mapping; mcl->args[0] = va; - if (sizeof(new_val) == sizeof(long)) { - mcl->args[1] = new_val.pte; - mcl->args[2] = flags; - } else { - mcl->args[1] = new_val.pte; - mcl->args[2] = new_val.pte >> 32; - mcl->args[3] = flags; - } + mcl->args[1] = new_val.pte; + mcl->args[2] = flags; - trace_xen_mc_entry(mcl, sizeof(new_val) == sizeof(long) ? 3 : 4); + trace_xen_mc_entry(mcl, 3); } static inline void @@ -478,19 +338,10 @@ MULTI_update_descriptor(struct multicall_entry *mcl, u64 maddr, struct desc_struct desc) { mcl->op = __HYPERVISOR_update_descriptor; - if (sizeof(maddr) == sizeof(long)) { - mcl->args[0] = maddr; - mcl->args[1] = *(unsigned long *)&desc; - } else { - u32 *p = (u32 *)&desc; + mcl->args[0] = maddr; + mcl->args[1] = *(unsigned long *)&desc; - mcl->args[0] = maddr; - mcl->args[1] = maddr >> 32; - mcl->args[2] = *p++; - mcl->args[3] = *p; - } - - trace_xen_mc_entry(mcl, sizeof(maddr) == sizeof(long) ? 2 : 4); + trace_xen_mc_entry(mcl, 2); } static inline void @@ -529,5 +380,125 @@ MULTI_stack_switch(struct multicall_entry *mcl, trace_xen_mc_entry(mcl, 2); } +#endif + +static inline int +HYPERVISOR_sched_op(int cmd, void *arg) +{ + return _hypercall2(int, sched_op, cmd, arg); +} + +static inline long +HYPERVISOR_set_timer_op(u64 timeout) +{ + unsigned long timeout_hi = (unsigned long)(timeout>>32); + unsigned long timeout_lo = (unsigned long)timeout; + return _hypercall2(long, set_timer_op, timeout_lo, timeout_hi); +} + +static inline int +HYPERVISOR_mca(struct xen_mc *mc_op) +{ + mc_op->interface_version = XEN_MCA_INTERFACE_VERSION; + return _hypercall1(int, mca, mc_op); +} + +static inline int +HYPERVISOR_platform_op(struct xen_platform_op *op) +{ + op->interface_version = XENPF_INTERFACE_VERSION; + return _hypercall1(int, platform_op, op); +} + +static inline long +HYPERVISOR_memory_op(unsigned int cmd, void *arg) +{ + return _hypercall2(long, memory_op, cmd, arg); +} + +static inline int +HYPERVISOR_multicall(void *call_list, uint32_t nr_calls) +{ + return _hypercall2(int, multicall, call_list, nr_calls); +} + +static inline int +HYPERVISOR_event_channel_op(int cmd, void *arg) +{ + return _hypercall2(int, event_channel_op, cmd, arg); +} + +static __always_inline int +HYPERVISOR_xen_version(int cmd, void *arg) +{ + return _hypercall2(int, xen_version, cmd, arg); +} + +static inline int +HYPERVISOR_console_io(int cmd, int count, char *str) +{ + return _hypercall3(int, console_io, cmd, count, str); +} + +static inline int +HYPERVISOR_physdev_op(int cmd, void *arg) +{ + return _hypercall2(int, physdev_op, cmd, arg); +} + +static inline int +HYPERVISOR_grant_table_op(unsigned int cmd, void *uop, unsigned int count) +{ + return _hypercall3(int, grant_table_op, cmd, uop, count); +} + +static inline int +HYPERVISOR_vm_assist(unsigned int cmd, unsigned int type) +{ + return _hypercall2(int, vm_assist, cmd, type); +} + +static inline int +HYPERVISOR_vcpu_op(int cmd, int vcpuid, void *extra_args) +{ + return _hypercall3(int, vcpu_op, cmd, vcpuid, extra_args); +} + +static inline int +HYPERVISOR_suspend(unsigned long start_info_mfn) +{ + struct sched_shutdown r = { .reason = SHUTDOWN_suspend }; + + /* + * For a PV guest the tools require that the start_info mfn be + * present in rdx/edx when the hypercall is made. Per the + * hypercall calling convention this is the third hypercall + * argument, which is start_info_mfn here. + */ + return _hypercall3(int, sched_op, SCHEDOP_shutdown, &r, start_info_mfn); +} + +static inline unsigned long __must_check +HYPERVISOR_hvm_op(int op, void *arg) +{ + return _hypercall2(unsigned long, hvm_op, op, arg); +} + +static inline int +HYPERVISOR_xenpmu_op(unsigned int op, void *arg) +{ + return _hypercall2(int, xenpmu_op, op, arg); +} + +static inline int +HYPERVISOR_dm_op( + domid_t dom, unsigned int nr_bufs, struct xen_dm_op_buf *bufs) +{ + int ret; + __xen_stac(); + ret = _hypercall3(int, dm_op, dom, nr_bufs, bufs); + __xen_clac(); + return ret; +} #endif /* _ASM_X86_XEN_HYPERCALL_H */ diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h index 5adab89512..16f548a661 100644 --- a/arch/x86/include/asm/xen/hypervisor.h +++ b/arch/x86/include/asm/xen/hypervisor.h @@ -43,18 +43,12 @@ static inline uint32_t xen_cpuid_base(void) return hypervisor_cpuid_base("XenVMMXenVMM", 2); } -#ifdef CONFIG_XEN -extern bool __init xen_hvm_need_lapic(void); +struct pci_dev; -static inline bool __init xen_x2apic_para_available(void) -{ - return xen_hvm_need_lapic(); -} +#ifdef CONFIG_XEN_PV_DOM0 +bool xen_initdom_restore_msi(struct pci_dev *dev); #else -static inline bool __init xen_x2apic_para_available(void) -{ - return (xen_cpuid_base() != 0); -} +static inline bool xen_initdom_restore_msi(struct pci_dev *dev) { return true; } #endif #ifdef CONFIG_HOTPLUG_CPU diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h index 1a162e5597..e989bc2269 100644 --- a/arch/x86/include/asm/xen/page.h +++ b/arch/x86/include/asm/xen/page.h @@ -96,11 +96,7 @@ static inline int xen_safe_write_ulong(unsigned long *addr, unsigned long val) asm volatile("1: mov %[val], %[ptr]\n" "2:\n" - ".section .fixup, \"ax\"\n" - "3: sub $1, %[ret]\n" - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %[ret]) : [ret] "+r" (ret), [ptr] "=m" (*addr) : [val] "r" (val)); @@ -110,16 +106,12 @@ static inline int xen_safe_write_ulong(unsigned long *addr, unsigned long val) static inline int xen_safe_read_ulong(const unsigned long *addr, unsigned long *val) { - int ret = 0; unsigned long rval = ~0ul; + int ret = 0; asm volatile("1: mov %[ptr], %[rval]\n" "2:\n" - ".section .fixup, \"ax\"\n" - "3: sub $1, %[ret]\n" - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %[ret]) : [ret] "+r" (ret), [rval] "+r" (rval) : [ptr] "m" (*addr)); *val = rval; diff --git a/arch/x86/include/asm/xen/pci.h b/arch/x86/include/asm/xen/pci.h index 4557f7cb0f..9015b888ed 100644 --- a/arch/x86/include/asm/xen/pci.h +++ b/arch/x86/include/asm/xen/pci.h @@ -22,25 +22,6 @@ static inline int __init pci_xen_initial_domain(void) return -1; } #endif -#ifdef CONFIG_XEN_DOM0 -int xen_find_device_domain_owner(struct pci_dev *dev); -int xen_register_device_domain_owner(struct pci_dev *dev, uint16_t domain); -int xen_unregister_device_domain_owner(struct pci_dev *dev); -#else -static inline int xen_find_device_domain_owner(struct pci_dev *dev) -{ - return -1; -} -static inline int xen_register_device_domain_owner(struct pci_dev *dev, - uint16_t domain) -{ - return -1; -} -static inline int xen_unregister_device_domain_owner(struct pci_dev *dev) -{ - return -1; -} -#endif #if defined(CONFIG_PCI_MSI) #if defined(CONFIG_PCI_XEN) diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 2ef1f6513c..bf6e96011d 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -373,9 +373,23 @@ struct kvm_debugregs { __u64 reserved[9]; }; -/* for KVM_CAP_XSAVE */ +/* for KVM_CAP_XSAVE and KVM_CAP_XSAVE2 */ struct kvm_xsave { + /* + * KVM_GET_XSAVE2 and KVM_SET_XSAVE write and read as many bytes + * as are returned by KVM_CHECK_EXTENSION(KVM_CAP_XSAVE2) + * respectively, when invoked on the vm file descriptor. + * + * The size value returned by KVM_CHECK_EXTENSION(KVM_CAP_XSAVE2) + * will always be at least 4096. Currently, it is only greater + * than 4096 if a dynamic feature has been enabled with + * ``arch_prctl()``, but this may change in the future. + * + * The offsets of the state save areas in struct kvm_xsave follow + * the contents of CPUID leaf 0xD on the host. + */ __u32 region[1024]; + __u32 extra[0]; }; #define KVM_MAX_XCRS 16 @@ -438,6 +452,9 @@ struct kvm_sync_regs { #define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE 0x00000001 +/* attributes for system fd (group 0) */ +#define KVM_X86_XCOMP_GUEST_SUPP 0 + struct kvm_vmx_nested_state_data { __u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE]; __u8 shadow_vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE]; @@ -504,4 +521,8 @@ struct kvm_pmu_event_filter { #define KVM_PMU_EVENT_ALLOW 0 #define KVM_PMU_EVENT_DENY 1 +/* for KVM_{GET,SET,HAS}_DEVICE_ATTR */ +#define KVM_VCPU_TSC_CTRL 0 /* control group for the timestamp counter (TSC) */ +#define KVM_VCPU_TSC_OFFSET 0 /* attribute for the TSC offset */ + #endif /* _ASM_X86_KVM_H */ diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h index 5146bbab84..6e64b27b2c 100644 --- a/arch/x86/include/uapi/asm/kvm_para.h +++ b/arch/x86/include/uapi/asm/kvm_para.h @@ -8,6 +8,7 @@ * should be used to determine that a VM is running under KVM. */ #define KVM_CPUID_SIGNATURE 0x40000000 +#define KVM_SIGNATURE "KVMKVMKVM\0\0\0" /* This CPUID returns two feature bitmaps in eax, edx. Before enabling * a particular paravirtualization, the appropriate feature bit should diff --git a/arch/x86/include/uapi/asm/prctl.h b/arch/x86/include/uapi/asm/prctl.h index 5a6aac9fa4..500b96e71f 100644 --- a/arch/x86/include/uapi/asm/prctl.h +++ b/arch/x86/include/uapi/asm/prctl.h @@ -2,16 +2,22 @@ #ifndef _ASM_X86_PRCTL_H #define _ASM_X86_PRCTL_H -#define ARCH_SET_GS 0x1001 -#define ARCH_SET_FS 0x1002 -#define ARCH_GET_FS 0x1003 -#define ARCH_GET_GS 0x1004 +#define ARCH_SET_GS 0x1001 +#define ARCH_SET_FS 0x1002 +#define ARCH_GET_FS 0x1003 +#define ARCH_GET_GS 0x1004 -#define ARCH_GET_CPUID 0x1011 -#define ARCH_SET_CPUID 0x1012 +#define ARCH_GET_CPUID 0x1011 +#define ARCH_SET_CPUID 0x1012 -#define ARCH_MAP_VDSO_X32 0x2001 -#define ARCH_MAP_VDSO_32 0x2002 -#define ARCH_MAP_VDSO_64 0x2003 +#define ARCH_GET_XCOMP_SUPP 0x1021 +#define ARCH_GET_XCOMP_PERM 0x1022 +#define ARCH_REQ_XCOMP_PERM 0x1023 +#define ARCH_GET_XCOMP_GUEST_PERM 0x1024 +#define ARCH_REQ_XCOMP_GUEST_PERM 0x1025 + +#define ARCH_MAP_VDSO_X32 0x2001 +#define ARCH_MAP_VDSO_32 0x2002 +#define ARCH_MAP_VDSO_64 0x2003 #endif /* _ASM_X86_PRCTL_H */ diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h index 9690d6899a..f4b81587e9 100644 --- a/arch/x86/include/uapi/asm/sgx.h +++ b/arch/x86/include/uapi/asm/sgx.h @@ -27,6 +27,8 @@ enum sgx_page_flags { _IOW(SGX_MAGIC, 0x02, struct sgx_enclave_init) #define SGX_IOC_ENCLAVE_PROVISION \ _IOW(SGX_MAGIC, 0x03, struct sgx_enclave_provision) +#define SGX_IOC_VEPC_REMOVE_ALL \ + _IO(SGX_MAGIC, 0x04) /** * struct sgx_enclave_create - parameter structure for the diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 2ff3e600f4..6aef9ee28a 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -84,7 +84,7 @@ obj-$(CONFIG_IA32_EMULATION) += tls.o obj-y += step.o obj-$(CONFIG_INTEL_TXT) += tboot.o obj-$(CONFIG_ISA_DMA_API) += i8237.o -obj-$(CONFIG_STACKTRACE) += stacktrace.o +obj-y += stacktrace.o obj-y += cpu/ obj-y += acpi/ obj-y += reboot.o diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 14bcd59bcd..0d01e7f507 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -62,6 +62,7 @@ int acpi_fix_pin2_polarity __initdata; #ifdef CONFIG_X86_LOCAL_APIC static u64 acpi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE; +static bool acpi_support_online_capable; #endif #ifdef CONFIG_X86_IO_APIC @@ -138,6 +139,8 @@ static int __init acpi_parse_madt(struct acpi_table_header *table) pr_debug("Local APIC address 0x%08x\n", madt->address); } + if (madt->header.revision >= 5) + acpi_support_online_capable = true; default_acpi_madt_oem_check(madt->header.oem_id, madt->header.oem_table_id); @@ -239,6 +242,12 @@ acpi_parse_lapic(union acpi_subtable_headers * header, const unsigned long end) if (processor->id == 0xff) return 0; + /* don't register processors that can not be onlined */ + if (acpi_support_online_capable && + !(processor->lapic_flags & ACPI_MADT_ENABLED) && + !(processor->lapic_flags & ACPI_MADT_ONLINE_CAPABLE)) + return 0; + /* * We need to register disabled CPU as well to permit * counting disabled CPUs. This allows us to size @@ -1319,6 +1328,17 @@ static int __init disable_acpi_pci(const struct dmi_system_id *d) return 0; } +static int __init disable_acpi_xsdt(const struct dmi_system_id *d) +{ + if (!acpi_force) { + pr_notice("%s detected: force use of acpi=rsdt\n", d->ident); + acpi_gbl_do_not_use_xsdt = TRUE; + } else { + pr_notice("Warning: DMI blacklist says broken, but acpi XSDT forced\n"); + } + return 0; +} + static int __init dmi_disable_acpi(const struct dmi_system_id *d) { if (!acpi_force) { @@ -1442,6 +1462,19 @@ static const struct dmi_system_id acpi_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "TravelMate 360"), }, }, + /* + * Boxes that need ACPI XSDT use disabled due to corrupted tables + */ + { + .callback = disable_acpi_xsdt, + .ident = "Advantech DAC-BJ01", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "NEC"), + DMI_MATCH(DMI_PRODUCT_NAME, "Bearlake CRB Board"), + DMI_MATCH(DMI_BIOS_VERSION, "V1.12"), + DMI_MATCH(DMI_BIOS_DATE, "02/01/2011"), + }, + }, {} }; diff --git a/arch/x86/kernel/acpi/cstate.c b/arch/x86/kernel/acpi/cstate.c index 7de599eba7..7945eae5b3 100644 --- a/arch/x86/kernel/acpi/cstate.c +++ b/arch/x86/kernel/acpi/cstate.c @@ -79,6 +79,21 @@ void acpi_processor_power_init_bm_check(struct acpi_processor_flags *flags, */ flags->bm_control = 0; } + if (c->x86_vendor == X86_VENDOR_AMD && c->x86 >= 0x17) { + /* + * For all AMD Zen or newer CPUs that support C3, caches + * should not be flushed by software while entering C3 + * type state. Set bm->check to 1 so that kernel doesn't + * need to execute cache flush operation. + */ + flags->bm_check = 1; + /* + * In current AMD C state implementation ARB_DIS is no longer + * used. So set bm_control to zero to indicate ARB_DIS is not + * required while entering C3 type state. + */ + flags->bm_control = 0; + } } EXPORT_SYMBOL(acpi_processor_power_init_bm_check); diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c index 3f85fcae45..1e97f944b4 100644 --- a/arch/x86/kernel/acpi/sleep.c +++ b/arch/x86/kernel/acpi/sleep.c @@ -139,8 +139,10 @@ static int __init acpi_sleep_setup(char *str) if (strncmp(str, "s3_beep", 7) == 0) acpi_realmode_flags |= 4; #ifdef CONFIG_HIBERNATION + if (strncmp(str, "s4_hwsig", 8) == 0) + acpi_check_s4_hw_signature(1); if (strncmp(str, "s4_nohwsig", 10) == 0) - acpi_no_s4_hw_signature(); + acpi_check_s4_hw_signature(0); #endif if (strncmp(str, "nonvs", 5) == 0) acpi_nvs_nosave(); diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index e9da3dc712..b4470eabf1 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -29,6 +29,7 @@ #include #include #include +#include int __read_mostly alternatives_patched; @@ -113,6 +114,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len) } } +extern s32 __retpoline_sites[], __retpoline_sites_end[]; extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; extern s32 __smp_locks[], __smp_locks_end[]; void text_poke_early(void *addr, const void *opcode, size_t len); @@ -221,7 +223,7 @@ static __always_inline int optimize_nops_range(u8 *instr, u8 instrlen, int off) * "noinline" to cause control flow change and thus invalidate I$ and * cause refetch after modification. */ -static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr) +static void __init_or_module noinline optimize_nops(u8 *instr, size_t len) { struct insn insn; int i = 0; @@ -239,11 +241,11 @@ static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *ins * optimized. */ if (insn.length == 1 && insn.opcode.bytes[0] == 0x90) - i += optimize_nops_range(instr, a->instrlen, i); + i += optimize_nops_range(instr, len, i); else i += insn.length; - if (i >= a->instrlen) + if (i >= len) return; } } @@ -331,10 +333,185 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, text_poke_early(instr, insn_buff, insn_buff_sz); next: - optimize_nops(a, instr); + optimize_nops(instr, a->instrlen); } } +#if defined(CONFIG_RETPOLINE) && defined(CONFIG_STACK_VALIDATION) + +/* + * CALL/JMP *%\reg + */ +static int emit_indirect(int op, int reg, u8 *bytes) +{ + int i = 0; + u8 modrm; + + switch (op) { + case CALL_INSN_OPCODE: + modrm = 0x10; /* Reg = 2; CALL r/m */ + break; + + case JMP32_INSN_OPCODE: + modrm = 0x20; /* Reg = 4; JMP r/m */ + break; + + default: + WARN_ON_ONCE(1); + return -1; + } + + if (reg >= 8) { + bytes[i++] = 0x41; /* REX.B prefix */ + reg -= 8; + } + + modrm |= 0xc0; /* Mod = 3 */ + modrm += reg; + + bytes[i++] = 0xff; /* opcode */ + bytes[i++] = modrm; + + return i; +} + +/* + * Rewrite the compiler generated retpoline thunk calls. + * + * For spectre_v2=off (!X86_FEATURE_RETPOLINE), rewrite them into immediate + * indirect instructions, avoiding the extra indirection. + * + * For example, convert: + * + * CALL __x86_indirect_thunk_\reg + * + * into: + * + * CALL *%\reg + * + * It also tries to inline spectre_v2=retpoline,lfence when size permits. + */ +static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) +{ + retpoline_thunk_t *target; + int reg, ret, i = 0; + u8 op, cc; + + target = addr + insn->length + insn->immediate.value; + reg = target - __x86_indirect_thunk_array; + + if (WARN_ON_ONCE(reg & ~0xf)) + return -1; + + /* If anyone ever does: CALL/JMP *%rsp, we're in deep trouble. */ + BUG_ON(reg == 4); + + if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) && + !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) + return -1; + + op = insn->opcode.bytes[0]; + + /* + * Convert: + * + * Jcc.d32 __x86_indirect_thunk_\reg + * + * into: + * + * Jncc.d8 1f + * [ LFENCE ] + * JMP *%\reg + * [ NOP ] + * 1: + */ + /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ + if (op == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80) { + cc = insn->opcode.bytes[1] & 0xf; + cc ^= 1; /* invert condition */ + + bytes[i++] = 0x70 + cc; /* Jcc.d8 */ + bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */ + + /* Continue as if: JMP.d32 __x86_indirect_thunk_\reg */ + op = JMP32_INSN_OPCODE; + } + + /* + * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE. + */ + if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { + bytes[i++] = 0x0f; + bytes[i++] = 0xae; + bytes[i++] = 0xe8; /* LFENCE */ + } + + ret = emit_indirect(op, reg, bytes + i); + if (ret < 0) + return ret; + i += ret; + + for (; i < insn->length;) + bytes[i++] = BYTES_NOP1; + + return i; +} + +/* + * Generated by 'objtool --retpoline'. + */ +void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) +{ + s32 *s; + + for (s = start; s < end; s++) { + void *addr = (void *)s + *s; + struct insn insn; + int len, ret; + u8 bytes[16]; + u8 op1, op2; + + ret = insn_decode_kernel(&insn, addr); + if (WARN_ON_ONCE(ret < 0)) + continue; + + op1 = insn.opcode.bytes[0]; + op2 = insn.opcode.bytes[1]; + + switch (op1) { + case CALL_INSN_OPCODE: + case JMP32_INSN_OPCODE: + break; + + case 0x0f: /* escape */ + if (op2 >= 0x80 && op2 <= 0x8f) + break; + fallthrough; + default: + WARN_ON_ONCE(1); + continue; + } + + DPRINTK("retpoline at: %pS (%px) len: %d to: %pS", + addr, addr, insn.length, + addr + insn.length + insn.immediate.value); + + len = patch_retpoline(addr, &insn, bytes); + if (len == insn.length) { + optimize_nops(bytes, len); + DUMP_BYTES(((u8*)addr), len, "%px: orig: ", addr); + DUMP_BYTES(((u8*)bytes), len, "%px: repl: ", addr); + text_poke_early(addr, bytes, len); + } + } +} + +#else /* !RETPOLINES || !CONFIG_STACK_VALIDATION */ + +void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { } + +#endif /* CONFIG_RETPOLINE && CONFIG_STACK_VALIDATION */ + #ifdef CONFIG_SMP static void alternatives_smp_lock(const s32 *start, const s32 *end, u8 *text, u8 *text_end) @@ -537,7 +714,7 @@ asm ( " .type int3_magic, @function\n" "int3_magic:\n" " movl $1, (%" _ASM_ARG1 ")\n" -" ret\n" + ASM_RET " .size int3_magic, .-int3_magic\n" " .popsection\n" ); @@ -642,6 +819,12 @@ void __init alternative_instructions(void) */ apply_paravirt(__parainstructions, __parainstructions_end); + /* + * Rewrite the retpolines, must be done before alternatives since + * those can rewrite the retpoline thunks. + */ + apply_retpolines(__retpoline_sites, __retpoline_sites_end); + /* * Then patch alternatives, such that those paravirt calls that are in * alternatives can be overwritten by their immediate fragments. @@ -930,10 +1113,13 @@ void text_poke_sync(void) } struct text_poke_loc { - s32 rel_addr; /* addr := _stext + rel_addr */ - s32 rel32; + /* addr := _stext + rel_addr */ + s32 rel_addr; + s32 disp; + u8 len; u8 opcode; const u8 text[POKE_MAX_OPCODE_SIZE]; + /* see text_poke_bp_batch() */ u8 old; }; @@ -948,7 +1134,8 @@ static struct bp_patching_desc *bp_desc; static __always_inline struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp) { - struct bp_patching_desc *desc = __READ_ONCE(*descp); /* rcu_dereference */ + /* rcu_dereference */ + struct bp_patching_desc *desc = __READ_ONCE(*descp); if (!desc || !arch_atomic_inc_not_zero(&desc->refs)) return NULL; @@ -982,7 +1169,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs) { struct bp_patching_desc *desc; struct text_poke_loc *tp; - int len, ret = 0; + int ret = 0; void *ip; if (user_mode(regs)) @@ -1022,8 +1209,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs) goto out_put; } - len = text_opcode_size(tp->opcode); - ip += len; + ip += tp->len; switch (tp->opcode) { case INT3_INSN_OPCODE: @@ -1038,12 +1224,12 @@ noinstr int poke_int3_handler(struct pt_regs *regs) break; case CALL_INSN_OPCODE: - int3_emulate_call(regs, (long)ip + tp->rel32); + int3_emulate_call(regs, (long)ip + tp->disp); break; case JMP32_INSN_OPCODE: case JMP8_INSN_OPCODE: - int3_emulate_jmp(regs, (long)ip + tp->rel32); + int3_emulate_jmp(regs, (long)ip + tp->disp); break; default: @@ -1118,7 +1304,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries */ for (do_sync = 0, i = 0; i < nr_entries; i++) { u8 old[POKE_MAX_OPCODE_SIZE] = { tp[i].old, }; - int len = text_opcode_size(tp[i].opcode); + int len = tp[i].len; if (len - INT3_INSN_SIZE > 0) { memcpy(old + INT3_INSN_SIZE, @@ -1195,20 +1381,36 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, const void *opcode, size_t len, const void *emulate) { struct insn insn; - int ret; + int ret, i; memcpy((void *)tp->text, opcode, len); if (!emulate) emulate = opcode; ret = insn_decode_kernel(&insn, emulate); - BUG_ON(ret < 0); - BUG_ON(len != insn.length); tp->rel_addr = addr - (void *)_stext; + tp->len = len; tp->opcode = insn.opcode.bytes[0]; + switch (tp->opcode) { + case RET_INSN_OPCODE: + case JMP32_INSN_OPCODE: + case JMP8_INSN_OPCODE: + /* + * Control flow instructions without implied execution of the + * next instruction can be padded with INT3. + */ + for (i = insn.length; i < len; i++) + BUG_ON(tp->text[i] != INT3_INSN_OPCODE); + break; + + default: + BUG_ON(len != insn.length); + }; + + switch (tp->opcode) { case INT3_INSN_OPCODE: case RET_INSN_OPCODE: @@ -1217,7 +1419,7 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, case CALL_INSN_OPCODE: case JMP32_INSN_OPCODE: case JMP8_INSN_OPCODE: - tp->rel32 = insn.immediate.value; + tp->disp = insn.immediate.value; break; default: /* assume NOP */ @@ -1225,13 +1427,13 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, case 2: /* NOP2 -- emulate as JMP8+0 */ BUG_ON(memcmp(emulate, x86_nops[len], len)); tp->opcode = JMP8_INSN_OPCODE; - tp->rel32 = 0; + tp->disp = 0; break; case 5: /* NOP5 -- emulate as JMP32+0 */ BUG_ON(memcmp(emulate, x86_nops[len], len)); tp->opcode = JMP32_INSN_OPCODE; - tp->rel32 = 0; + tp->disp = 0; break; default: /* unknown instruction */ diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index c92c9c774c..020c906f79 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -19,17 +19,19 @@ #define PCI_DEVICE_ID_AMD_17H_M10H_ROOT 0x15d0 #define PCI_DEVICE_ID_AMD_17H_M30H_ROOT 0x1480 #define PCI_DEVICE_ID_AMD_17H_M60H_ROOT 0x1630 +#define PCI_DEVICE_ID_AMD_19H_M10H_ROOT 0x14a4 #define PCI_DEVICE_ID_AMD_17H_DF_F4 0x1464 #define PCI_DEVICE_ID_AMD_17H_M10H_DF_F4 0x15ec #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F4 0x1494 #define PCI_DEVICE_ID_AMD_17H_M60H_DF_F4 0x144c #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F4 0x1444 #define PCI_DEVICE_ID_AMD_19H_DF_F4 0x1654 +#define PCI_DEVICE_ID_AMD_19H_M10H_DF_F4 0x14b1 #define PCI_DEVICE_ID_AMD_19H_M40H_ROOT 0x14b5 #define PCI_DEVICE_ID_AMD_19H_M40H_DF_F4 0x167d #define PCI_DEVICE_ID_AMD_19H_M50H_DF_F4 0x166e -/* Protect the PCI config register pairs used for SMN and DF indirect access. */ +/* Protect the PCI config register pairs used for SMN. */ static DEFINE_MUTEX(smn_mutex); static u32 *flush_words; @@ -39,6 +41,7 @@ static const struct pci_device_id amd_root_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_ROOT) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_ROOT) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_ROOT) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M10H_ROOT) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M40H_ROOT) }, {} }; @@ -61,6 +64,7 @@ static const struct pci_device_id amd_nb_misc_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F3) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M10H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M40H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M50H_DF_F3) }, {} @@ -78,6 +82,7 @@ static const struct pci_device_id amd_nb_link_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F4) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M10H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M40H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M50H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F4) }, @@ -182,53 +187,6 @@ int amd_smn_write(u16 node, u32 address, u32 value) } EXPORT_SYMBOL_GPL(amd_smn_write); -/* - * Data Fabric Indirect Access uses FICAA/FICAD. - * - * Fabric Indirect Configuration Access Address (FICAA): Constructed based - * on the device's Instance Id and the PCI function and register offset of - * the desired register. - * - * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO - * and FICAD HI registers but so far we only need the LO register. - */ -int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo) -{ - struct pci_dev *F4; - u32 ficaa; - int err = -ENODEV; - - if (node >= amd_northbridges.num) - goto out; - - F4 = node_to_amd_nb(node)->link; - if (!F4) - goto out; - - ficaa = 1; - ficaa |= reg & 0x3FC; - ficaa |= (func & 0x7) << 11; - ficaa |= instance_id << 16; - - mutex_lock(&smn_mutex); - - err = pci_write_config_dword(F4, 0x5C, ficaa); - if (err) { - pr_warn("Error writing DF Indirect FICAA, FICAA=0x%x\n", ficaa); - goto out_unlock; - } - - err = pci_read_config_dword(F4, 0x98, lo); - if (err) - pr_warn("Error reading DF Indirect FICAD LO, FICAA=0x%x.\n", ficaa); - -out_unlock: - mutex_unlock(&smn_mutex); - -out: - return err; -} -EXPORT_SYMBOL_GPL(amd_df_indirect_read); int amd_cache_northbridges(void) { diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c index 10562885f5..af3ba08b68 100644 --- a/arch/x86/kernel/aperture_64.c +++ b/arch/x86/kernel/aperture_64.c @@ -73,12 +73,23 @@ static int gart_mem_pfn_is_ram(unsigned long pfn) (pfn >= aperture_pfn_start + aperture_page_count)); } +#ifdef CONFIG_PROC_VMCORE +static bool gart_oldmem_pfn_is_ram(struct vmcore_cb *cb, unsigned long pfn) +{ + return !!gart_mem_pfn_is_ram(pfn); +} + +static struct vmcore_cb gart_vmcore_cb = { + .pfn_is_ram = gart_oldmem_pfn_is_ram, +}; +#endif + static void __init exclude_from_core(u64 aper_base, u32 aper_order) { aperture_pfn_start = aper_base >> PAGE_SHIFT; aperture_page_count = (32 * 1024 * 1024) << aper_order >> PAGE_SHIFT; #ifdef CONFIG_PROC_VMCORE - WARN_ON(register_oldmem_pfn_is_ram(&gart_mem_pfn_is_ram)); + register_vmcore_cb(&gart_vmcore_cb); #endif #ifdef CONFIG_PROC_KCORE WARN_ON(register_mem_pfn_is_ram(&gart_mem_pfn_is_ram)); diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c index dbacb9ec88..7517eb05bd 100644 --- a/arch/x86/kernel/apic/msi.c +++ b/arch/x86/kernel/apic/msi.c @@ -19,6 +19,7 @@ #include #include #include +#include struct irq_domain *x86_pci_msi_default_domain __ro_after_init; @@ -159,11 +160,8 @@ static struct irq_chip pci_msi_controller = { int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec, msi_alloc_info_t *arg) { - struct pci_dev *pdev = to_pci_dev(dev); - struct msi_desc *desc = first_pci_msi_entry(pdev); - init_irq_alloc_info(arg, NULL); - if (desc->msi_attrib.is_msix) { + if (to_pci_dev(dev)->msix_enabled) { arg->type = X86_IRQ_ALLOC_TYPE_PCI_MSIX; } else { arg->type = X86_IRQ_ALLOC_TYPE_PCI_MSI; @@ -345,3 +343,8 @@ void dmar_free_hwirq(int irq) irq_domain_free_irqs(irq, 1); } #endif + +bool arch_restore_msi_irqs(struct pci_dev *dev) +{ + return xen_initdom_restore_msi(dev); +} diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index c132daabe6..3e6f6b448f 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -760,9 +760,9 @@ void __init lapic_update_legacy_vectors(void) void __init lapic_assign_system_vectors(void) { - unsigned int i, vector = 0; + unsigned int i, vector; - for_each_set_bit_from(vector, system_vectors, NR_VECTORS) + for_each_set_bit(vector, system_vectors, NR_VECTORS) irq_matrix_assign_system(vector_matrix, vector, false); if (nr_legacy_irqs() > 1) diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index f4da9bb69a..e696e22d05 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -15,9 +15,15 @@ struct cluster_mask { struct cpumask mask; }; -static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid); +/* + * __x2apic_send_IPI_mask() possibly needs to read + * x86_cpu_to_logical_apicid for all online cpus in a sequential way. + * Using per cpu variable would cost one cache line per cpu. + */ +static u32 *x86_cpu_to_logical_apicid __read_mostly; + static DEFINE_PER_CPU(cpumask_var_t, ipi_mask); -static DEFINE_PER_CPU(struct cluster_mask *, cluster_masks); +static DEFINE_PER_CPU_READ_MOSTLY(struct cluster_mask *, cluster_masks); static struct cluster_mask *cluster_hotplug_mask; static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id) @@ -27,7 +33,7 @@ static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id) static void x2apic_send_IPI(int cpu, int vector) { - u32 dest = per_cpu(x86_cpu_to_logical_apicid, cpu); + u32 dest = x86_cpu_to_logical_apicid[cpu]; /* x2apic MSRs are special and need a special fence: */ weak_wrmsr_fence(); @@ -58,7 +64,7 @@ __x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest) dest = 0; for_each_cpu_and(clustercpu, tmpmsk, &cmsk->mask) - dest |= per_cpu(x86_cpu_to_logical_apicid, clustercpu); + dest |= x86_cpu_to_logical_apicid[clustercpu]; if (!dest) continue; @@ -94,7 +100,7 @@ static void x2apic_send_IPI_all(int vector) static u32 x2apic_calc_apicid(unsigned int cpu) { - return per_cpu(x86_cpu_to_logical_apicid, cpu); + return x86_cpu_to_logical_apicid[cpu]; } static void init_x2apic_ldr(void) @@ -103,7 +109,7 @@ static void init_x2apic_ldr(void) u32 cluster, apicid = apic_read(APIC_LDR); unsigned int cpu; - this_cpu_write(x86_cpu_to_logical_apicid, apicid); + x86_cpu_to_logical_apicid[smp_processor_id()] = apicid; if (cmsk) goto update; @@ -166,12 +172,21 @@ static int x2apic_dead_cpu(unsigned int dead_cpu) static int x2apic_cluster_probe(void) { + u32 slots; + if (!x2apic_mode) return 0; + slots = max_t(u32, L1_CACHE_BYTES/sizeof(u32), nr_cpu_ids); + x86_cpu_to_logical_apicid = kcalloc(slots, sizeof(u32), GFP_KERNEL); + if (!x86_cpu_to_logical_apicid) + return 0; + if (cpuhp_setup_state(CPUHP_X2APIC_PREPARE, "x86/x2apic:prepare", x2apic_prepare_cpu, x2apic_dead_cpu) < 0) { pr_err("Failed to register X2APIC_PREPARE\n"); + kfree(x86_cpu_to_logical_apicid); + x86_cpu_to_logical_apicid = NULL; return 0; } init_x2apic_ldr(); diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index ecd3fd6993..9fb0a2f8b6 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -37,9 +37,6 @@ static void __used common(void) OFFSET(TASK_stack_canary, task_struct, stack_canary); #endif - BLANK(); - OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx); - BLANK(); OFFSET(pbe_address, pbe, address); OFFSET(pbe_orig_address, pbe, orig_address); diff --git a/arch/x86/kernel/audit_64.c b/arch/x86/kernel/audit_64.c index 83d9cad4e6..44c3601cfd 100644 --- a/arch/x86/kernel/audit_64.c +++ b/arch/x86/kernel/audit_64.c @@ -47,14 +47,16 @@ int audit_classify_syscall(int abi, unsigned syscall) #endif switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_execve: case __NR_execveat: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/x86/kernel/cc_platform.c b/arch/x86/kernel/cc_platform.c index 03bb2f343d..6a6ffcd978 100644 --- a/arch/x86/kernel/cc_platform.c +++ b/arch/x86/kernel/cc_platform.c @@ -11,6 +11,7 @@ #include #include +#include #include static bool __maybe_unused intel_cc_platform_has(enum cc_attr attr) @@ -50,6 +51,14 @@ static bool amd_cc_platform_has(enum cc_attr attr) case CC_ATTR_GUEST_STATE_ENCRYPT: return sev_status & MSR_AMD64_SEV_ES_ENABLED; + /* + * With SEV, the rep string I/O instructions need to be unrolled + * but SEV-ES supports them through the #VC handler. + */ + case CC_ATTR_GUEST_UNROLL_STRING_IO: + return (sev_status & MSR_AMD64_SEV_ENABLED) && + !(sev_status & MSR_AMD64_SEV_ES_ENABLED); + default: return false; } @@ -58,12 +67,19 @@ static bool amd_cc_platform_has(enum cc_attr attr) #endif } +static bool hyperv_cc_platform_has(enum cc_attr attr) +{ + return attr == CC_ATTR_GUEST_MEM_ENCRYPT; +} bool cc_platform_has(enum cc_attr attr) { if (sme_me_mask) return amd_cc_platform_has(attr); + if (hv_is_isolation_supported()) + return hyperv_cc_platform_has(attr); + return false; } EXPORT_SYMBOL_GPL(cc_platform_has); diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 637b499450..9661e3e802 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -43,6 +43,7 @@ obj-$(CONFIG_CPU_SUP_CENTAUR) += centaur.o obj-$(CONFIG_CPU_SUP_TRANSMETA_32) += transmeta.o obj-$(CONFIG_CPU_SUP_UMC_32) += umc.o obj-$(CONFIG_CPU_SUP_ZHAOXIN) += zhaoxin.o +obj-$(CONFIG_CPU_SUP_VORTEX_32) += vortex.o obj-$(CONFIG_X86_MCE) += mce/ obj-$(CONFIG_MTRR) += mtrr/ diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 13320a06e1..6296e1ebed 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include @@ -795,11 +795,11 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) case SPECTRE_V2_USER_CMD_FORCE: mode = SPECTRE_V2_USER_STRICT; break; + case SPECTRE_V2_USER_CMD_AUTO: case SPECTRE_V2_USER_CMD_PRCTL: case SPECTRE_V2_USER_CMD_PRCTL_IBPB: mode = SPECTRE_V2_USER_PRCTL; break; - case SPECTRE_V2_USER_CMD_AUTO: case SPECTRE_V2_USER_CMD_SECCOMP: case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: if (IS_ENABLED(CONFIG_SECCOMP)) @@ -1254,7 +1254,6 @@ static enum ssb_mitigation __init __ssb_select_mitigation(void) return mode; switch (cmd) { - case SPEC_STORE_BYPASS_CMD_AUTO: case SPEC_STORE_BYPASS_CMD_SECCOMP: /* * Choose prctl+seccomp as the default mode if seccomp is @@ -1268,6 +1267,7 @@ static enum ssb_mitigation __init __ssb_select_mitigation(void) case SPEC_STORE_BYPASS_CMD_ON: mode = SPEC_STORE_BYPASS_DISABLE; break; + case SPEC_STORE_BYPASS_CMD_AUTO: case SPEC_STORE_BYPASS_CMD_PRCTL: mode = SPEC_STORE_BYPASS_PRCTL; break; diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index b5e36bd042..fe98a1465b 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -846,6 +846,7 @@ void init_intel_cacheinfo(struct cpuinfo_x86 *c) l2 = new_l2; #ifdef CONFIG_SMP per_cpu(cpu_llc_id, cpu) = l2_id; + per_cpu(cpu_l2c_id, cpu) = l2_id; #endif } diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 58b1416c05..7b8382c117 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -42,7 +42,7 @@ #include #include #include -#include +#include #include #include #include @@ -85,6 +85,9 @@ u16 get_llc_id(unsigned int cpu) } EXPORT_SYMBOL_GPL(get_llc_id); +/* L2 cache ID of each logical CPU */ +DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id) = BAD_APICID; + /* correctly size the local cpu masks */ void __init setup_cpu_local_masks(void) { @@ -381,7 +384,7 @@ void native_write_cr0(unsigned long val) } EXPORT_SYMBOL(native_write_cr0); -void native_write_cr4(unsigned long val) +void __no_profile native_write_cr4(unsigned long val) { unsigned long bits_changed = 0; @@ -1045,6 +1048,8 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { VULNWL(CENTAUR, 5, X86_MODEL_ANY, NO_SPECULATION), VULNWL(INTEL, 5, X86_MODEL_ANY, NO_SPECULATION), VULNWL(NSC, 5, X86_MODEL_ANY, NO_SPECULATION), + VULNWL(VORTEX, 5, X86_MODEL_ANY, NO_SPECULATION), + VULNWL(VORTEX, 6, X86_MODEL_ANY, NO_SPECULATION), /* Intel Family 6 */ VULNWL_INTEL(ATOM_SALTWELL, NO_SPECULATION | NO_ITLB_MULTIHIT), @@ -1782,6 +1787,17 @@ EXPORT_PER_CPU_SYMBOL(__preempt_count); DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK; +static void wrmsrl_cstar(unsigned long val) +{ + /* + * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR + * is so far ignored by the CPU, but raises a #VE trap in a TDX + * guest. Avoid the pointless write on all Intel CPUs. + */ + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + wrmsrl(MSR_CSTAR, val); +} + /* May not be marked __init: used by software suspend */ void syscall_init(void) { @@ -1789,7 +1805,7 @@ void syscall_init(void) wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64); #ifdef CONFIG_IA32_EMULATION - wrmsrl(MSR_CSTAR, (unsigned long)entry_SYSCALL_compat); + wrmsrl_cstar((unsigned long)entry_SYSCALL_compat); /* * This only works on Intel CPUs. * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP. @@ -1801,7 +1817,7 @@ void syscall_init(void) (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1)); wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat); #else - wrmsrl(MSR_CSTAR, (unsigned long)ignore_sysret); + wrmsrl_cstar((unsigned long)ignore_sysret); wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG); wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL); wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL); diff --git a/arch/x86/kernel/cpu/cpuid-deps.c b/arch/x86/kernel/cpu/cpuid-deps.c index defda61f37..c881bcafba 100644 --- a/arch/x86/kernel/cpu/cpuid-deps.c +++ b/arch/x86/kernel/cpu/cpuid-deps.c @@ -75,6 +75,9 @@ static const struct cpuid_dep cpuid_deps[] = { { X86_FEATURE_SGX_LC, X86_FEATURE_SGX }, { X86_FEATURE_SGX1, X86_FEATURE_SGX }, { X86_FEATURE_SGX2, X86_FEATURE_SGX1 }, + { X86_FEATURE_XFD, X86_FEATURE_XSAVES }, + { X86_FEATURE_XFD, X86_FEATURE_XGETBV1 }, + { X86_FEATURE_AMX_TILE, X86_FEATURE_XFD }, {} }; diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c index f4dd73396f..fbaf12e43f 100644 --- a/arch/x86/kernel/cpu/intel_epb.c +++ b/arch/x86/kernel/cpu/intel_epb.c @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -58,6 +59,22 @@ static DEFINE_PER_CPU(u8, saved_epb); #define EPB_SAVED 0x10ULL #define MAX_EPB EPB_MASK +enum energy_perf_value_index { + EPB_INDEX_PERFORMANCE, + EPB_INDEX_BALANCE_PERFORMANCE, + EPB_INDEX_NORMAL, + EPB_INDEX_BALANCE_POWERSAVE, + EPB_INDEX_POWERSAVE, +}; + +static u8 energ_perf_values[] = { + [EPB_INDEX_PERFORMANCE] = ENERGY_PERF_BIAS_PERFORMANCE, + [EPB_INDEX_BALANCE_PERFORMANCE] = ENERGY_PERF_BIAS_BALANCE_PERFORMANCE, + [EPB_INDEX_NORMAL] = ENERGY_PERF_BIAS_NORMAL, + [EPB_INDEX_BALANCE_POWERSAVE] = ENERGY_PERF_BIAS_BALANCE_POWERSAVE, + [EPB_INDEX_POWERSAVE] = ENERGY_PERF_BIAS_POWERSAVE, +}; + static int intel_epb_save(void) { u64 epb; @@ -90,7 +107,7 @@ static void intel_epb_restore(void) */ val = epb & EPB_MASK; if (val == ENERGY_PERF_BIAS_PERFORMANCE) { - val = ENERGY_PERF_BIAS_NORMAL; + val = energ_perf_values[EPB_INDEX_NORMAL]; pr_warn_once("ENERGY_PERF_BIAS: Set to 'normal', was 'performance'\n"); } } @@ -103,18 +120,11 @@ static struct syscore_ops intel_epb_syscore_ops = { }; static const char * const energy_perf_strings[] = { - "performance", - "balance-performance", - "normal", - "balance-power", - "power" -}; -static const u8 energ_perf_values[] = { - ENERGY_PERF_BIAS_PERFORMANCE, - ENERGY_PERF_BIAS_BALANCE_PERFORMANCE, - ENERGY_PERF_BIAS_NORMAL, - ENERGY_PERF_BIAS_BALANCE_POWERSAVE, - ENERGY_PERF_BIAS_POWERSAVE + [EPB_INDEX_PERFORMANCE] = "performance", + [EPB_INDEX_BALANCE_PERFORMANCE] = "balance-performance", + [EPB_INDEX_NORMAL] = "normal", + [EPB_INDEX_BALANCE_POWERSAVE] = "balance-power", + [EPB_INDEX_POWERSAVE] = "power", }; static ssize_t energy_perf_bias_show(struct device *dev, @@ -193,13 +203,22 @@ static int intel_epb_offline(unsigned int cpu) return 0; } +static const struct x86_cpu_id intel_epb_normal[] = { + X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, 7), + {} +}; + static __init int intel_epb_init(void) { + const struct x86_cpu_id *id = x86_match_cpu(intel_epb_normal); int ret; if (!boot_cpu_has(X86_FEATURE_EPB)) return -ENODEV; + if (id) + energ_perf_values[EPB_INDEX_NORMAL] = id->driver_data; + ret = cpuhp_setup_state(CPUHP_AP_X86_INTEL_EPB_ONLINE, "x86/intel/epb:online", intel_epb_online, intel_epb_offline); diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index c0c57bd05f..9f4b508886 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -71,6 +71,22 @@ static const char * const smca_umc_block_names[] = { "misc_umc" }; +#define HWID_MCATYPE(hwid, mcatype) (((hwid) << 16) | (mcatype)) + +struct smca_hwid { + unsigned int bank_type; /* Use with smca_bank_types for easy indexing. */ + u32 hwid_mcatype; /* (hwid,mcatype) tuple */ +}; + +struct smca_bank { + const struct smca_hwid *hwid; + u32 id; /* Value of MCA_IPID[InstanceId]. */ + u8 sysfs_id; /* Value used for sysfs name. */ +}; + +static DEFINE_PER_CPU_READ_MOSTLY(struct smca_bank[MAX_NR_BANKS], smca_banks); +static DEFINE_PER_CPU_READ_MOSTLY(u8[N_SMCA_BANK_TYPES], smca_bank_counts); + struct smca_bank_name { const char *name; /* Short name for sysfs */ const char *long_name; /* Long name for pretty-printing */ @@ -95,11 +111,18 @@ static struct smca_bank_name smca_names[] = { [SMCA_PSP ... SMCA_PSP_V2] = { "psp", "Platform Security Processor" }, [SMCA_SMU ... SMCA_SMU_V2] = { "smu", "System Management Unit" }, [SMCA_MP5] = { "mp5", "Microprocessor 5 Unit" }, + [SMCA_MPDMA] = { "mpdma", "MPDMA Unit" }, [SMCA_NBIO] = { "nbio", "Northbridge IO Unit" }, [SMCA_PCIE ... SMCA_PCIE_V2] = { "pcie", "PCI Express Unit" }, [SMCA_XGMI_PCS] = { "xgmi_pcs", "Ext Global Memory Interconnect PCS Unit" }, + [SMCA_NBIF] = { "nbif", "NBIF Unit" }, + [SMCA_SHUB] = { "shub", "System Hub Unit" }, + [SMCA_SATA] = { "sata", "SATA Unit" }, + [SMCA_USB] = { "usb", "USB Unit" }, + [SMCA_GMI_PCS] = { "gmi_pcs", "Global Memory Interconnect PCS Unit" }, [SMCA_XGMI_PHY] = { "xgmi_phy", "Ext Global Memory Interconnect PHY Unit" }, [SMCA_WAFL_PHY] = { "wafl_phy", "WAFL PHY Unit" }, + [SMCA_GMI_PHY] = { "gmi_phy", "Global Memory Interconnect PHY Unit" }, }; static const char *smca_get_name(enum smca_bank_types t) @@ -119,21 +142,22 @@ const char *smca_get_long_name(enum smca_bank_types t) } EXPORT_SYMBOL_GPL(smca_get_long_name); -static enum smca_bank_types smca_get_bank_type(unsigned int bank) +enum smca_bank_types smca_get_bank_type(unsigned int cpu, unsigned int bank) { struct smca_bank *b; if (bank >= MAX_NR_BANKS) return N_SMCA_BANK_TYPES; - b = &smca_banks[bank]; + b = &per_cpu(smca_banks, cpu)[bank]; if (!b->hwid) return N_SMCA_BANK_TYPES; return b->hwid->bank_type; } +EXPORT_SYMBOL_GPL(smca_get_bank_type); -static struct smca_hwid smca_hwid_mcatypes[] = { +static const struct smca_hwid smca_hwid_mcatypes[] = { /* { bank_type, hwid_mcatype } */ /* Reserved type */ @@ -173,6 +197,9 @@ static struct smca_hwid smca_hwid_mcatypes[] = { /* Microprocessor 5 Unit MCA type */ { SMCA_MP5, HWID_MCATYPE(0x01, 0x2) }, + /* MPDMA MCA type */ + { SMCA_MPDMA, HWID_MCATYPE(0x01, 0x3) }, + /* Northbridge IO Unit MCA type */ { SMCA_NBIO, HWID_MCATYPE(0x18, 0x0) }, @@ -180,19 +207,17 @@ static struct smca_hwid smca_hwid_mcatypes[] = { { SMCA_PCIE, HWID_MCATYPE(0x46, 0x0) }, { SMCA_PCIE_V2, HWID_MCATYPE(0x46, 0x1) }, - /* xGMI PCS MCA type */ { SMCA_XGMI_PCS, HWID_MCATYPE(0x50, 0x0) }, - - /* xGMI PHY MCA type */ + { SMCA_NBIF, HWID_MCATYPE(0x6C, 0x0) }, + { SMCA_SHUB, HWID_MCATYPE(0x80, 0x0) }, + { SMCA_SATA, HWID_MCATYPE(0xA8, 0x0) }, + { SMCA_USB, HWID_MCATYPE(0xAA, 0x0) }, + { SMCA_GMI_PCS, HWID_MCATYPE(0x241, 0x0) }, { SMCA_XGMI_PHY, HWID_MCATYPE(0x259, 0x0) }, - - /* WAFL PHY MCA type */ { SMCA_WAFL_PHY, HWID_MCATYPE(0x267, 0x0) }, + { SMCA_GMI_PHY, HWID_MCATYPE(0x269, 0x0) }, }; -struct smca_bank smca_banks[MAX_NR_BANKS]; -EXPORT_SYMBOL_GPL(smca_banks); - /* * In SMCA enabled processors, we can have multiple banks for a given IP type. * So to define a unique name for each bank, we use a temp c-string to append @@ -248,8 +273,9 @@ static void smca_set_misc_banks_map(unsigned int bank, unsigned int cpu) static void smca_configure(unsigned int bank, unsigned int cpu) { + u8 *bank_counts = this_cpu_ptr(smca_bank_counts); + const struct smca_hwid *s_hwid; unsigned int i, hwid_mcatype; - struct smca_hwid *s_hwid; u32 high, low; u32 smca_config = MSR_AMD64_SMCA_MCx_CONFIG(bank); @@ -285,10 +311,6 @@ static void smca_configure(unsigned int bank, unsigned int cpu) smca_set_misc_banks_map(bank, cpu); - /* Return early if this bank was already initialized. */ - if (smca_banks[bank].hwid && smca_banks[bank].hwid->hwid_mcatype != 0) - return; - if (rdmsr_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &low, &high)) { pr_warn("Failed to read MCA_IPID for bank %d\n", bank); return; @@ -299,10 +321,11 @@ static void smca_configure(unsigned int bank, unsigned int cpu) for (i = 0; i < ARRAY_SIZE(smca_hwid_mcatypes); i++) { s_hwid = &smca_hwid_mcatypes[i]; + if (hwid_mcatype == s_hwid->hwid_mcatype) { - smca_banks[bank].hwid = s_hwid; - smca_banks[bank].id = low; - smca_banks[bank].sysfs_id = s_hwid->count++; + this_cpu_ptr(smca_banks)[bank].hwid = s_hwid; + this_cpu_ptr(smca_banks)[bank].id = low; + this_cpu_ptr(smca_banks)[bank].sysfs_id = bank_counts[s_hwid->bank_type]++; break; } } @@ -526,7 +549,7 @@ static u32 get_block_address(u32 current_addr, u32 low, u32 high, /* Fall back to method we used for older processors: */ switch (block) { case 0: - addr = msr_ops.misc(bank); + addr = mca_msr_reg(bank, MCA_MISC); break; case 1: offset = ((low & MASK_BLKPTR_LO) >> 21); @@ -588,7 +611,7 @@ prepare_threshold_block(unsigned int bank, unsigned int block, u32 addr, bool amd_filter_mce(struct mce *m) { - enum smca_bank_types bank_type = smca_get_bank_type(m->bank); + enum smca_bank_types bank_type = smca_get_bank_type(m->extcpu, m->bank); struct cpuinfo_x86 *c = &boot_cpu_data; /* See Family 17h Models 10h-2Fh Erratum #1114. */ @@ -626,7 +649,7 @@ static void disable_err_thresholding(struct cpuinfo_x86 *c, unsigned int bank) } else if (c->x86 == 0x17 && (c->x86_model >= 0x10 && c->x86_model <= 0x2F)) { - if (smca_get_bank_type(bank) != SMCA_IF) + if (smca_get_bank_type(smp_processor_id(), bank) != SMCA_IF) return; msrs[0] = MSR_AMD64_SMCA_MCx_MISC(bank); @@ -688,213 +711,13 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c) deferred_error_interrupt_enable(c); } -int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) -{ - u64 dram_base_addr, dram_limit_addr, dram_hole_base; - /* We start from the normalized address */ - u64 ret_addr = norm_addr; - - u32 tmp; - - u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask; - u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets; - u8 intlv_addr_sel, intlv_addr_bit; - u8 num_intlv_bits, hashed_bit; - u8 lgcy_mmio_hole_en, base = 0; - u8 cs_mask, cs_id = 0; - bool hash_enabled = false; - - /* Read D18F0x1B4 (DramOffset), check if base 1 is used. */ - if (amd_df_indirect_read(nid, 0, 0x1B4, umc, &tmp)) - goto out_err; - - /* Remove HiAddrOffset from normalized address, if enabled: */ - if (tmp & BIT(0)) { - u64 hi_addr_offset = (tmp & GENMASK_ULL(31, 20)) << 8; - - if (norm_addr >= hi_addr_offset) { - ret_addr -= hi_addr_offset; - base = 1; - } - } - - /* Read D18F0x110 (DramBaseAddress). */ - if (amd_df_indirect_read(nid, 0, 0x110 + (8 * base), umc, &tmp)) - goto out_err; - - /* Check if address range is valid. */ - if (!(tmp & BIT(0))) { - pr_err("%s: Invalid DramBaseAddress range: 0x%x.\n", - __func__, tmp); - goto out_err; - } - - lgcy_mmio_hole_en = tmp & BIT(1); - intlv_num_chan = (tmp >> 4) & 0xF; - intlv_addr_sel = (tmp >> 8) & 0x7; - dram_base_addr = (tmp & GENMASK_ULL(31, 12)) << 16; - - /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */ - if (intlv_addr_sel > 3) { - pr_err("%s: Invalid interleave address select %d.\n", - __func__, intlv_addr_sel); - goto out_err; - } - - /* Read D18F0x114 (DramLimitAddress). */ - if (amd_df_indirect_read(nid, 0, 0x114 + (8 * base), umc, &tmp)) - goto out_err; - - intlv_num_sockets = (tmp >> 8) & 0x1; - intlv_num_dies = (tmp >> 10) & 0x3; - dram_limit_addr = ((tmp & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0); - - intlv_addr_bit = intlv_addr_sel + 8; - - /* Re-use intlv_num_chan by setting it equal to log2(#channels) */ - switch (intlv_num_chan) { - case 0: intlv_num_chan = 0; break; - case 1: intlv_num_chan = 1; break; - case 3: intlv_num_chan = 2; break; - case 5: intlv_num_chan = 3; break; - case 7: intlv_num_chan = 4; break; - - case 8: intlv_num_chan = 1; - hash_enabled = true; - break; - default: - pr_err("%s: Invalid number of interleaved channels %d.\n", - __func__, intlv_num_chan); - goto out_err; - } - - num_intlv_bits = intlv_num_chan; - - if (intlv_num_dies > 2) { - pr_err("%s: Invalid number of interleaved nodes/dies %d.\n", - __func__, intlv_num_dies); - goto out_err; - } - - num_intlv_bits += intlv_num_dies; - - /* Add a bit if sockets are interleaved. */ - num_intlv_bits += intlv_num_sockets; - - /* Assert num_intlv_bits <= 4 */ - if (num_intlv_bits > 4) { - pr_err("%s: Invalid interleave bits %d.\n", - __func__, num_intlv_bits); - goto out_err; - } - - if (num_intlv_bits > 0) { - u64 temp_addr_x, temp_addr_i, temp_addr_y; - u8 die_id_bit, sock_id_bit, cs_fabric_id; - - /* - * Read FabricBlockInstanceInformation3_CS[BlockFabricID]. - * This is the fabric id for this coherent slave. Use - * umc/channel# as instance id of the coherent slave - * for FICAA. - */ - if (amd_df_indirect_read(nid, 0, 0x50, umc, &tmp)) - goto out_err; - - cs_fabric_id = (tmp >> 8) & 0xFF; - die_id_bit = 0; - - /* If interleaved over more than 1 channel: */ - if (intlv_num_chan) { - die_id_bit = intlv_num_chan; - cs_mask = (1 << die_id_bit) - 1; - cs_id = cs_fabric_id & cs_mask; - } - - sock_id_bit = die_id_bit; - - /* Read D18F1x208 (SystemFabricIdMask). */ - if (intlv_num_dies || intlv_num_sockets) - if (amd_df_indirect_read(nid, 1, 0x208, umc, &tmp)) - goto out_err; - - /* If interleaved over more than 1 die. */ - if (intlv_num_dies) { - sock_id_bit = die_id_bit + intlv_num_dies; - die_id_shift = (tmp >> 24) & 0xF; - die_id_mask = (tmp >> 8) & 0xFF; - - cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit; - } - - /* If interleaved over more than 1 socket. */ - if (intlv_num_sockets) { - socket_id_shift = (tmp >> 28) & 0xF; - socket_id_mask = (tmp >> 16) & 0xFF; - - cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit; - } - - /* - * The pre-interleaved address consists of XXXXXXIIIYYYYY - * where III is the ID for this CS, and XXXXXXYYYYY are the - * address bits from the post-interleaved address. - * "num_intlv_bits" has been calculated to tell us how many "I" - * bits there are. "intlv_addr_bit" tells us how many "Y" bits - * there are (where "I" starts). - */ - temp_addr_y = ret_addr & GENMASK_ULL(intlv_addr_bit-1, 0); - temp_addr_i = (cs_id << intlv_addr_bit); - temp_addr_x = (ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits; - ret_addr = temp_addr_x | temp_addr_i | temp_addr_y; - } - - /* Add dram base address */ - ret_addr += dram_base_addr; - - /* If legacy MMIO hole enabled */ - if (lgcy_mmio_hole_en) { - if (amd_df_indirect_read(nid, 0, 0x104, umc, &tmp)) - goto out_err; - - dram_hole_base = tmp & GENMASK(31, 24); - if (ret_addr >= dram_hole_base) - ret_addr += (BIT_ULL(32) - dram_hole_base); - } - - if (hash_enabled) { - /* Save some parentheses and grab ls-bit at the end. */ - hashed_bit = (ret_addr >> 12) ^ - (ret_addr >> 18) ^ - (ret_addr >> 21) ^ - (ret_addr >> 30) ^ - cs_id; - - hashed_bit &= BIT(0); - - if (hashed_bit != ((ret_addr >> intlv_addr_bit) & BIT(0))) - ret_addr ^= BIT(intlv_addr_bit); - } - - /* Is calculated system address is above DRAM limit address? */ - if (ret_addr > dram_limit_addr) - goto out_err; - - *sys_addr = ret_addr; - return 0; - -out_err: - return -EINVAL; -} -EXPORT_SYMBOL_GPL(umc_normaddr_to_sysaddr); - bool amd_mce_is_memory_error(struct mce *m) { /* ErrCodeExt[20:16] */ u8 xec = (m->status >> 16) & 0x1f; if (mce_flags.smca) - return smca_get_bank_type(m->bank) == SMCA_UMC && xec == 0x0; + return smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC && xec == 0x0; return m->bank == 4 && xec == 0x8; } @@ -978,8 +801,8 @@ static void log_error_deferred(unsigned int bank) { bool defrd; - defrd = _log_error_bank(bank, msr_ops.status(bank), - msr_ops.addr(bank), 0); + defrd = _log_error_bank(bank, mca_msr_reg(bank, MCA_STATUS), + mca_msr_reg(bank, MCA_ADDR), 0); if (!mce_flags.smca) return; @@ -1009,7 +832,7 @@ static void amd_deferred_error_interrupt(void) static void log_error_thresholding(unsigned int bank, u64 misc) { - _log_error_bank(bank, msr_ops.status(bank), msr_ops.addr(bank), misc); + _log_error_bank(bank, mca_msr_reg(bank, MCA_STATUS), mca_msr_reg(bank, MCA_ADDR), misc); } static void log_and_reset_block(struct threshold_block *block) @@ -1210,7 +1033,7 @@ static struct kobj_type threshold_ktype = { .release = threshold_block_release, }; -static const char *get_name(unsigned int bank, struct threshold_block *b) +static const char *get_name(unsigned int cpu, unsigned int bank, struct threshold_block *b) { enum smca_bank_types bank_type; @@ -1221,7 +1044,7 @@ static const char *get_name(unsigned int bank, struct threshold_block *b) return th_names[bank]; } - bank_type = smca_get_bank_type(bank); + bank_type = smca_get_bank_type(cpu, bank); if (bank_type >= N_SMCA_BANK_TYPES) return NULL; @@ -1231,12 +1054,12 @@ static const char *get_name(unsigned int bank, struct threshold_block *b) return NULL; } - if (smca_banks[bank].hwid->count == 1) + if (per_cpu(smca_bank_counts, cpu)[bank_type] == 1) return smca_get_name(bank_type); snprintf(buf_mcatype, MAX_MCATYPE_NAME_LEN, - "%s_%x", smca_get_name(bank_type), - smca_banks[bank].sysfs_id); + "%s_%u", smca_get_name(bank_type), + per_cpu(smca_banks, cpu)[bank].sysfs_id); return buf_mcatype; } @@ -1292,7 +1115,7 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb else tb->blocks = b; - err = kobject_init_and_add(&b->kobj, &threshold_ktype, tb->kobj, get_name(bank, b)); + err = kobject_init_and_add(&b->kobj, &threshold_ktype, tb->kobj, get_name(cpu, bank, b)); if (err) goto out_free; recurse: @@ -1347,7 +1170,7 @@ static int threshold_create_bank(struct threshold_bank **bp, unsigned int cpu, struct device *dev = this_cpu_read(mce_device); struct amd_northbridge *nb = NULL; struct threshold_bank *b = NULL; - const char *name = get_name(bank, NULL); + const char *name = get_name(cpu, bank, NULL); int err = 0; if (!dev) @@ -1397,7 +1220,7 @@ static int threshold_create_bank(struct threshold_bank **bp, unsigned int cpu, } } - err = allocate_threshold_blocks(cpu, b, bank, 0, msr_ops.misc(bank)); + err = allocate_threshold_blocks(cpu, b, bank, 0, mca_msr_reg(bank, MCA_MISC)); if (err) goto out_kobj; diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index e23e74e2f9..5818b837fd 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -99,7 +99,6 @@ struct mca_config mca_cfg __read_mostly = { static DEFINE_PER_CPU(struct mce, mces_seen); static unsigned long mce_need_notify; -static int cpu_missing; /* * MCA banks polled by the period polling timer for corrected events. @@ -121,8 +120,6 @@ mce_banks_t mce_banks_ce_disabled; static struct work_struct mce_work; static struct irq_work mce_irq_work; -static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs); - /* * CPU/chipset specific EDAC code can register a notifier call here to print * MCE errors in a human-readable form. @@ -130,7 +127,7 @@ static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs); BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain); /* Do initial initialization of a struct mce */ -noinstr void mce_setup(struct mce *m) +void mce_setup(struct mce *m) { memset(m, 0, sizeof(struct mce)); m->cpu = m->extcpu = smp_processor_id(); @@ -176,53 +173,27 @@ void mce_unregister_decode_chain(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(mce_unregister_decode_chain); -static inline u32 ctl_reg(int bank) +u32 mca_msr_reg(int bank, enum mca_msr reg) { - return MSR_IA32_MCx_CTL(bank); -} + if (mce_flags.smca) { + switch (reg) { + case MCA_CTL: return MSR_AMD64_SMCA_MCx_CTL(bank); + case MCA_ADDR: return MSR_AMD64_SMCA_MCx_ADDR(bank); + case MCA_MISC: return MSR_AMD64_SMCA_MCx_MISC(bank); + case MCA_STATUS: return MSR_AMD64_SMCA_MCx_STATUS(bank); + } + } -static inline u32 status_reg(int bank) -{ - return MSR_IA32_MCx_STATUS(bank); -} + switch (reg) { + case MCA_CTL: return MSR_IA32_MCx_CTL(bank); + case MCA_ADDR: return MSR_IA32_MCx_ADDR(bank); + case MCA_MISC: return MSR_IA32_MCx_MISC(bank); + case MCA_STATUS: return MSR_IA32_MCx_STATUS(bank); + } -static inline u32 addr_reg(int bank) -{ - return MSR_IA32_MCx_ADDR(bank); + return 0; } -static inline u32 misc_reg(int bank) -{ - return MSR_IA32_MCx_MISC(bank); -} - -static inline u32 smca_ctl_reg(int bank) -{ - return MSR_AMD64_SMCA_MCx_CTL(bank); -} - -static inline u32 smca_status_reg(int bank) -{ - return MSR_AMD64_SMCA_MCx_STATUS(bank); -} - -static inline u32 smca_addr_reg(int bank) -{ - return MSR_AMD64_SMCA_MCx_ADDR(bank); -} - -static inline u32 smca_misc_reg(int bank) -{ - return MSR_AMD64_SMCA_MCx_MISC(bank); -} - -struct mca_msr_regs msr_ops = { - .ctl = ctl_reg, - .status = status_reg, - .addr = addr_reg, - .misc = misc_reg -}; - static void __print_mce(struct mce *m) { pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n", @@ -348,8 +319,6 @@ static noinstr void mce_panic(const char *msg, struct mce *final, char *exp) if (!apei_err) apei_err = apei_write_mce(final); } - if (cpu_missing) - pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n"); if (exp) pr_emerg(HW_ERR "Machine check: %s\n", exp); if (!fake_panic) { @@ -371,24 +340,27 @@ static int msr_to_offset(u32 msr) if (msr == mca_cfg.rip_msr) return offsetof(struct mce, ip); - if (msr == msr_ops.status(bank)) + if (msr == mca_msr_reg(bank, MCA_STATUS)) return offsetof(struct mce, status); - if (msr == msr_ops.addr(bank)) + if (msr == mca_msr_reg(bank, MCA_ADDR)) return offsetof(struct mce, addr); - if (msr == msr_ops.misc(bank)) + if (msr == mca_msr_reg(bank, MCA_MISC)) return offsetof(struct mce, misc); if (msr == MSR_IA32_MCG_STATUS) return offsetof(struct mce, mcgstatus); return -1; } -__visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) +void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr) { - pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", - (unsigned int)regs->cx, regs->ip, (void *)regs->ip); + if (wrmsr) { + pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", + (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax, + regs->ip, (void *)regs->ip); + } else { + pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", + (unsigned int)regs->cx, regs->ip, (void *)regs->ip); + } show_stack_regs(regs); @@ -396,12 +368,10 @@ __visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup, while (true) cpu_relax(); - - return true; } /* MSR access wrappers used for error injection */ -static noinstr u64 mce_rdmsrl(u32 msr) +noinstr u64 mce_rdmsrl(u32 msr) { DECLARE_ARGS(val, low, high); @@ -429,32 +399,13 @@ static noinstr u64 mce_rdmsrl(u32 msr) */ asm volatile("1: rdmsr\n" "2:\n" - _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_rdmsr_fault) + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR_IN_MCE) : EAX_EDX_RET(val, low, high) : "c" (msr)); return EAX_EDX_VAL(val, low, high); } -__visible bool ex_handler_wrmsr_fault(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) -{ - pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", - (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax, - regs->ip, (void *)regs->ip); - - show_stack_regs(regs); - - panic("MCA architectural violation!\n"); - - while (true) - cpu_relax(); - - return true; -} - static noinstr void mce_wrmsrl(u32 msr, u64 v) { u32 low, high; @@ -479,7 +430,7 @@ static noinstr void mce_wrmsrl(u32 msr, u64 v) /* See comment in mce_rdmsrl() */ asm volatile("1: wrmsr\n" "2:\n" - _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_wrmsr_fault) + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR_IN_MCE) : : "c" (msr), "a"(low), "d" (high) : "memory"); } @@ -488,9 +439,15 @@ static noinstr void mce_wrmsrl(u32 msr, u64 v) * check into our "mce" struct so that we can use it later to assess * the severity of the problem as we read per-bank specific details. */ -static inline void mce_gather_info(struct mce *m, struct pt_regs *regs) +static noinstr void mce_gather_info(struct mce *m, struct pt_regs *regs) { + /* + * Enable instrumentation around mce_setup() which calls external + * facilities. + */ + instrumentation_begin(); mce_setup(m); + instrumentation_end(); m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); if (regs) { @@ -694,10 +651,10 @@ static struct notifier_block mce_default_nb = { static noinstr void mce_read_aux(struct mce *m, int i) { if (m->status & MCI_STATUS_MISCV) - m->misc = mce_rdmsrl(msr_ops.misc(i)); + m->misc = mce_rdmsrl(mca_msr_reg(i, MCA_MISC)); if (m->status & MCI_STATUS_ADDRV) { - m->addr = mce_rdmsrl(msr_ops.addr(i)); + m->addr = mce_rdmsrl(mca_msr_reg(i, MCA_ADDR)); /* * Mask the reported address by the reported granularity. @@ -767,7 +724,7 @@ bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b) m.bank = i; barrier(); - m.status = mce_rdmsrl(msr_ops.status(i)); + m.status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); /* If this entry is not valid, ignore it */ if (!(m.status & MCI_STATUS_VAL)) @@ -835,7 +792,7 @@ bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b) /* * Clear state for this bank. */ - mce_wrmsrl(msr_ops.status(i), 0); + mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); } /* @@ -849,6 +806,34 @@ bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b) } EXPORT_SYMBOL_GPL(machine_check_poll); +/* + * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and + * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM + * Vol 3B Table 15-20). But this confuses both the code that determines + * whether the machine check occurred in kernel or user mode, and also + * the severity assessment code. Pretend that EIPV was set, and take the + * ip/cs values from the pt_regs that mce_gather_info() ignored earlier. + */ +static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) +{ + if (bank != 0) + return; + if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0) + return; + if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC| + MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV| + MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR| + MCACOD)) != + (MCI_STATUS_UC|MCI_STATUS_EN| + MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S| + MCI_STATUS_AR|MCACOD_INSTR)) + return; + + m->mcgstatus |= MCG_STATUS_EIPV; + m->ip = regs->ip; + m->cs = regs->cs; +} + /* * Do a quick check if any of the events requires a panic. * This decides if we keep the events around or clear them. @@ -860,13 +845,13 @@ static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp, int i; for (i = 0; i < this_cpu_read(mce_num_banks); i++) { - m->status = mce_rdmsrl(msr_ops.status(i)); + m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); if (!(m->status & MCI_STATUS_VAL)) continue; __set_bit(i, validp); - if (quirk_no_way_out) - quirk_no_way_out(i, m, regs); + if (mce_flags.snb_ifu_quirk) + quirk_sandybridge_ifu(i, m, regs); m->bank = i; if (mce_severity(m, regs, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) { @@ -898,8 +883,13 @@ static cpumask_t mce_missing_cpus = CPU_MASK_ALL; /* * Check if a timeout waiting for other CPUs happened. */ -static int mce_timed_out(u64 *t, const char *msg) +static noinstr int mce_timed_out(u64 *t, const char *msg) { + int ret = 0; + + /* Enable instrumentation around calls to external facilities */ + instrumentation_begin(); + /* * The others already did panic for some reason. * Bail out like in a timeout. @@ -918,13 +908,17 @@ static int mce_timed_out(u64 *t, const char *msg) cpumask_pr_args(&mce_missing_cpus)); mce_panic(msg, NULL, NULL); } - cpu_missing = 1; - return 1; + ret = 1; + goto out; } *t -= SPINUNIT; + out: touch_nmi_watchdog(); - return 0; + + instrumentation_end(); + + return ret; } /* @@ -1013,14 +1007,13 @@ static atomic_t global_nwo; * in the entry order. * TBD double check parallel CPU hotunplug */ -static int mce_start(int *no_way_out) +static noinstr int mce_start(int *no_way_out) { - int order; - int cpus = num_online_cpus(); u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; + int order, ret = -1; if (!timeout) - return -1; + return ret; atomic_add(*no_way_out, &global_nwo); /* @@ -1030,14 +1023,17 @@ static int mce_start(int *no_way_out) order = atomic_inc_return(&mce_callin); cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus); + /* Enable instrumentation around calls to external facilities */ + instrumentation_begin(); + /* * Wait for everyone. */ - while (atomic_read(&mce_callin) != cpus) { + while (atomic_read(&mce_callin) != num_online_cpus()) { if (mce_timed_out(&timeout, "Timeout: Not all CPUs entered broadcast exception handler")) { atomic_set(&global_nwo, 0); - return -1; + goto out; } ndelay(SPINUNIT); } @@ -1063,7 +1059,7 @@ static int mce_start(int *no_way_out) if (mce_timed_out(&timeout, "Timeout: Subject CPUs unable to finish machine check processing")) { atomic_set(&global_nwo, 0); - return -1; + goto out; } ndelay(SPINUNIT); } @@ -1074,7 +1070,12 @@ static int mce_start(int *no_way_out) */ *no_way_out = atomic_read(&global_nwo); - return order; + ret = order; + +out: + instrumentation_end(); + + return ret; } /* @@ -1100,14 +1101,11 @@ static noinstr int mce_end(int order) atomic_inc(&mce_executing); if (order == 1) { - /* CHECKME: Can this race with a parallel hotplug? */ - int cpus = num_online_cpus(); - /* * Monarch: Wait for everyone to go through their scanning * loops. */ - while (atomic_read(&mce_executing) <= cpus) { + while (atomic_read(&mce_executing) <= num_online_cpus()) { if (mce_timed_out(&timeout, "Timeout: Monarch CPU unable to finish machine check processing")) goto reset; @@ -1161,7 +1159,7 @@ static void mce_clear_state(unsigned long *toclear) for (i = 0; i < this_cpu_read(mce_num_banks); i++) { if (test_bit(i, toclear)) - mce_wrmsrl(msr_ops.status(i), 0); + mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); } } @@ -1200,13 +1198,14 @@ static noinstr bool mce_check_crashing_cpu(void) return false; } -static void __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final, - unsigned long *toclear, unsigned long *valid_banks, - int no_way_out, int *worst) +static __always_inline int +__mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final, + unsigned long *toclear, unsigned long *valid_banks, int no_way_out, + int *worst) { struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); struct mca_config *cfg = &mca_cfg; - int severity, i; + int severity, i, taint = 0; for (i = 0; i < this_cpu_read(mce_num_banks); i++) { __clear_bit(i, toclear); @@ -1220,7 +1219,7 @@ static void __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *fin m->addr = 0; m->bank = i; - m->status = mce_rdmsrl(msr_ops.status(i)); + m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); if (!(m->status & MCI_STATUS_VAL)) continue; @@ -1233,7 +1232,7 @@ static void __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *fin continue; /* Set taint even when machine check was not enabled. */ - add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); + taint++; severity = mce_severity(m, regs, cfg->tolerant, NULL, true); @@ -1256,7 +1255,13 @@ static void __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *fin /* assuming valid severity level != 0 */ m->severity = severity; + /* + * Enable instrumentation around the mce_log() call which is + * done in #MC context, where instrumentation is disabled. + */ + instrumentation_begin(); mce_log(m); + instrumentation_end(); if (severity > *worst) { *final = *m; @@ -1266,6 +1271,8 @@ static void __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *fin /* mce_clear_state will clear *final, save locally for use later */ *m = *final; + + return taint; } static void kill_me_now(struct callback_head *ch) @@ -1289,7 +1296,7 @@ static void kill_me_maybe(struct callback_head *cb) flags |= MF_MUST_KILL; ret = memory_failure(p->mce_addr >> PAGE_SHIFT, flags); - if (!ret && !(p->mce_kflags & MCE_IN_KERNEL_COPYIN)) { + if (!ret) { set_mce_nospec(p->mce_addr >> PAGE_SHIFT, p->mce_whole_page); sync_core(); return; @@ -1303,15 +1310,21 @@ static void kill_me_maybe(struct callback_head *cb) if (ret == -EHWPOISON) return; - if (p->mce_vaddr != (void __user *)-1l) { - force_sig_mceerr(BUS_MCEERR_AR, p->mce_vaddr, PAGE_SHIFT); - } else { - pr_err("Memory error not recovered"); - kill_me_now(cb); - } + pr_err("Memory error not recovered"); + kill_me_now(cb); } -static void queue_task_work(struct mce *m, char *msg, int kill_current_task) +static void kill_me_never(struct callback_head *cb) +{ + struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); + + p->mce_count = 0; + pr_err("Kernel accessed poison in user space at %llx\n", p->mce_addr); + if (!memory_failure(p->mce_addr >> PAGE_SHIFT, 0)) + set_mce_nospec(p->mce_addr >> PAGE_SHIFT, p->mce_whole_page); +} + +static void queue_task_work(struct mce *m, char *msg, void (*func)(struct callback_head *)) { int count = ++current->mce_count; @@ -1321,11 +1334,7 @@ static void queue_task_work(struct mce *m, char *msg, int kill_current_task) current->mce_kflags = m->kflags; current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV); current->mce_whole_page = whole_page(m); - - if (kill_current_task) - current->mce_kill_me.func = kill_me_now; - else - current->mce_kill_me.func = kill_me_maybe; + current->mce_kill_me.func = func; } /* Ten is likely overkill. Don't expect more than two faults before task_work() */ @@ -1343,12 +1352,21 @@ static void queue_task_work(struct mce *m, char *msg, int kill_current_task) task_work_add(current, ¤t->mce_kill_me, TWA_RESUME); } +/* Handle unconfigured int18 (should never happen) */ +static noinstr void unexpected_machine_check(struct pt_regs *regs) +{ + instrumentation_begin(); + pr_err("CPU#%d: Unexpected int18 (Machine Check)\n", + smp_processor_id()); + instrumentation_end(); +} + /* - * The actual machine check handler. This only handles real - * exceptions when something got corrupted coming in through int 18. + * The actual machine check handler. This only handles real exceptions when + * something got corrupted coming in through int 18. * - * This is executed in NMI context not subject to normal locking rules. This - * implies that most kernel services cannot be safely used. Don't even + * This is executed in #MC context not subject to normal locking rules. + * This implies that most kernel services cannot be safely used. Don't even * think about putting a printk in there! * * On Intel systems this is entered on all CPUs in parallel through @@ -1360,39 +1378,54 @@ static void queue_task_work(struct mce *m, char *msg, int kill_current_task) * issues: if the machine check was due to a failure of the memory * backing the user stack, tracing that reads the user stack will cause * potentially infinite recursion. + * + * Currently, the #MC handler calls out to a number of external facilities + * and, therefore, allows instrumentation around them. The optimal thing to + * have would be to do the absolutely minimal work required in #MC context + * and have instrumentation disabled only around that. Further processing can + * then happen in process context where instrumentation is allowed. Achieving + * that requires careful auditing and modifications. Until then, the code + * allows instrumentation temporarily, where required. * */ noinstr void do_machine_check(struct pt_regs *regs) { - DECLARE_BITMAP(valid_banks, MAX_NR_BANKS); - DECLARE_BITMAP(toclear, MAX_NR_BANKS); + int worst = 0, order, no_way_out, kill_current_task, lmce, taint = 0; + DECLARE_BITMAP(valid_banks, MAX_NR_BANKS) = { 0 }; + DECLARE_BITMAP(toclear, MAX_NR_BANKS) = { 0 }; struct mca_config *cfg = &mca_cfg; struct mce m, *final; char *msg = NULL; - int worst = 0; + + if (unlikely(mce_flags.p5)) + return pentium_machine_check(regs); + else if (unlikely(mce_flags.winchip)) + return winchip_machine_check(regs); + else if (unlikely(!mca_cfg.initialized)) + return unexpected_machine_check(regs); /* * Establish sequential order between the CPUs entering the machine * check handler. */ - int order = -1; + order = -1; /* * If no_way_out gets set, there is no safe way to recover from this * MCE. If mca_cfg.tolerant is cranked up, we'll try anyway. */ - int no_way_out = 0; + no_way_out = 0; /* * If kill_current_task is not set, there might be a way to recover from this * error. */ - int kill_current_task = 0; + kill_current_task = 0; /* * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES * on Intel. */ - int lmce = 1; + lmce = 1; this_cpu_inc(mce_exception_count); @@ -1402,7 +1435,6 @@ noinstr void do_machine_check(struct pt_regs *regs) final = this_cpu_ptr(&mces_seen); *final = m; - memset(valid_banks, 0, sizeof(valid_banks)); no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs); barrier(); @@ -1436,7 +1468,7 @@ noinstr void do_machine_check(struct pt_regs *regs) order = mce_start(&no_way_out); } - __mc_scan_banks(&m, regs, final, toclear, valid_banks, no_way_out, &worst); + taint = __mc_scan_banks(&m, regs, final, toclear, valid_banks, no_way_out, &worst); if (!no_way_out) mce_clear_state(toclear); @@ -1468,23 +1500,28 @@ noinstr void do_machine_check(struct pt_regs *regs) } } - if (worst != MCE_AR_SEVERITY && !kill_current_task) - goto out; - /* - * Enable instrumentation around the external facilities like - * task_work_add() (via queue_task_work()), fixup_exception() etc. - * For now, that is. Fixing this properly would need a lot more involved - * reorganization. + * Enable instrumentation around the external facilities like task_work_add() + * (via queue_task_work()), fixup_exception() etc. For now, that is. Fixing this + * properly would need a lot more involved reorganization. */ instrumentation_begin(); + if (taint) + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); + + if (worst != MCE_AR_SEVERITY && !kill_current_task) + goto out; + /* Fault was in user mode and we need to take some action */ if ((m.cs & 3) == 3) { /* If this triggers there is no way to recover. Die hard. */ BUG_ON(!on_thread_stack() || !user_mode(regs)); - queue_task_work(&m, msg, kill_current_task); + if (kill_current_task) + queue_task_work(&m, msg, kill_me_now); + else + queue_task_work(&m, msg, kill_me_maybe); } else { /* @@ -1502,12 +1539,12 @@ noinstr void do_machine_check(struct pt_regs *regs) } if (m.kflags & MCE_IN_KERNEL_COPYIN) - queue_task_work(&m, msg, kill_current_task); + queue_task_work(&m, msg, kill_me_never); } +out: instrumentation_end(); -out: mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); } EXPORT_SYMBOL_GPL(do_machine_check); @@ -1715,8 +1752,8 @@ static void __mcheck_cpu_init_clear_banks(void) if (!b->init) continue; - wrmsrl(msr_ops.ctl(i), b->ctl); - wrmsrl(msr_ops.status(i), 0); + wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl); + wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); } } @@ -1742,39 +1779,11 @@ static void __mcheck_cpu_check_banks(void) if (!b->init) continue; - rdmsrl(msr_ops.ctl(i), msrval); + rdmsrl(mca_msr_reg(i, MCA_CTL), msrval); b->init = !!msrval; } } -/* - * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and - * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM - * Vol 3B Table 15-20). But this confuses both the code that determines - * whether the machine check occurred in kernel or user mode, and also - * the severity assessment code. Pretend that EIPV was set, and take the - * ip/cs values from the pt_regs that mce_gather_info() ignored earlier. - */ -static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) -{ - if (bank != 0) - return; - if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0) - return; - if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC| - MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV| - MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR| - MCACOD)) != - (MCI_STATUS_UC|MCI_STATUS_EN| - MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S| - MCI_STATUS_AR|MCACOD_INSTR)) - return; - - m->mcgstatus |= MCG_STATUS_EIPV; - m->ip = regs->ip; - m->cs = regs->cs; -} - /* Add per CPU specific workarounds here */ static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) { @@ -1848,7 +1857,7 @@ static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) cfg->bootlog = 0; if (c->x86 == 6 && c->x86_model == 45) - quirk_no_way_out = quirk_sandybridge_ifu; + mce_flags.snb_ifu_quirk = 1; } if (c->x86_vendor == X86_VENDOR_ZHAOXIN) { @@ -1878,9 +1887,11 @@ static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) switch (c->x86_vendor) { case X86_VENDOR_INTEL: intel_p5_mcheck_init(c); + mce_flags.p5 = 1; return 1; case X86_VENDOR_CENTAUR: winchip_mcheck_init(c); + mce_flags.winchip = 1; return 1; default: return 0; @@ -1899,13 +1910,6 @@ static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c) mce_flags.succor = !!cpu_has(c, X86_FEATURE_SUCCOR); mce_flags.smca = !!cpu_has(c, X86_FEATURE_SMCA); mce_flags.amd_threshold = 1; - - if (mce_flags.smca) { - msr_ops.ctl = smca_ctl_reg; - msr_ops.status = smca_status_reg; - msr_ops.addr = smca_addr_reg; - msr_ops.misc = smca_misc_reg; - } } } @@ -2035,18 +2039,6 @@ bool filter_mce(struct mce *m) return false; } -/* Handle unconfigured int18 (should never happen) */ -static noinstr void unexpected_machine_check(struct pt_regs *regs) -{ - instrumentation_begin(); - pr_err("CPU#%d: Unexpected int18 (Machine Check)\n", - smp_processor_id()); - instrumentation_end(); -} - -/* Call the installed machine check handler for this CPU setup. */ -void (*machine_check_vector)(struct pt_regs *) = unexpected_machine_check; - static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) { irqentry_state_t irq_state; @@ -2057,31 +2049,22 @@ static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) * Only required when from kernel mode. See * mce_check_crashing_cpu() for details. */ - if (machine_check_vector == do_machine_check && - mce_check_crashing_cpu()) + if (mca_cfg.initialized && mce_check_crashing_cpu()) return; irq_state = irqentry_nmi_enter(regs); - /* - * The call targets are marked noinstr, but objtool can't figure - * that out because it's an indirect call. Annotate it. - */ - instrumentation_begin(); - machine_check_vector(regs); + do_machine_check(regs); - instrumentation_end(); irqentry_nmi_exit(regs, irq_state); } static __always_inline void exc_machine_check_user(struct pt_regs *regs) { irqentry_enter_from_user_mode(regs); - instrumentation_begin(); - machine_check_vector(regs); + do_machine_check(regs); - instrumentation_end(); irqentry_exit_to_user_mode(regs); } @@ -2148,7 +2131,7 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c) return; } - machine_check_vector = do_machine_check; + mca_cfg.initialized = 1; __mcheck_cpu_init_early(c); __mcheck_cpu_init_generic(); @@ -2256,7 +2239,6 @@ int __init mcheck_init(void) mce_register_decode_chain(&early_nb); mce_register_decode_chain(&mce_uc_nb); mce_register_decode_chain(&mce_default_nb); - mcheck_vendor_init_severity(); INIT_WORK(&mce_work, mce_gen_pool_process); init_irq_work(&mce_irq_work, mce_irq_work_cb); @@ -2281,7 +2263,7 @@ static void mce_disable_error_reporting(void) struct mce_bank *b = &mce_banks[i]; if (b->init) - wrmsrl(msr_ops.ctl(i), 0); + wrmsrl(mca_msr_reg(i, MCA_CTL), 0); } return; } @@ -2633,7 +2615,7 @@ static void mce_reenable_cpu(void) struct mce_bank *b = &mce_banks[i]; if (b->init) - wrmsrl(msr_ops.ctl(i), b->ctl); + wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl); } } @@ -2782,7 +2764,6 @@ struct dentry *mce_get_debugfs_dir(void) static void mce_reset(void) { - cpu_missing = 0; atomic_set(&mce_fake_panicked, 0); atomic_set(&mce_executing, 0); atomic_set(&mce_callin, 0); diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index b63b548497..5fbd7ffb32 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -74,7 +74,6 @@ MCE_INJECT_SET(status); MCE_INJECT_SET(misc); MCE_INJECT_SET(addr); MCE_INJECT_SET(synd); -MCE_INJECT_SET(ipid); #define MCE_INJECT_GET(reg) \ static int inj_##reg##_get(void *data, u64 *val) \ @@ -95,6 +94,20 @@ DEFINE_SIMPLE_ATTRIBUTE(status_fops, inj_status_get, inj_status_set, "%llx\n"); DEFINE_SIMPLE_ATTRIBUTE(misc_fops, inj_misc_get, inj_misc_set, "%llx\n"); DEFINE_SIMPLE_ATTRIBUTE(addr_fops, inj_addr_get, inj_addr_set, "%llx\n"); DEFINE_SIMPLE_ATTRIBUTE(synd_fops, inj_synd_get, inj_synd_set, "%llx\n"); + +/* Use the user provided IPID value on a sw injection. */ +static int inj_ipid_set(void *data, u64 val) +{ + struct mce *m = (struct mce *)data; + + if (cpu_feature_enabled(X86_FEATURE_SMCA)) { + if (inj_type == SW_INJ) + m->ipid = val; + } + + return 0; +} + DEFINE_SIMPLE_ATTRIBUTE(ipid_fops, inj_ipid_get, inj_ipid_set, "%llx\n"); static void setup_inj_struct(struct mce *m) @@ -490,6 +503,8 @@ static void do_inject(void) i_mce.tsc = rdtsc_ordered(); + i_mce.status |= MCI_STATUS_VAL; + if (i_mce.misc) i_mce.status |= MCI_STATUS_MISCV; @@ -577,6 +592,33 @@ static int inj_bank_set(void *data, u64 val) } m->bank = val; + + /* + * sw-only injection allows to write arbitrary values into the MCA + * registers because it tests only the decoding paths. + */ + if (inj_type == SW_INJ) + goto inject; + + /* + * Read IPID value to determine if a bank is populated on the target + * CPU. + */ + if (cpu_feature_enabled(X86_FEATURE_SMCA)) { + u64 ipid; + + if (rdmsrl_on_cpu(m->extcpu, MSR_AMD64_SMCA_MCx_IPID(val), &ipid)) { + pr_err("Error reading IPID on CPU%d\n", m->extcpu); + return -EINVAL; + } + + if (!ipid) { + pr_err("Cannot inject into unpopulated bank %llu\n", val); + return -ENODEV; + } + } + +inject: do_inject(); /* Reset injection struct */ diff --git a/arch/x86/kernel/cpu/mce/internal.h b/arch/x86/kernel/cpu/mce/internal.h index 88dcc79cfb..52c633950b 100644 --- a/arch/x86/kernel/cpu/mce/internal.h +++ b/arch/x86/kernel/cpu/mce/internal.h @@ -8,9 +8,6 @@ #include #include -/* Pointer to the installed machine check handler for this CPU setup. */ -extern void (*machine_check_vector)(struct pt_regs *); - enum severity_level { MCE_NO_SEVERITY, MCE_DEFERRED_SEVERITY, @@ -38,8 +35,7 @@ int mce_gen_pool_add(struct mce *mce); int mce_gen_pool_init(void); struct llist_node *mce_gen_pool_prepare_records(void); -extern int (*mce_severity)(struct mce *a, struct pt_regs *regs, - int tolerant, char **msg, bool is_excp); +int mce_severity(struct mce *a, struct pt_regs *regs, int tolerant, char **msg, bool is_excp); struct dentry *mce_get_debugfs_dir(void); extern mce_banks_t mce_banks_ce_disabled; @@ -61,7 +57,7 @@ static inline void cmci_disable_bank(int bank) { } static inline void intel_init_cmci(void) { } static inline void intel_init_lmce(void) { } static inline void intel_clear_lmce(void) { } -static inline bool intel_filter_mce(struct mce *m) { return false; }; +static inline bool intel_filter_mce(struct mce *m) { return false; } #endif void mce_timer_kick(unsigned long interval); @@ -117,23 +113,25 @@ static inline void mce_unregister_injector_chain(struct notifier_block *nb) { } #endif struct mca_config { - bool dont_log_ce; - bool cmci_disabled; - bool ignore_ce; - bool print_all; - __u64 lmce_disabled : 1, disabled : 1, ser : 1, recovery : 1, bios_cmci_threshold : 1, - __reserved : 59; + /* Proper #MC exception handler is set */ + initialized : 1, + __reserved : 58; + + bool dont_log_ce; + bool cmci_disabled; + bool ignore_ce; + bool print_all; - s8 bootlog; int tolerant; int monarch_timeout; int panic_timeout; u32 rip_msr; + s8 bootlog; }; extern struct mca_config mca_cfg; @@ -163,19 +161,28 @@ struct mce_vendor_flags { /* AMD-style error thresholding banks present. */ amd_threshold : 1, - __reserved_0 : 60; + /* Pentium, family 5-style MCA */ + p5 : 1, + + /* Centaur Winchip C6-style MCA */ + winchip : 1, + + /* SandyBridge IFU quirk */ + snb_ifu_quirk : 1, + + __reserved_0 : 57; }; extern struct mce_vendor_flags mce_flags; -struct mca_msr_regs { - u32 (*ctl) (int bank); - u32 (*status) (int bank); - u32 (*addr) (int bank); - u32 (*misc) (int bank); +enum mca_msr { + MCA_CTL, + MCA_STATUS, + MCA_ADDR, + MCA_MISC, }; -extern struct mca_msr_regs msr_ops; +u32 mca_msr_reg(int bank, enum mca_msr reg); /* Decide whether to add MCE record to MCE event pool or filter it out. */ extern bool filter_mce(struct mce *m); @@ -183,17 +190,23 @@ extern bool filter_mce(struct mce *m); #ifdef CONFIG_X86_MCE_AMD extern bool amd_filter_mce(struct mce *m); #else -static inline bool amd_filter_mce(struct mce *m) { return false; }; +static inline bool amd_filter_mce(struct mce *m) { return false; } #endif -__visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr); +#ifdef CONFIG_X86_ANCIENT_MCE +void intel_p5_mcheck_init(struct cpuinfo_x86 *c); +void winchip_mcheck_init(struct cpuinfo_x86 *c); +noinstr void pentium_machine_check(struct pt_regs *regs); +noinstr void winchip_machine_check(struct pt_regs *regs); +static inline void enable_p5_mce(void) { mce_p5_enabled = 1; } +#else +static inline void intel_p5_mcheck_init(struct cpuinfo_x86 *c) {} +static inline void winchip_mcheck_init(struct cpuinfo_x86 *c) {} +static inline void enable_p5_mce(void) {} +static inline void pentium_machine_check(struct pt_regs *regs) {} +static inline void winchip_machine_check(struct pt_regs *regs) {} +#endif -__visible bool ex_handler_wrmsr_fault(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr); +noinstr u64 mce_rdmsrl(u32 msr); #endif /* __X86_MCE_INTERNAL_H__ */ diff --git a/arch/x86/kernel/cpu/mce/p5.c b/arch/x86/kernel/cpu/mce/p5.c index 19e90cae8e..2272ad53fc 100644 --- a/arch/x86/kernel/cpu/mce/p5.c +++ b/arch/x86/kernel/cpu/mce/p5.c @@ -21,7 +21,7 @@ int mce_p5_enabled __read_mostly; /* Machine check handler for Pentium class Intel CPUs: */ -static noinstr void pentium_machine_check(struct pt_regs *regs) +noinstr void pentium_machine_check(struct pt_regs *regs) { u32 loaddr, hi, lotype; @@ -54,10 +54,6 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c) if (!cpu_has(c, X86_FEATURE_MCE)) return; - machine_check_vector = pentium_machine_check; - /* Make sure the vector pointer is visible before we enable MCEs: */ - wmb(); - /* Read registers before enabling: */ rdmsr(MSR_IA32_P5_MC_ADDR, l, h); rdmsr(MSR_IA32_P5_MC_TYPE, l, h); diff --git a/arch/x86/kernel/cpu/mce/severity.c b/arch/x86/kernel/cpu/mce/severity.c index 17e6314431..7aa2bda93c 100644 --- a/arch/x86/kernel/cpu/mce/severity.c +++ b/arch/x86/kernel/cpu/mce/severity.c @@ -222,6 +222,9 @@ static bool is_copy_from_user(struct pt_regs *regs) struct insn insn; int ret; + if (!regs) + return false; + if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip, MAX_INSN_SIZE)) return false; @@ -263,33 +266,44 @@ static bool is_copy_from_user(struct pt_regs *regs) * distinguish an exception taken in user from from one * taken in the kernel. */ -static int error_context(struct mce *m, struct pt_regs *regs) +static noinstr int error_context(struct mce *m, struct pt_regs *regs) { - enum handler_type t; + int fixup_type; + bool copy_user; if ((m->cs & 3) == 3) return IN_USER; + if (!mc_recoverable(m->mcgstatus)) return IN_KERNEL; - t = ex_get_fault_handler_type(m->ip); - if (t == EX_HANDLER_FAULT) { - m->kflags |= MCE_IN_KERNEL_RECOV; - return IN_KERNEL_RECOV; - } - if (t == EX_HANDLER_UACCESS && regs && is_copy_from_user(regs)) { - m->kflags |= MCE_IN_KERNEL_RECOV; - m->kflags |= MCE_IN_KERNEL_COPYIN; - return IN_KERNEL_RECOV; - } + /* Allow instrumentation around external facilities usage. */ + instrumentation_begin(); + fixup_type = ex_get_fixup_type(m->ip); + copy_user = is_copy_from_user(regs); + instrumentation_end(); - return IN_KERNEL; + switch (fixup_type) { + case EX_TYPE_UACCESS: + case EX_TYPE_COPY: + if (!copy_user) + return IN_KERNEL; + m->kflags |= MCE_IN_KERNEL_COPYIN; + fallthrough; + + case EX_TYPE_FAULT_MCE_SAFE: + case EX_TYPE_DEFAULT_MCE_SAFE: + m->kflags |= MCE_IN_KERNEL_RECOV; + return IN_KERNEL_RECOV; + + default: + return IN_KERNEL; + } } static int mce_severity_amd_smca(struct mce *m, enum context err_ctx) { - u32 addr = MSR_AMD64_SMCA_MCx_CONFIG(m->bank); - u32 low, high; + u64 mcx_cfg; /* * We need to look at the following bits: @@ -300,11 +314,10 @@ static int mce_severity_amd_smca(struct mce *m, enum context err_ctx) if (!mce_flags.succor) return MCE_PANIC_SEVERITY; - if (rdmsr_safe(addr, &low, &high)) - return MCE_PANIC_SEVERITY; + mcx_cfg = mce_rdmsrl(MSR_AMD64_SMCA_MCx_CONFIG(m->bank)); /* TCC (Task context corrupt). If set and if IN_KERNEL, panic. */ - if ((low & MCI_CONFIG_MCAX) && + if ((mcx_cfg & MCI_CONFIG_MCAX) && (m->status & MCI_STATUS_TCC) && (err_ctx == IN_KERNEL)) return MCE_PANIC_SEVERITY; @@ -317,8 +330,8 @@ static int mce_severity_amd_smca(struct mce *m, enum context err_ctx) * See AMD Error Scope Hierarchy table in a newer BKDG. For example * 49125_15h_Models_30h-3Fh_BKDG.pdf, section "RAS Features" */ -static int mce_severity_amd(struct mce *m, struct pt_regs *regs, int tolerant, - char **msg, bool is_excp) +static noinstr int mce_severity_amd(struct mce *m, struct pt_regs *regs, int tolerant, + char **msg, bool is_excp) { enum context ctx = error_context(m, regs); @@ -370,8 +383,8 @@ static int mce_severity_amd(struct mce *m, struct pt_regs *regs, int tolerant, return MCE_KEEP_SEVERITY; } -static int mce_severity_intel(struct mce *m, struct pt_regs *regs, - int tolerant, char **msg, bool is_excp) +static noinstr int mce_severity_intel(struct mce *m, struct pt_regs *regs, + int tolerant, char **msg, bool is_excp) { enum exception excp = (is_excp ? EXCP_CONTEXT : NO_EXCP); enum context ctx = error_context(m, regs); @@ -407,15 +420,14 @@ static int mce_severity_intel(struct mce *m, struct pt_regs *regs, } } -/* Default to mce_severity_intel */ -int (*mce_severity)(struct mce *m, struct pt_regs *regs, int tolerant, char **msg, bool is_excp) = - mce_severity_intel; - -void __init mcheck_vendor_init_severity(void) +int noinstr mce_severity(struct mce *m, struct pt_regs *regs, int tolerant, char **msg, + bool is_excp) { if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) - mce_severity = mce_severity_amd; + return mce_severity_amd(m, regs, tolerant, msg, is_excp); + else + return mce_severity_intel(m, regs, tolerant, msg, is_excp); } #ifdef CONFIG_DEBUG_FS diff --git a/arch/x86/kernel/cpu/mce/winchip.c b/arch/x86/kernel/cpu/mce/winchip.c index 9c9f0abd2d..6c99f29419 100644 --- a/arch/x86/kernel/cpu/mce/winchip.c +++ b/arch/x86/kernel/cpu/mce/winchip.c @@ -17,7 +17,7 @@ #include "internal.h" /* Machine check handler for WinChip C6: */ -static noinstr void winchip_machine_check(struct pt_regs *regs) +noinstr void winchip_machine_check(struct pt_regs *regs) { instrumentation_begin(); pr_emerg("CPU0: Machine Check Exception.\n"); @@ -30,10 +30,6 @@ void winchip_mcheck_init(struct cpuinfo_x86 *c) { u32 lo, hi; - machine_check_vector = winchip_machine_check; - /* Make sure the vector pointer is visible before we enable MCEs: */ - wmb(); - rdmsr(MSR_IDT_FCR1, lo, hi); lo |= (1<<2); /* Enable EIERRINT (int 18 MCE) */ lo &= ~(1<<4); /* Enable MCE */ diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 3d4a483360..8b2fcdfa6d 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -456,17 +456,23 @@ apply_microcode_early_amd(u32 cpuid_1_eax, void *ucode, size_t size, bool save_p static bool get_builtin_microcode(struct cpio_data *cp, unsigned int family) { -#ifdef CONFIG_X86_64 char fw_name[36] = "amd-ucode/microcode_amd.bin"; + struct firmware fw; + + if (IS_ENABLED(CONFIG_X86_32)) + return false; if (family >= 0x15) snprintf(fw_name, sizeof(fw_name), "amd-ucode/microcode_amd_fam%.2xh.bin", family); - return get_builtin_firmware(cp, fw_name); -#else + if (firmware_request_builtin(&fw, fw_name)) { + cp->size = fw.size; + cp->data = (void *)fw.data; + return true; + } + return false; -#endif } static void __load_ucode_amd(unsigned int cpuid_1_eax, struct cpio_data *ret) diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index efb69be41a..f955d25076 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -140,23 +140,6 @@ static bool __init check_loader_disabled_bsp(void) return *res; } -extern struct builtin_fw __start_builtin_fw[]; -extern struct builtin_fw __end_builtin_fw[]; - -bool get_builtin_firmware(struct cpio_data *cd, const char *name) -{ - struct builtin_fw *b_fw; - - for (b_fw = __start_builtin_fw; b_fw != __end_builtin_fw; b_fw++) { - if (!strcmp(name, b_fw->name)) { - cd->size = b_fw->size; - cd->data = b_fw->data; - return true; - } - } - return false; -} - void __init load_ucode_bsp(void) { unsigned int cpuid_1_eax; diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index 7e8e07bddd..d28a9f8f3f 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -456,6 +456,7 @@ static void save_mc_for_early(struct ucode_cpu_info *uci, u8 *mc, unsigned int s static bool load_builtin_intel_microcode(struct cpio_data *cp) { unsigned int eax = 1, ebx, ecx = 0, edx; + struct firmware fw; char name[30]; if (IS_ENABLED(CONFIG_X86_32)) @@ -466,7 +467,13 @@ static bool load_builtin_intel_microcode(struct cpio_data *cp) sprintf(name, "intel-ucode/%02x-%02x-%02x", x86_family(eax), x86_model(eax), x86_stepping(eax)); - return get_builtin_firmware(cp, name); + if (firmware_request_builtin(&fw, name)) { + cp->size = fw.size; + cp->data = (void *)fw.data; + return true; + } + + return false; } /* diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index ef6316fef9..5a99f993e6 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -79,7 +80,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0) inc_irq_stat(hyperv_stimer0_count); if (hv_stimer0_handler) hv_stimer0_handler(); - add_interrupt_randomness(HYPERV_STIMER0_VECTOR, 0); + add_interrupt_randomness(HYPERV_STIMER0_VECTOR); ack_APIC_irq(); set_irq_regs(old_regs); @@ -323,9 +324,26 @@ static void __init ms_hyperv_init_platform(void) if (ms_hyperv.priv_high & HV_ISOLATION) { ms_hyperv.isolation_config_a = cpuid_eax(HYPERV_CPUID_ISOLATION_CONFIG); ms_hyperv.isolation_config_b = cpuid_ebx(HYPERV_CPUID_ISOLATION_CONFIG); + ms_hyperv.shared_gpa_boundary = + BIT_ULL(ms_hyperv.shared_gpa_boundary_bits); pr_info("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n", ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b); + + if (hv_get_isolation_type() == HV_ISOLATION_TYPE_SNP) { + static_branch_enable(&isolation_type_snp); +#ifdef CONFIG_SWIOTLB + swiotlb_unencrypted_base = ms_hyperv.shared_gpa_boundary; +#endif + } + +#ifdef CONFIG_SWIOTLB + /* + * Enable swiotlb force mode in Isolation VM to + * use swiotlb bounce buffer for dma transaction. + */ + swiotlb_force = SWIOTLB_FORCE; +#endif } if (hv_max_functions_eax >= HYPERV_CPUID_NESTED_FEATURES) { diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index c9f0f3d63f..eaf25a234f 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -282,7 +282,7 @@ static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width) u64 shift = 64 - width, chunks; chunks = (cur_msr << shift) - (prev_msr << shift); - return chunks >>= shift; + return chunks >> shift; } static u64 __mon_event_count(u32 rmid, struct rmid_read *rr) diff --git a/arch/x86/kernel/cpu/sgx/encls.h b/arch/x86/kernel/cpu/sgx/encls.h index 9b204843b7..fa04a73daf 100644 --- a/arch/x86/kernel/cpu/sgx/encls.h +++ b/arch/x86/kernel/cpu/sgx/encls.h @@ -11,26 +11,8 @@ #include #include "sgx.h" -/** - * ENCLS_FAULT_FLAG - flag signifying an ENCLS return code is a trapnr - * - * ENCLS has its own (positive value) error codes and also generates - * ENCLS specific #GP and #PF faults. And the ENCLS values get munged - * with system error codes as everything percolates back up the stack. - * Unfortunately (for us), we need to precisely identify each unique - * error code, e.g. the action taken if EWB fails varies based on the - * type of fault and on the exact SGX error code, i.e. we can't simply - * convert all faults to -EFAULT. - * - * To make all three error types coexist, we set bit 30 to identify an - * ENCLS fault. Bit 31 (technically bits N:31) is used to differentiate - * between positive (faults and SGX error codes) and negative (system - * error codes) values. - */ -#define ENCLS_FAULT_FLAG 0x40000000 - /* Retrieve the encoded trapnr from the specified return code. */ -#define ENCLS_TRAPNR(r) ((r) & ~ENCLS_FAULT_FLAG) +#define ENCLS_TRAPNR(r) ((r) & ~SGX_ENCLS_FAULT_FLAG) /* Issue a WARN() about an ENCLS function. */ #define ENCLS_WARN(r, name) { \ @@ -50,7 +32,7 @@ */ static inline bool encls_faulted(int ret) { - return ret & ENCLS_FAULT_FLAG; + return ret & SGX_ENCLS_FAULT_FLAG; } /** @@ -88,11 +70,7 @@ static inline bool encls_failed(int ret) asm volatile( \ "1: .byte 0x0f, 0x01, 0xcf;\n\t" \ "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: orl $"__stringify(ENCLS_FAULT_FLAG)",%%eax\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE_FAULT(1b, 3b) \ + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_SGX) \ : "=a"(ret) \ : "a"(rax), inputs \ : "memory", "cc"); \ @@ -127,7 +105,7 @@ static inline bool encls_failed(int ret) * * Return: * 0 on success, - * trapnr with ENCLS_FAULT_FLAG set on fault + * trapnr with SGX_ENCLS_FAULT_FLAG set on fault */ #define __encls_N(rax, rbx_out, inputs...) \ ({ \ @@ -136,11 +114,7 @@ static inline bool encls_failed(int ret) "1: .byte 0x0f, 0x01, 0xcf;\n\t" \ " xor %%eax,%%eax;\n" \ "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: orl $"__stringify(ENCLS_FAULT_FLAG)",%%eax\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE_FAULT(1b, 3b) \ + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_SGX) \ : "=a"(ret), "=b"(rbx_out) \ : "a"(rax), inputs \ : "memory"); \ diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 8471a8b9b4..8e4bc6453d 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -6,11 +6,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include "driver.h" #include "encl.h" @@ -20,6 +22,7 @@ struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; static int sgx_nr_epc_sections; static struct task_struct *ksgxd_tsk; static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq); +static DEFINE_XARRAY(sgx_epc_address_space); /* * These variables are part of the state of the reclaimer, and must be accessed @@ -60,6 +63,24 @@ static void __sgx_sanitize_pages(struct list_head *dirty_page_list) page = list_first_entry(dirty_page_list, struct sgx_epc_page, list); + /* + * Checking page->poison without holding the node->lock + * is racy, but losing the race (i.e. poison is set just + * after the check) just means __eremove() will be uselessly + * called for a page that sgx_free_epc_page() will put onto + * the node->sgx_poison_page_list later. + */ + if (page->poison) { + struct sgx_epc_section *section = &sgx_epc_sections[page->section]; + struct sgx_numa_node *node = section->node; + + spin_lock(&node->lock); + list_move(&page->list, &node->sgx_poison_page_list); + spin_unlock(&node->lock); + + continue; + } + ret = __eremove(sgx_get_epc_virt_addr(page)); if (!ret) { /* @@ -323,10 +344,8 @@ static void sgx_reclaim_pages(void) { struct sgx_epc_page *chunk[SGX_NR_TO_SCAN]; struct sgx_backing backing[SGX_NR_TO_SCAN]; - struct sgx_epc_section *section; struct sgx_encl_page *encl_page; struct sgx_epc_page *epc_page; - struct sgx_numa_node *node; pgoff_t page_index; int cnt = 0; int ret; @@ -397,13 +416,7 @@ static void sgx_reclaim_pages(void) kref_put(&encl_page->encl->refcount, sgx_encl_release); epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; - section = &sgx_epc_sections[epc_page->section]; - node = section->node; - - spin_lock(&node->lock); - list_add_tail(&epc_page->list, &node->free_page_list); - spin_unlock(&node->lock); - atomic_long_inc(&sgx_nr_free_pages); + sgx_free_epc_page(epc_page); } } @@ -471,6 +484,7 @@ static struct sgx_epc_page *__sgx_alloc_epc_page_from_node(int nid) page = list_first_entry(&node->free_page_list, struct sgx_epc_page, list); list_del_init(&page->list); + page->flags = 0; spin_unlock(&node->lock); atomic_long_dec(&sgx_nr_free_pages); @@ -624,7 +638,12 @@ void sgx_free_epc_page(struct sgx_epc_page *page) spin_lock(&node->lock); - list_add_tail(&page->list, &node->free_page_list); + page->owner = NULL; + if (page->poison) + list_add(&page->list, &node->sgx_poison_page_list); + else + list_add_tail(&page->list, &node->free_page_list); + page->flags = SGX_EPC_PAGE_IS_FREE; spin_unlock(&node->lock); atomic_long_inc(&sgx_nr_free_pages); @@ -648,17 +667,102 @@ static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size, } section->phys_addr = phys_addr; + xa_store_range(&sgx_epc_address_space, section->phys_addr, + phys_addr + size - 1, section, GFP_KERNEL); for (i = 0; i < nr_pages; i++) { section->pages[i].section = index; section->pages[i].flags = 0; section->pages[i].owner = NULL; + section->pages[i].poison = 0; list_add_tail(§ion->pages[i].list, &sgx_dirty_page_list); } return true; } +bool arch_is_platform_page(u64 paddr) +{ + return !!xa_load(&sgx_epc_address_space, paddr); +} +EXPORT_SYMBOL_GPL(arch_is_platform_page); + +static struct sgx_epc_page *sgx_paddr_to_page(u64 paddr) +{ + struct sgx_epc_section *section; + + section = xa_load(&sgx_epc_address_space, paddr); + if (!section) + return NULL; + + return §ion->pages[PFN_DOWN(paddr - section->phys_addr)]; +} + +/* + * Called in process context to handle a hardware reported + * error in an SGX EPC page. + * If the MF_ACTION_REQUIRED bit is set in flags, then the + * context is the task that consumed the poison data. Otherwise + * this is called from a kernel thread unrelated to the page. + */ +int arch_memory_failure(unsigned long pfn, int flags) +{ + struct sgx_epc_page *page = sgx_paddr_to_page(pfn << PAGE_SHIFT); + struct sgx_epc_section *section; + struct sgx_numa_node *node; + + /* + * mm/memory-failure.c calls this routine for all errors + * where there isn't a "struct page" for the address. But that + * includes other address ranges besides SGX. + */ + if (!page) + return -ENXIO; + + /* + * If poison was consumed synchronously. Send a SIGBUS to + * the task. Hardware has already exited the SGX enclave and + * will not allow re-entry to an enclave that has a memory + * error. The signal may help the task understand why the + * enclave is broken. + */ + if (flags & MF_ACTION_REQUIRED) + force_sig(SIGBUS); + + section = &sgx_epc_sections[page->section]; + node = section->node; + + spin_lock(&node->lock); + + /* Already poisoned? Nothing more to do */ + if (page->poison) + goto out; + + page->poison = 1; + + /* + * If the page is on a free list, move it to the per-node + * poison page list. + */ + if (page->flags & SGX_EPC_PAGE_IS_FREE) { + list_move(&page->list, &node->sgx_poison_page_list); + goto out; + } + + /* + * TBD: Add additional plumbing to enable pre-emptive + * action for asynchronous poison notification. Until + * then just hope that the poison: + * a) is not accessed - sgx_free_epc_page() will deal with it + * when the user gives it back + * b) results in a recoverable machine check rather than + * a fatal one + */ +out: + spin_unlock(&node->lock); + return 0; +} + /** * A section metric is concatenated in a way that @low bits 12-31 define the * bits 12-31 of the metric and @high bits 0-19 define the bits 32-51 of the @@ -670,6 +774,48 @@ static inline u64 __init sgx_calc_section_metric(u64 low, u64 high) ((high & GENMASK_ULL(19, 0)) << 32); } +#ifdef CONFIG_NUMA +static ssize_t sgx_total_bytes_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%lu\n", sgx_numa_nodes[dev->id].size); +} +static DEVICE_ATTR_RO(sgx_total_bytes); + +static umode_t arch_node_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int idx) +{ + /* Make all x86/ attributes invisible when SGX is not initialized: */ + if (nodes_empty(sgx_numa_mask)) + return 0; + + return attr->mode; +} + +static struct attribute *arch_node_dev_attrs[] = { + &dev_attr_sgx_total_bytes.attr, + NULL, +}; + +const struct attribute_group arch_node_dev_group = { + .name = "x86", + .attrs = arch_node_dev_attrs, + .is_visible = arch_node_attr_is_visible, +}; + +static void __init arch_update_sysfs_visibility(int nid) +{ + struct node *node = node_devices[nid]; + int ret; + + ret = sysfs_update_group(&node->dev.kobj, &arch_node_dev_group); + + if (ret) + pr_err("sysfs update failed (%d), files may be invisible", ret); +} +#else /* !CONFIG_NUMA */ +static void __init arch_update_sysfs_visibility(int nid) {} +#endif + static bool __init sgx_page_cache_init(void) { u32 eax, ebx, ecx, edx, type; @@ -713,10 +859,16 @@ static bool __init sgx_page_cache_init(void) if (!node_isset(nid, sgx_numa_mask)) { spin_lock_init(&sgx_numa_nodes[nid].lock); INIT_LIST_HEAD(&sgx_numa_nodes[nid].free_page_list); + INIT_LIST_HEAD(&sgx_numa_nodes[nid].sgx_poison_page_list); node_set(nid, sgx_numa_mask); + sgx_numa_nodes[nid].size = 0; + + /* Make SGX-specific node sysfs files visible: */ + arch_update_sysfs_visibility(nid); } sgx_epc_sections[i].node = &sgx_numa_nodes[nid]; + sgx_numa_nodes[nid].size += size; sgx_nr_epc_sections++; } diff --git a/arch/x86/kernel/cpu/sgx/sgx.h b/arch/x86/kernel/cpu/sgx/sgx.h index 4628acec00..0f17def9fe 100644 --- a/arch/x86/kernel/cpu/sgx/sgx.h +++ b/arch/x86/kernel/cpu/sgx/sgx.h @@ -26,9 +26,13 @@ /* Pages, which are being tracked by the page reclaimer. */ #define SGX_EPC_PAGE_RECLAIMER_TRACKED BIT(0) +/* Pages on free list */ +#define SGX_EPC_PAGE_IS_FREE BIT(1) + struct sgx_epc_page { unsigned int section; - unsigned int flags; + u16 flags; + u16 poison; struct sgx_encl_page *owner; struct list_head list; }; @@ -39,6 +43,8 @@ struct sgx_epc_page { */ struct sgx_numa_node { struct list_head free_page_list; + struct list_head sgx_poison_page_list; + unsigned long size; spinlock_t lock; }; diff --git a/arch/x86/kernel/cpu/sgx/virt.c b/arch/x86/kernel/cpu/sgx/virt.c index 64511c4a52..6a77a14eee 100644 --- a/arch/x86/kernel/cpu/sgx/virt.c +++ b/arch/x86/kernel/cpu/sgx/virt.c @@ -111,10 +111,8 @@ static int sgx_vepc_mmap(struct file *file, struct vm_area_struct *vma) return 0; } -static int sgx_vepc_free_page(struct sgx_epc_page *epc_page) +static int sgx_vepc_remove_page(struct sgx_epc_page *epc_page) { - int ret; - /* * Take a previously guest-owned EPC page and return it to the * general EPC page pool. @@ -124,7 +122,12 @@ static int sgx_vepc_free_page(struct sgx_epc_page *epc_page) * case that a guest properly EREMOVE'd this page, a superfluous * EREMOVE is harmless. */ - ret = __eremove(sgx_get_epc_virt_addr(epc_page)); + return __eremove(sgx_get_epc_virt_addr(epc_page)); +} + +static int sgx_vepc_free_page(struct sgx_epc_page *epc_page) +{ + int ret = sgx_vepc_remove_page(epc_page); if (ret) { /* * Only SGX_CHILD_PRESENT is expected, which is because of @@ -144,10 +147,44 @@ static int sgx_vepc_free_page(struct sgx_epc_page *epc_page) } sgx_free_epc_page(epc_page); - return 0; } +static long sgx_vepc_remove_all(struct sgx_vepc *vepc) +{ + struct sgx_epc_page *entry; + unsigned long index; + long failures = 0; + + xa_for_each(&vepc->page_array, index, entry) { + int ret = sgx_vepc_remove_page(entry); + if (ret) { + if (ret == SGX_CHILD_PRESENT) { + /* The page is a SECS, userspace will retry. */ + failures++; + } else { + /* + * Report errors due to #GP or SGX_ENCLAVE_ACT; do not + * WARN, as userspace can induce said failures by + * calling the ioctl concurrently on multiple vEPCs or + * while one or more CPUs is running the enclave. Only + * a #PF on EREMOVE indicates a kernel/hardware issue. + */ + WARN_ON_ONCE(encls_faulted(ret) && + ENCLS_TRAPNR(ret) != X86_TRAP_GP); + return -EBUSY; + } + } + cond_resched(); + } + + /* + * Return the number of SECS pages that failed to be removed, so + * userspace knows that it has to retry. + */ + return failures; +} + static int sgx_vepc_release(struct inode *inode, struct file *file) { struct sgx_vepc *vepc = file->private_data; @@ -233,9 +270,27 @@ static int sgx_vepc_open(struct inode *inode, struct file *file) return 0; } +static long sgx_vepc_ioctl(struct file *file, + unsigned int cmd, unsigned long arg) +{ + struct sgx_vepc *vepc = file->private_data; + + switch (cmd) { + case SGX_IOC_VEPC_REMOVE_ALL: + if (arg) + return -EINVAL; + return sgx_vepc_remove_all(vepc); + + default: + return -ENOTTY; + } +} + static const struct file_operations sgx_vepc_fops = { .owner = THIS_MODULE, .open = sgx_vepc_open, + .unlocked_ioctl = sgx_vepc_ioctl, + .compat_ioctl = sgx_vepc_ioctl, .release = sgx_vepc_release, .mmap = sgx_vepc_mmap, }; diff --git a/arch/x86/kernel/crash_dump_64.c b/arch/x86/kernel/crash_dump_64.c index 045e82e894..a7f617a398 100644 --- a/arch/x86/kernel/crash_dump_64.c +++ b/arch/x86/kernel/crash_dump_64.c @@ -10,6 +10,7 @@ #include #include #include +#include static ssize_t __copy_oldmem_page(unsigned long pfn, char *buf, size_t csize, unsigned long offset, int userbuf, @@ -73,5 +74,6 @@ ssize_t copy_oldmem_page_encrypted(unsigned long pfn, char *buf, size_t csize, ssize_t elfcorehdr_read(char *buf, size_t count, u64 *ppos) { - return read_from_oldmem(buf, count, ppos, 0, sev_active()); + return read_from_oldmem(buf, count, ppos, 0, + cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)); } diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c index 6a4cb71c24..5cd51f25f4 100644 --- a/arch/x86/kernel/devicetree.c +++ b/arch/x86/kernel/devicetree.c @@ -31,11 +31,6 @@ char __initdata cmd_line[COMMAND_LINE_SIZE]; int __initdata of_ioapic; -void __init early_init_dt_scan_chosen_arch(unsigned long node) -{ - BUG(); -} - void __init early_init_dt_add_memory_arch(u64 base, u64 size) { BUG(); @@ -139,12 +134,11 @@ static void __init dtb_cpu_setup(void) { struct device_node *dn; u32 apic_id, version; - int ret; version = GET_APIC_VERSION(apic_read(APIC_LVR)); for_each_of_cpu_node(dn) { - ret = of_property_read_u32(dn, "reg", &apic_id); - if (ret < 0) { + apic_id = of_get_cpu_hwid(dn, 0); + if (apic_id == ~0U) { pr_warn("%pOF: missing local APIC ID\n", dn); continue; } diff --git a/arch/x86/kernel/doublefault_32.c b/arch/x86/kernel/doublefault_32.c index d1d49e3d53..3b58d87030 100644 --- a/arch/x86/kernel/doublefault_32.c +++ b/arch/x86/kernel/doublefault_32.c @@ -77,9 +77,6 @@ asmlinkage noinstr void __noreturn doublefault_shim(void) * some way to reconstruct CR3. We could make a credible guess based * on cpu_tlbstate, but that would be racy and would not account for * PTI. - * - * Instead, don't bother. We can return through - * rewind_stack_do_exit() instead. */ panic("cannot return from double fault\n"); } diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index ea4fe19218..53de044e56 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -351,7 +351,7 @@ unsigned long oops_begin(void) } NOKPROBE_SYMBOL(oops_begin); -void __noreturn rewind_stack_do_exit(int signr); +void __noreturn rewind_stack_and_make_dead(int signr); void oops_end(unsigned long flags, struct pt_regs *regs, int signr) { @@ -386,7 +386,7 @@ void oops_end(unsigned long flags, struct pt_regs *regs, int signr) * reuse the task stack and that existing poisons are invalid. */ kasan_unpoison_task_stack(current); - rewind_stack_do_exit(signr); + rewind_stack_and_make_dead(signr); } NOKPROBE_SYMBOL(oops_end); diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c index 8690fab95a..dc7da08bc7 100644 --- a/arch/x86/kernel/early-quirks.c +++ b/arch/x86/kernel/early-quirks.c @@ -555,6 +555,7 @@ static const struct pci_device_id intel_early_ids[] __initconst = { INTEL_RKL_IDS(&gen11_early_ops), INTEL_ADLS_IDS(&gen11_early_ops), INTEL_ADLP_IDS(&gen11_early_ops), + INTEL_RPLS_IDS(&gen11_early_ops), }; struct resource intel_graphics_stolen_res __ro_after_init = DEFINE_RES_MEM(0, 0); diff --git a/arch/x86/kernel/fpu/bugs.c b/arch/x86/kernel/fpu/bugs.c index 2954fab15e..794e701512 100644 --- a/arch/x86/kernel/fpu/bugs.c +++ b/arch/x86/kernel/fpu/bugs.c @@ -2,7 +2,7 @@ /* * x86 FPU bug checks: */ -#include +#include /* * Boot time CPU/FPU FDIV bug detection code: diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 7ada7bd03a..8dea01ffc5 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -6,8 +6,9 @@ * General FPU state handling cleanups * Gareth Hughes , May 2000 */ -#include +#include #include +#include #include #include #include @@ -15,15 +16,30 @@ #include #include +#include + +#include "context.h" +#include "internal.h" +#include "legacy.h" +#include "xstate.h" #define CREATE_TRACE_POINTS #include +#ifdef CONFIG_X86_64 +DEFINE_STATIC_KEY_FALSE(__fpu_state_size_dynamic); +DEFINE_PER_CPU(u64, xfd_state); +#endif + +/* The FPU state configuration data for kernel and user space */ +struct fpu_state_config fpu_kernel_cfg __ro_after_init; +struct fpu_state_config fpu_user_cfg __ro_after_init; + /* * Represents the initial FPU state. It's mostly (but not completely) zeroes, * depending on the FPU hardware format: */ -union fpregs_state init_fpstate __ro_after_init; +struct fpstate init_fpstate __ro_after_init; /* * Track whether the kernel is using the FPU state @@ -83,7 +99,20 @@ bool irq_fpu_usable(void) EXPORT_SYMBOL(irq_fpu_usable); /* - * Save the FPU register state in fpu->state. The register state is + * Track AVX512 state use because it is known to slow the max clock + * speed of the core. + */ +static void update_avx_timestamp(struct fpu *fpu) +{ + +#define AVX512_TRACKING_MASK (XFEATURE_MASK_ZMM_Hi256 | XFEATURE_MASK_Hi16_ZMM) + + if (fpu->fpstate->regs.xsave.header.xfeatures & AVX512_TRACKING_MASK) + fpu->avx512_timestamp = jiffies; +} + +/* + * Save the FPU register state in fpu->fpstate->regs. The register state is * preserved. * * Must be called with fpregs_lock() held. @@ -99,19 +128,13 @@ EXPORT_SYMBOL(irq_fpu_usable); void save_fpregs_to_fpstate(struct fpu *fpu) { if (likely(use_xsave())) { - os_xsave(&fpu->state.xsave); - - /* - * AVX512 state is tracked here because its use is - * known to slow the max clock speed of the core. - */ - if (fpu->state.xsave.header.xfeatures & XFEATURE_MASK_AVX512) - fpu->avx512_timestamp = jiffies; + os_xsave(fpu->fpstate); + update_avx_timestamp(fpu); return; } if (likely(use_fxsr())) { - fxsave(&fpu->state.fxsave); + fxsave(&fpu->fpstate->regs.fxsave); return; } @@ -119,12 +142,11 @@ void save_fpregs_to_fpstate(struct fpu *fpu) * Legacy FPU register saving, FNSAVE always clears FPU registers, * so we have to reload them from the memory state. */ - asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->state.fsave)); - frstor(&fpu->state.fsave); + asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->fpstate->regs.fsave)); + frstor(&fpu->fpstate->regs.fsave); } -EXPORT_SYMBOL(save_fpregs_to_fpstate); -void __restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask) +void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask) { /* * AMD K7/K8 and later CPUs up to Zen don't save/restore @@ -141,15 +163,265 @@ void __restore_fpregs_from_fpstate(union fpregs_state *fpstate, u64 mask) } if (use_xsave()) { - os_xrstor(&fpstate->xsave, mask); + /* + * Dynamically enabled features are enabled in XCR0, but + * usage requires also that the corresponding bits in XFD + * are cleared. If the bits are set then using a related + * instruction will raise #NM. This allows to do the + * allocation of the larger FPU buffer lazy from #NM or if + * the task has no permission to kill it which would happen + * via #UD if the feature is disabled in XCR0. + * + * XFD state is following the same life time rules as + * XSTATE and to restore state correctly XFD has to be + * updated before XRSTORS otherwise the component would + * stay in or go into init state even if the bits are set + * in fpstate::regs::xsave::xfeatures. + */ + xfd_update_state(fpstate); + + /* + * Restoring state always needs to modify all features + * which are in @mask even if the current task cannot use + * extended features. + * + * So fpstate->xfeatures cannot be used here, because then + * a feature for which the task has no permission but was + * used by the previous task would not go into init state. + */ + mask = fpu_kernel_cfg.max_features & mask; + + os_xrstor(fpstate, mask); } else { if (use_fxsr()) - fxrstor(&fpstate->fxsave); + fxrstor(&fpstate->regs.fxsave); else - frstor(&fpstate->fsave); + frstor(&fpstate->regs.fsave); } } -EXPORT_SYMBOL_GPL(__restore_fpregs_from_fpstate); + +void fpu_reset_from_exception_fixup(void) +{ + restore_fpregs_from_fpstate(&init_fpstate, XFEATURE_MASK_FPSTATE); +} + +#if IS_ENABLED(CONFIG_KVM) +static void __fpstate_reset(struct fpstate *fpstate, u64 xfd); + +static void fpu_init_guest_permissions(struct fpu_guest *gfpu) +{ + struct fpu_state_perm *fpuperm; + u64 perm; + + if (!IS_ENABLED(CONFIG_X86_64)) + return; + + spin_lock_irq(¤t->sighand->siglock); + fpuperm = ¤t->group_leader->thread.fpu.guest_perm; + perm = fpuperm->__state_perm; + + /* First fpstate allocation locks down permissions. */ + WRITE_ONCE(fpuperm->__state_perm, perm | FPU_GUEST_PERM_LOCKED); + + spin_unlock_irq(¤t->sighand->siglock); + + gfpu->perm = perm & ~FPU_GUEST_PERM_LOCKED; +} + +bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu) +{ + struct fpstate *fpstate; + unsigned int size; + + size = fpu_user_cfg.default_size + ALIGN(offsetof(struct fpstate, regs), 64); + fpstate = vzalloc(size); + if (!fpstate) + return false; + + /* Leave xfd to 0 (the reset value defined by spec) */ + __fpstate_reset(fpstate, 0); + fpstate_init_user(fpstate); + fpstate->is_valloc = true; + fpstate->is_guest = true; + + gfpu->fpstate = fpstate; + gfpu->xfeatures = fpu_user_cfg.default_features; + gfpu->perm = fpu_user_cfg.default_features; + gfpu->uabi_size = fpu_user_cfg.default_size; + fpu_init_guest_permissions(gfpu); + + return true; +} +EXPORT_SYMBOL_GPL(fpu_alloc_guest_fpstate); + +void fpu_free_guest_fpstate(struct fpu_guest *gfpu) +{ + struct fpstate *fps = gfpu->fpstate; + + if (!fps) + return; + + if (WARN_ON_ONCE(!fps->is_valloc || !fps->is_guest || fps->in_use)) + return; + + gfpu->fpstate = NULL; + vfree(fps); +} +EXPORT_SYMBOL_GPL(fpu_free_guest_fpstate); + +/* + * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable + * @guest_fpu: Pointer to the guest FPU container + * @xfeatures: Features requested by guest CPUID + * + * Enable all dynamic xfeatures according to guest perm and requested CPUID. + * + * Return: 0 on success, error code otherwise + */ +int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures) +{ + lockdep_assert_preemption_enabled(); + + /* Nothing to do if all requested features are already enabled. */ + xfeatures &= ~guest_fpu->xfeatures; + if (!xfeatures) + return 0; + + return __xfd_enable_feature(xfeatures, guest_fpu); +} +EXPORT_SYMBOL_GPL(fpu_enable_guest_xfd_features); + +#ifdef CONFIG_X86_64 +void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) +{ + fpregs_lock(); + guest_fpu->fpstate->xfd = xfd; + if (guest_fpu->fpstate->in_use) + xfd_update_state(guest_fpu->fpstate); + fpregs_unlock(); +} +EXPORT_SYMBOL_GPL(fpu_update_guest_xfd); + +/** + * fpu_sync_guest_vmexit_xfd_state - Synchronize XFD MSR and software state + * + * Must be invoked from KVM after a VMEXIT before enabling interrupts when + * XFD write emulation is disabled. This is required because the guest can + * freely modify XFD and the state at VMEXIT is not guaranteed to be the + * same as the state on VMENTER. So software state has to be udpated before + * any operation which depends on it can take place. + * + * Note: It can be invoked unconditionally even when write emulation is + * enabled for the price of a then pointless MSR read. + */ +void fpu_sync_guest_vmexit_xfd_state(void) +{ + struct fpstate *fps = current->thread.fpu.fpstate; + + lockdep_assert_irqs_disabled(); + if (fpu_state_size_dynamic()) { + rdmsrl(MSR_IA32_XFD, fps->xfd); + __this_cpu_write(xfd_state, fps->xfd); + } +} +EXPORT_SYMBOL_GPL(fpu_sync_guest_vmexit_xfd_state); +#endif /* CONFIG_X86_64 */ + +int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest) +{ + struct fpstate *guest_fps = guest_fpu->fpstate; + struct fpu *fpu = ¤t->thread.fpu; + struct fpstate *cur_fps = fpu->fpstate; + + fpregs_lock(); + if (!cur_fps->is_confidential && !test_thread_flag(TIF_NEED_FPU_LOAD)) + save_fpregs_to_fpstate(fpu); + + /* Swap fpstate */ + if (enter_guest) { + fpu->__task_fpstate = cur_fps; + fpu->fpstate = guest_fps; + guest_fps->in_use = true; + } else { + guest_fps->in_use = false; + fpu->fpstate = fpu->__task_fpstate; + fpu->__task_fpstate = NULL; + } + + cur_fps = fpu->fpstate; + + if (!cur_fps->is_confidential) { + /* Includes XFD update */ + restore_fpregs_from_fpstate(cur_fps, XFEATURE_MASK_FPSTATE); + } else { + /* + * XSTATE is restored by firmware from encrypted + * memory. Make sure XFD state is correct while + * running with guest fpstate + */ + xfd_update_state(cur_fps); + } + + fpregs_mark_activate(); + fpregs_unlock(); + return 0; +} +EXPORT_SYMBOL_GPL(fpu_swap_kvm_fpstate); + +void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf, + unsigned int size, u32 pkru) +{ + struct fpstate *kstate = gfpu->fpstate; + union fpregs_state *ustate = buf; + struct membuf mb = { .p = buf, .left = size }; + + if (cpu_feature_enabled(X86_FEATURE_XSAVE)) { + __copy_xstate_to_uabi_buf(mb, kstate, pkru, XSTATE_COPY_XSAVE); + } else { + memcpy(&ustate->fxsave, &kstate->regs.fxsave, + sizeof(ustate->fxsave)); + /* Make it restorable on a XSAVE enabled host */ + ustate->xsave.header.xfeatures = XFEATURE_MASK_FPSSE; + } +} +EXPORT_SYMBOL_GPL(fpu_copy_guest_fpstate_to_uabi); + +int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, + u64 xcr0, u32 *vpkru) +{ + struct fpstate *kstate = gfpu->fpstate; + const union fpregs_state *ustate = buf; + struct pkru_state *xpkru; + int ret; + + if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) { + if (ustate->xsave.header.xfeatures & ~XFEATURE_MASK_FPSSE) + return -EINVAL; + if (ustate->fxsave.mxcsr & ~mxcsr_feature_mask) + return -EINVAL; + memcpy(&kstate->regs.fxsave, &ustate->fxsave, sizeof(ustate->fxsave)); + return 0; + } + + if (ustate->xsave.header.xfeatures & ~xcr0) + return -EINVAL; + + ret = copy_uabi_from_kernel_to_xstate(kstate, ustate); + if (ret) + return ret; + + /* Retrieve PKRU if not in init state */ + if (kstate->regs.xsave.header.xfeatures & XFEATURE_MASK_PKRU) { + xpkru = get_xsave_addr(&kstate->regs.xsave, XFEATURE_PKRU); + *vpkru = xpkru->pkru; + } + + /* Ensure that XCOMP_BV is set up for XSAVES */ + xstate_init_xcomp_bv(&kstate->regs.xsave, kstate->xfeatures); + return 0; +} +EXPORT_SYMBOL_GPL(fpu_copy_uabi_to_guest_fpstate); +#endif /* CONFIG_KVM */ void kernel_fpu_begin_mask(unsigned int kfpu_mask) { @@ -203,52 +475,91 @@ void fpu_sync_fpstate(struct fpu *fpu) fpregs_unlock(); } -static inline void fpstate_init_xstate(struct xregs_state *xsave) +static inline unsigned int init_fpstate_copy_size(void) { - /* - * XRSTORS requires these bits set in xcomp_bv, or it will - * trigger #GP: - */ - xsave->header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT | xfeatures_mask_all; + if (!use_xsave()) + return fpu_kernel_cfg.default_size; + + /* XSAVE(S) just needs the legacy and the xstate header part */ + return sizeof(init_fpstate.regs.xsave); } -static inline void fpstate_init_fxstate(struct fxregs_state *fx) +static inline void fpstate_init_fxstate(struct fpstate *fpstate) { - fx->cwd = 0x37f; - fx->mxcsr = MXCSR_DEFAULT; + fpstate->regs.fxsave.cwd = 0x37f; + fpstate->regs.fxsave.mxcsr = MXCSR_DEFAULT; } /* * Legacy x87 fpstate state init: */ -static inline void fpstate_init_fstate(struct fregs_state *fp) +static inline void fpstate_init_fstate(struct fpstate *fpstate) { - fp->cwd = 0xffff037fu; - fp->swd = 0xffff0000u; - fp->twd = 0xffffffffu; - fp->fos = 0xffff0000u; + fpstate->regs.fsave.cwd = 0xffff037fu; + fpstate->regs.fsave.swd = 0xffff0000u; + fpstate->regs.fsave.twd = 0xffffffffu; + fpstate->regs.fsave.fos = 0xffff0000u; } -void fpstate_init(union fpregs_state *state) +/* + * Used in two places: + * 1) Early boot to setup init_fpstate for non XSAVE systems + * 2) fpu_init_fpstate_user() which is invoked from KVM + */ +void fpstate_init_user(struct fpstate *fpstate) { - if (!static_cpu_has(X86_FEATURE_FPU)) { - fpstate_init_soft(&state->soft); + if (!cpu_feature_enabled(X86_FEATURE_FPU)) { + fpstate_init_soft(&fpstate->regs.soft); return; } - memset(state, 0, fpu_kernel_xstate_size); + xstate_init_xcomp_bv(&fpstate->regs.xsave, fpstate->xfeatures); - if (static_cpu_has(X86_FEATURE_XSAVES)) - fpstate_init_xstate(&state->xsave); - if (static_cpu_has(X86_FEATURE_FXSR)) - fpstate_init_fxstate(&state->fxsave); + if (cpu_feature_enabled(X86_FEATURE_FXSR)) + fpstate_init_fxstate(fpstate); else - fpstate_init_fstate(&state->fsave); + fpstate_init_fstate(fpstate); +} + +static void __fpstate_reset(struct fpstate *fpstate, u64 xfd) +{ + /* Initialize sizes and feature masks */ + fpstate->size = fpu_kernel_cfg.default_size; + fpstate->user_size = fpu_user_cfg.default_size; + fpstate->xfeatures = fpu_kernel_cfg.default_features; + fpstate->user_xfeatures = fpu_user_cfg.default_features; + fpstate->xfd = xfd; +} + +void fpstate_reset(struct fpu *fpu) +{ + /* Set the fpstate pointer to the default fpstate */ + fpu->fpstate = &fpu->__fpstate; + __fpstate_reset(fpu->fpstate, init_fpstate.xfd); + + /* Initialize the permission related info in fpu */ + fpu->perm.__state_perm = fpu_kernel_cfg.default_features; + fpu->perm.__state_size = fpu_kernel_cfg.default_size; + fpu->perm.__user_state_size = fpu_user_cfg.default_size; + /* Same defaults for guests */ + fpu->guest_perm = fpu->perm; +} + +static inline void fpu_inherit_perms(struct fpu *dst_fpu) +{ + if (fpu_state_size_dynamic()) { + struct fpu *src_fpu = ¤t->group_leader->thread.fpu; + + spin_lock_irq(¤t->sighand->siglock); + /* Fork also inherits the permissions of the parent */ + dst_fpu->perm = src_fpu->perm; + dst_fpu->guest_perm = src_fpu->guest_perm; + spin_unlock_irq(¤t->sighand->siglock); + } } -EXPORT_SYMBOL_GPL(fpstate_init); /* Clone current's FPU state on fork */ -int fpu_clone(struct task_struct *dst) +int fpu_clone(struct task_struct *dst, unsigned long clone_flags) { struct fpu *src_fpu = ¤t->thread.fpu; struct fpu *dst_fpu = &dst->thread.fpu; @@ -256,36 +567,67 @@ int fpu_clone(struct task_struct *dst) /* The new task's FPU state cannot be valid in the hardware. */ dst_fpu->last_cpu = -1; + fpstate_reset(dst_fpu); + if (!cpu_feature_enabled(X86_FEATURE_FPU)) return 0; /* - * Don't let 'init optimized' areas of the XSAVE area - * leak into the child task: + * Enforce reload for user space tasks and prevent kernel threads + * from trying to save the FPU registers on context switch. */ - memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size); + set_tsk_thread_flag(dst, TIF_NEED_FPU_LOAD); /* - * If the FPU registers are not owned by current just memcpy() the - * state. Otherwise save the FPU registers directly into the - * child's FPU context, without any memory-to-memory copying. + * No FPU state inheritance for kernel threads and IO + * worker threads. + */ + if (dst->flags & (PF_KTHREAD | PF_IO_WORKER)) { + /* Clear out the minimal state */ + memcpy(&dst_fpu->fpstate->regs, &init_fpstate.regs, + init_fpstate_copy_size()); + return 0; + } + + /* + * If a new feature is added, ensure all dynamic features are + * caller-saved from here! + */ + BUILD_BUG_ON(XFEATURE_MASK_USER_DYNAMIC != XFEATURE_MASK_XTILE_DATA); + + /* + * Save the default portion of the current FPU state into the + * clone. Assume all dynamic features to be defined as caller- + * saved, which enables skipping both the expansion of fpstate + * and the copying of any dynamic state. + * + * Do not use memcpy() when TIF_NEED_FPU_LOAD is set because + * copying is not valid when current uses non-default states. */ fpregs_lock(); if (test_thread_flag(TIF_NEED_FPU_LOAD)) - memcpy(&dst_fpu->state, &src_fpu->state, fpu_kernel_xstate_size); - - else - save_fpregs_to_fpstate(dst_fpu); + fpregs_restore_userregs(); + save_fpregs_to_fpstate(dst_fpu); + if (!(clone_flags & CLONE_THREAD)) + fpu_inherit_perms(dst_fpu); fpregs_unlock(); - set_tsk_thread_flag(dst, TIF_NEED_FPU_LOAD); - trace_x86_fpu_copy_src(src_fpu); trace_x86_fpu_copy_dst(dst_fpu); return 0; } +/* + * Whitelist the FPU register state embedded into task_struct for hardened + * usercopy. + */ +void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size) +{ + *offset = offsetof(struct thread_struct, fpu.__fpstate.regs); + *size = fpu_kernel_cfg.default_size; +} + /* * Drops current FPU state: deactivates the fpregs and * the fpstate. NOTE: it still leaves previous contents @@ -319,28 +661,19 @@ void fpu__drop(struct fpu *fpu) static inline void restore_fpregs_from_init_fpstate(u64 features_mask) { if (use_xsave()) - os_xrstor(&init_fpstate.xsave, features_mask); + os_xrstor(&init_fpstate, features_mask); else if (use_fxsr()) - fxrstor(&init_fpstate.fxsave); + fxrstor(&init_fpstate.regs.fxsave); else - frstor(&init_fpstate.fsave); + frstor(&init_fpstate.regs.fsave); pkru_write_default(); } -static inline unsigned int init_fpstate_copy_size(void) -{ - if (!use_xsave()) - return fpu_kernel_xstate_size; - - /* XSAVE(S) just needs the legacy and the xstate header part */ - return sizeof(init_fpstate.xsave); -} - /* * Reset current->fpu memory state to the init values. */ -static void fpu_reset_fpstate(void) +static void fpu_reset_fpregs(void) { struct fpu *fpu = ¤t->thread.fpu; @@ -359,7 +692,7 @@ static void fpu_reset_fpstate(void) * user space as PKRU is eagerly written in switch_to() and * flush_thread(). */ - memcpy(&fpu->state, &init_fpstate, init_fpstate_copy_size()); + memcpy(&fpu->fpstate->regs, &init_fpstate.regs, init_fpstate_copy_size()); set_thread_flag(TIF_NEED_FPU_LOAD); fpregs_unlock(); } @@ -375,7 +708,7 @@ void fpu__clear_user_states(struct fpu *fpu) fpregs_lock(); if (!cpu_feature_enabled(X86_FEATURE_FPU)) { - fpu_reset_fpstate(); + fpu_reset_fpregs(); fpregs_unlock(); return; } @@ -385,12 +718,11 @@ void fpu__clear_user_states(struct fpu *fpu) * corresponding registers. */ if (xfeatures_mask_supervisor() && - !fpregs_state_valid(fpu, smp_processor_id())) { - os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor()); - } + !fpregs_state_valid(fpu, smp_processor_id())) + os_xrstor_supervisor(fpu->fpstate); /* Reset user states in registers. */ - restore_fpregs_from_init_fpstate(xfeatures_mask_restore_user()); + restore_fpregs_from_init_fpstate(XFEATURE_MASK_USER_RESTORE); /* * Now all FPU registers have their desired values. Inform the FPU @@ -405,7 +737,8 @@ void fpu__clear_user_states(struct fpu *fpu) void fpu_flush_thread(void) { - fpu_reset_fpstate(); + fpstate_reset(¤t->thread.fpu); + fpu_reset_fpregs(); } /* * Load FPU context before returning to userspace. @@ -445,7 +778,6 @@ void fpregs_mark_activate(void) fpu->last_cpu = smp_processor_id(); clear_thread_flag(TIF_NEED_FPU_LOAD); } -EXPORT_SYMBOL_GPL(fpregs_mark_activate); /* * x87 math exception handling: @@ -468,11 +800,11 @@ int fpu__exception_code(struct fpu *fpu, int trap_nr) * fully reproduce the context of the exception. */ if (boot_cpu_has(X86_FEATURE_FXSR)) { - cwd = fpu->state.fxsave.cwd; - swd = fpu->state.fxsave.swd; + cwd = fpu->fpstate->regs.fxsave.cwd; + swd = fpu->fpstate->regs.fxsave.swd; } else { - cwd = (unsigned short)fpu->state.fsave.cwd; - swd = (unsigned short)fpu->state.fsave.swd; + cwd = (unsigned short)fpu->fpstate->regs.fsave.cwd; + swd = (unsigned short)fpu->fpstate->regs.fsave.swd; } err = swd & ~cwd; @@ -486,7 +818,7 @@ int fpu__exception_code(struct fpu *fpu, int trap_nr) unsigned short mxcsr = MXCSR_DEFAULT; if (boot_cpu_has(X86_FEATURE_XMM)) - mxcsr = fpu->state.fxsave.mxcsr; + mxcsr = fpu->fpstate->regs.fxsave.mxcsr; err = ~(mxcsr >> 7) & mxcsr; } diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index 64e29927cc..621f4b6cac 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -2,7 +2,7 @@ /* * x86 FPU boot time init code: */ -#include +#include #include #include @@ -10,6 +10,10 @@ #include #include +#include "internal.h" +#include "legacy.h" +#include "xstate.h" + /* * Initialize the registers found in all CPUs, CR0 and CR4: */ @@ -34,7 +38,7 @@ static void fpu__init_cpu_generic(void) /* Flush out any pending x87 state: */ #ifdef CONFIG_MATH_EMULATION if (!boot_cpu_has(X86_FEATURE_FPU)) - fpstate_init_soft(¤t->thread.fpu.state.soft); + fpstate_init_soft(¤t->thread.fpu.fpstate->regs.soft); else #endif asm volatile ("fninit"); @@ -121,23 +125,14 @@ static void __init fpu__init_system_mxcsr(void) static void __init fpu__init_system_generic(void) { /* - * Set up the legacy init FPU context. (xstate init might overwrite this - * with a more modern format, if the CPU supports it.) + * Set up the legacy init FPU context. Will be updated when the + * CPU supports XSAVE[S]. */ - fpstate_init(&init_fpstate); + fpstate_init_user(&init_fpstate); fpu__init_system_mxcsr(); } -/* - * Size of the FPU context state. All tasks in the system use the - * same context size, regardless of what portion they use. - * This is inherent to the XSAVE architecture which puts all state - * components into a single, continuous memory block: - */ -unsigned int fpu_kernel_xstate_size __ro_after_init; -EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size); - /* Get alignment of the TYPE. */ #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test) @@ -162,13 +157,13 @@ static void __init fpu__init_task_struct_size(void) * Subtract off the static size of the register state. * It potentially has a bunch of padding. */ - task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state); + task_size -= sizeof(current->thread.fpu.__fpstate.regs); /* * Add back the dynamically-calculated register state * size. */ - task_size += fpu_kernel_xstate_size; + task_size += fpu_kernel_cfg.default_size; /* * We dynamically size 'struct fpu', so we require that @@ -177,7 +172,7 @@ static void __init fpu__init_task_struct_size(void) * you hit a compile error here, check the structure to * see if something got added to the end. */ - CHECK_MEMBER_AT_END_OF(struct fpu, state); + CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate); CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu); CHECK_MEMBER_AT_END_OF(struct task_struct, thread); @@ -192,37 +187,34 @@ static void __init fpu__init_task_struct_size(void) */ static void __init fpu__init_system_xstate_size_legacy(void) { - static int on_boot_cpu __initdata = 1; - - WARN_ON_FPU(!on_boot_cpu); - on_boot_cpu = 0; + unsigned int size; /* - * Note that xstate sizes might be overwritten later during - * fpu__init_system_xstate(). + * Note that the size configuration might be overwritten later + * during fpu__init_system_xstate(). */ - - if (!boot_cpu_has(X86_FEATURE_FPU)) { - fpu_kernel_xstate_size = sizeof(struct swregs_state); + if (!cpu_feature_enabled(X86_FEATURE_FPU)) { + size = sizeof(struct swregs_state); + } else if (cpu_feature_enabled(X86_FEATURE_FXSR)) { + size = sizeof(struct fxregs_state); + fpu_user_cfg.legacy_features = XFEATURE_MASK_FPSSE; } else { - if (boot_cpu_has(X86_FEATURE_FXSR)) - fpu_kernel_xstate_size = - sizeof(struct fxregs_state); - else - fpu_kernel_xstate_size = - sizeof(struct fregs_state); + size = sizeof(struct fregs_state); + fpu_user_cfg.legacy_features = XFEATURE_MASK_FP; } - fpu_user_xstate_size = fpu_kernel_xstate_size; + fpu_kernel_cfg.max_size = size; + fpu_kernel_cfg.default_size = size; + fpu_user_cfg.max_size = size; + fpu_user_cfg.default_size = size; + fpstate_reset(¤t->thread.fpu); } -/* Legacy code to initialize eager fpu mode. */ -static void __init fpu__init_system_ctx_switch(void) +static void __init fpu__init_init_fpstate(void) { - static bool on_boot_cpu __initdata = 1; - - WARN_ON_FPU(!on_boot_cpu); - on_boot_cpu = 0; + /* Bring init_fpstate size and features up to date */ + init_fpstate.size = fpu_kernel_cfg.max_size; + init_fpstate.xfeatures = fpu_kernel_cfg.max_features; } /* @@ -231,6 +223,7 @@ static void __init fpu__init_system_ctx_switch(void) */ void __init fpu__init_system(struct cpuinfo_x86 *c) { + fpstate_reset(¤t->thread.fpu); fpu__init_system_early_generic(c); /* @@ -241,8 +234,7 @@ void __init fpu__init_system(struct cpuinfo_x86 *c) fpu__init_system_generic(); fpu__init_system_xstate_size_legacy(); - fpu__init_system_xstate(); + fpu__init_system_xstate(fpu_kernel_cfg.max_size); fpu__init_task_struct_size(); - - fpu__init_system_ctx_switch(); + fpu__init_init_fpstate(); } diff --git a/arch/x86/kernel/fpu/regset.c b/arch/x86/kernel/fpu/regset.c index 125cbbe10f..75ffaef8c2 100644 --- a/arch/x86/kernel/fpu/regset.c +++ b/arch/x86/kernel/fpu/regset.c @@ -5,10 +5,14 @@ #include #include -#include +#include #include #include -#include + +#include "context.h" +#include "internal.h" +#include "legacy.h" +#include "xstate.h" /* * The xstateregs_active() routine is the same as the regset_fpregs_active() routine, @@ -74,8 +78,8 @@ int xfpregs_get(struct task_struct *target, const struct user_regset *regset, sync_fpstate(fpu); if (!use_xsave()) { - return membuf_write(&to, &fpu->state.fxsave, - sizeof(fpu->state.fxsave)); + return membuf_write(&to, &fpu->fpstate->regs.fxsave, + sizeof(fpu->fpstate->regs.fxsave)); } copy_xstate_to_uabi_buf(to, target, XSTATE_COPY_FX); @@ -108,16 +112,16 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset, fpu_force_restore(fpu); /* Copy the state */ - memcpy(&fpu->state.fxsave, &newstate, sizeof(newstate)); + memcpy(&fpu->fpstate->regs.fxsave, &newstate, sizeof(newstate)); /* Clear xmm8..15 for 32-bit callers */ - BUILD_BUG_ON(sizeof(fpu->state.fxsave.xmm_space) != 16 * 16); + BUILD_BUG_ON(sizeof(fpu->__fpstate.regs.fxsave.xmm_space) != 16 * 16); if (in_ia32_syscall()) - memset(&fpu->state.fxsave.xmm_space[8*4], 0, 8 * 16); + memset(&fpu->fpstate->regs.fxsave.xmm_space[8*4], 0, 8 * 16); /* Mark FP and SSE as in use when XSAVE is enabled */ if (use_xsave()) - fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE; + fpu->fpstate->regs.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE; return 0; } @@ -148,7 +152,7 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset, /* * A whole standard-format XSAVE buffer is needed: */ - if (pos != 0 || count != fpu_user_xstate_size) + if (pos != 0 || count != fpu_user_cfg.max_size) return -EFAULT; if (!kbuf) { @@ -163,7 +167,7 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset, } fpu_force_restore(fpu); - ret = copy_uabi_from_kernel_to_xstate(&fpu->state.xsave, kbuf ?: tmpbuf); + ret = copy_uabi_from_kernel_to_xstate(fpu->fpstate, kbuf ?: tmpbuf); out: vfree(tmpbuf); @@ -282,7 +286,7 @@ static void __convert_from_fxsr(struct user_i387_ia32_struct *env, void convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk) { - __convert_from_fxsr(env, tsk, &tsk->thread.fpu.state.fxsave); + __convert_from_fxsr(env, tsk, &tsk->thread.fpu.fpstate->regs.fxsave); } void convert_to_fxsr(struct fxregs_state *fxsave, @@ -325,7 +329,7 @@ int fpregs_get(struct task_struct *target, const struct user_regset *regset, return fpregs_soft_get(target, regset, to); if (!cpu_feature_enabled(X86_FEATURE_FXSR)) { - return membuf_write(&to, &fpu->state.fsave, + return membuf_write(&to, &fpu->fpstate->regs.fsave, sizeof(struct fregs_state)); } @@ -336,7 +340,7 @@ int fpregs_get(struct task_struct *target, const struct user_regset *regset, copy_xstate_to_uabi_buf(mb, target, XSTATE_COPY_FP); fx = &fxsave; } else { - fx = &fpu->state.fxsave; + fx = &fpu->fpstate->regs.fxsave; } __convert_from_fxsr(&env, target, fx); @@ -365,16 +369,16 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset, fpu_force_restore(fpu); if (cpu_feature_enabled(X86_FEATURE_FXSR)) - convert_to_fxsr(&fpu->state.fxsave, &env); + convert_to_fxsr(&fpu->fpstate->regs.fxsave, &env); else - memcpy(&fpu->state.fsave, &env, sizeof(env)); + memcpy(&fpu->fpstate->regs.fsave, &env, sizeof(env)); /* * Update the header bit in the xsave header, indicating the * presence of FP. */ if (cpu_feature_enabled(X86_FEATURE_XSAVE)) - fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FP; + fpu->fpstate->regs.xsave.header.xfeatures |= XFEATURE_MASK_FP; return 0; } diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c index 831b25c5e7..91d4b6de58 100644 --- a/arch/x86/kernel/fpu/signal.c +++ b/arch/x86/kernel/fpu/signal.c @@ -7,23 +7,25 @@ #include #include -#include #include #include #include #include +#include #include -static struct _fpx_sw_bytes fx_sw_reserved __ro_after_init; -static struct _fpx_sw_bytes fx_sw_reserved_ia32 __ro_after_init; +#include "context.h" +#include "internal.h" +#include "legacy.h" +#include "xstate.h" /* * Check for the presence of extended state information in the * user fpstate pointer in the sigcontext. */ -static inline int check_xstate_in_sigframe(struct fxregs_state __user *fxbuf, - struct _fpx_sw_bytes *fx_sw) +static inline bool check_xstate_in_sigframe(struct fxregs_state __user *fxbuf, + struct _fpx_sw_bytes *fx_sw) { int min_xstate_size = sizeof(struct fxregs_state) + sizeof(struct xstate_header); @@ -31,12 +33,12 @@ static inline int check_xstate_in_sigframe(struct fxregs_state __user *fxbuf, unsigned int magic2; if (__copy_from_user(fx_sw, &fxbuf->sw_reserved[0], sizeof(*fx_sw))) - return -EFAULT; + return false; /* Check for the first magic field and other error scenarios. */ if (fx_sw->magic1 != FP_XSTATE_MAGIC1 || fx_sw->xstate_size < min_xstate_size || - fx_sw->xstate_size > fpu_user_xstate_size || + fx_sw->xstate_size > current->thread.fpu.fpstate->user_size || fx_sw->xstate_size > fx_sw->extended_size) goto setfx; @@ -47,10 +49,10 @@ static inline int check_xstate_in_sigframe(struct fxregs_state __user *fxbuf, * in the memory layout. */ if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size))) - return -EFAULT; + return false; if (likely(magic2 == FP_XSTATE_MAGIC2)) - return 0; + return true; setfx: trace_x86_fpu_xstate_check_failed(¤t->thread.fpu); @@ -58,22 +60,22 @@ static inline int check_xstate_in_sigframe(struct fxregs_state __user *fxbuf, fx_sw->magic1 = 0; fx_sw->xstate_size = sizeof(struct fxregs_state); fx_sw->xfeatures = XFEATURE_MASK_FPSSE; - return 0; + return true; } /* * Signal frame handlers. */ -static inline int save_fsave_header(struct task_struct *tsk, void __user *buf) +static inline bool save_fsave_header(struct task_struct *tsk, void __user *buf) { if (use_fxsr()) { - struct xregs_state *xsave = &tsk->thread.fpu.state.xsave; + struct xregs_state *xsave = &tsk->thread.fpu.fpstate->regs.xsave; struct user_i387_ia32_struct env; struct _fpstate_32 __user *fp = buf; fpregs_lock(); if (!test_thread_flag(TIF_NEED_FPU_LOAD)) - fxsave(&tsk->thread.fpu.state.fxsave); + fxsave(&tsk->thread.fpu.fpstate->regs.fxsave); fpregs_unlock(); convert_from_fxsr(&env, tsk); @@ -81,33 +83,54 @@ static inline int save_fsave_header(struct task_struct *tsk, void __user *buf) if (__copy_to_user(buf, &env, sizeof(env)) || __put_user(xsave->i387.swd, &fp->status) || __put_user(X86_FXSR_MAGIC, &fp->magic)) - return -1; + return false; } else { struct fregs_state __user *fp = buf; u32 swd; + if (__get_user(swd, &fp->swd) || __put_user(swd, &fp->status)) - return -1; + return false; } - return 0; + return true; } -static inline int save_xstate_epilog(void __user *buf, int ia32_frame) +/* + * Prepare the SW reserved portion of the fxsave memory layout, indicating + * the presence of the extended state information in the memory layout + * pointed to by the fpstate pointer in the sigcontext. + * This is saved when ever the FP and extended state context is + * saved on the user stack during the signal handler delivery to the user. + */ +static inline void save_sw_bytes(struct _fpx_sw_bytes *sw_bytes, bool ia32_frame, + struct fpstate *fpstate) +{ + sw_bytes->magic1 = FP_XSTATE_MAGIC1; + sw_bytes->extended_size = fpstate->user_size + FP_XSTATE_MAGIC2_SIZE; + sw_bytes->xfeatures = fpstate->user_xfeatures; + sw_bytes->xstate_size = fpstate->user_size; + + if (ia32_frame) + sw_bytes->extended_size += sizeof(struct fregs_state); +} + +static inline bool save_xstate_epilog(void __user *buf, int ia32_frame, + struct fpstate *fpstate) { struct xregs_state __user *x = buf; - struct _fpx_sw_bytes *sw_bytes; + struct _fpx_sw_bytes sw_bytes = {}; u32 xfeatures; int err; /* Setup the bytes not touched by the [f]xsave and reserved for SW. */ - sw_bytes = ia32_frame ? &fx_sw_reserved_ia32 : &fx_sw_reserved; - err = __copy_to_user(&x->i387.sw_reserved, sw_bytes, sizeof(*sw_bytes)); + save_sw_bytes(&sw_bytes, ia32_frame, fpstate); + err = __copy_to_user(&x->i387.sw_reserved, &sw_bytes, sizeof(sw_bytes)); if (!use_xsave()) - return err; + return !err; err |= __put_user(FP_XSTATE_MAGIC2, - (__u32 __user *)(buf + fpu_user_xstate_size)); + (__u32 __user *)(buf + fpstate->user_size)); /* * Read the xfeatures which we copied (directly from the cpu or @@ -130,23 +153,17 @@ static inline int save_xstate_epilog(void __user *buf, int ia32_frame) err |= __put_user(xfeatures, (__u32 __user *)&x->header.xfeatures); - return err; + return !err; } static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf) { - int err; - if (use_xsave()) - err = xsave_to_user_sigframe(buf); - else if (use_fxsr()) - err = fxsave_to_user_sigframe((struct fxregs_state __user *) buf); + return xsave_to_user_sigframe(buf); + if (use_fxsr()) + return fxsave_to_user_sigframe((struct fxregs_state __user *) buf); else - err = fnsave_to_user_sigframe((struct fregs_state __user *) buf); - - if (unlikely(err) && __clear_user(buf, fpu_user_xstate_size)) - err = -EFAULT; - return err; + return fnsave_to_user_sigframe((struct fregs_state __user *) buf); } /* @@ -159,10 +176,8 @@ static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf) * buf == buf_fx for 64-bit frames and 32-bit fsave frame. * buf != buf_fx for 32-bit frames with fxstate. * - * Try to save it directly to the user frame with disabled page fault handler. - * If this fails then do the slow path where the FPU state is first saved to - * task's fpu->state and then copy it to the user frame pointed to by the - * aligned pointer 'buf_fx'. + * Save it directly to the user frame with disabled page fault handler. If + * that faults, try to clear the frame which handles the page fault. * * If this is a 32-bit frame with fxstate, put a fsave header before * the aligned state at 'buf_fx'. @@ -170,10 +185,11 @@ static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf) * For [f]xsave state, update the SW reserved fields in the [f]xsave frame * indicating the absence/presence of the extended state to the user. */ -int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size) +bool copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size) { struct task_struct *tsk = current; - int ia32_fxstate = (buf != buf_fx); + struct fpstate *fpstate = tsk->thread.fpu.fpstate; + bool ia32_fxstate = (buf != buf_fx); int ret; ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) || @@ -181,13 +197,25 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size) if (!static_cpu_has(X86_FEATURE_FPU)) { struct user_i387_ia32_struct fp; + fpregs_soft_get(current, NULL, (struct membuf){.p = &fp, .left = sizeof(fp)}); - return copy_to_user(buf, &fp, sizeof(fp)) ? -EFAULT : 0; + return !copy_to_user(buf, &fp, sizeof(fp)); } if (!access_ok(buf, size)) - return -EACCES; + return false; + + if (use_xsave()) { + struct xregs_state __user *xbuf = buf_fx; + + /* + * Clear the xsave header first, so that reserved fields are + * initialized to zero. + */ + if (__clear_user(&xbuf->header, sizeof(xbuf->header))) + return false; + } retry: /* * Load the FPU registers if they are not valid for the current task. @@ -205,26 +233,26 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size) fpregs_unlock(); if (ret) { - if (!fault_in_pages_writeable(buf_fx, fpu_user_xstate_size)) + if (!__clear_user(buf_fx, fpstate->user_size)) goto retry; - return -EFAULT; + return false; } /* Save the fsave header for the 32-bit frames. */ - if ((ia32_fxstate || !use_fxsr()) && save_fsave_header(tsk, buf)) - return -1; + if ((ia32_fxstate || !use_fxsr()) && !save_fsave_header(tsk, buf)) + return false; - if (use_fxsr() && save_xstate_epilog(buf_fx, ia32_fxstate)) - return -1; + if (use_fxsr() && !save_xstate_epilog(buf_fx, ia32_fxstate, fpstate)) + return false; - return 0; + return true; } -static int __restore_fpregs_from_user(void __user *buf, u64 xrestore, - bool fx_only) +static int __restore_fpregs_from_user(void __user *buf, u64 ufeatures, + u64 xrestore, bool fx_only) { if (use_xsave()) { - u64 init_bv = xfeatures_mask_uabi() & ~xrestore; + u64 init_bv = ufeatures & ~xrestore; int ret; if (likely(!fx_only)) @@ -233,7 +261,7 @@ static int __restore_fpregs_from_user(void __user *buf, u64 xrestore, ret = fxrstor_from_user_sigframe(buf); if (!ret && unlikely(init_bv)) - os_xrstor(&init_fpstate.xsave, init_bv); + os_xrstor(&init_fpstate, init_bv); return ret; } else if (use_fxsr()) { return fxrstor_from_user_sigframe(buf); @@ -246,16 +274,19 @@ static int __restore_fpregs_from_user(void __user *buf, u64 xrestore, * Attempt to restore the FPU registers directly from user memory. * Pagefaults are handled and any errors returned are fatal. */ -static int restore_fpregs_from_user(void __user *buf, u64 xrestore, - bool fx_only, unsigned int size) +static bool restore_fpregs_from_user(void __user *buf, u64 xrestore, + bool fx_only, unsigned int size) { struct fpu *fpu = ¤t->thread.fpu; int ret; retry: fpregs_lock(); + /* Ensure that XFD is up to date */ + xfd_update_state(fpu->fpstate); pagefault_disable(); - ret = __restore_fpregs_from_user(buf, xrestore, fx_only); + ret = __restore_fpregs_from_user(buf, fpu->fpstate->user_xfeatures, + xrestore, fx_only); pagefault_enable(); if (unlikely(ret)) { @@ -275,13 +306,12 @@ static int restore_fpregs_from_user(void __user *buf, u64 xrestore, fpregs_unlock(); /* Try to handle #PF, but anything else is fatal. */ - if (ret != -EFAULT) - return -EINVAL; + if (ret != X86_TRAP_PF) + return false; - ret = fault_in_pages_readable(buf, size); - if (!ret) + if (!fault_in_readable(buf, size)) goto retry; - return ret; + return false; } /* @@ -294,45 +324,40 @@ static int restore_fpregs_from_user(void __user *buf, u64 xrestore, * been restored from a user buffer directly. */ if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor()) - os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor()); + os_xrstor_supervisor(fpu->fpstate); fpregs_mark_activate(); fpregs_unlock(); - return 0; + return true; } -static int __fpu_restore_sig(void __user *buf, void __user *buf_fx, - bool ia32_fxstate) +static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx, + bool ia32_fxstate) { - int state_size = fpu_kernel_xstate_size; struct task_struct *tsk = current; struct fpu *fpu = &tsk->thread.fpu; struct user_i387_ia32_struct env; + bool success, fx_only = false; + union fpregs_state *fpregs; + unsigned int state_size; u64 user_xfeatures = 0; - bool fx_only = false; - int ret; if (use_xsave()) { struct _fpx_sw_bytes fx_sw_user; - ret = check_xstate_in_sigframe(buf_fx, &fx_sw_user); - if (unlikely(ret)) - return ret; + if (!check_xstate_in_sigframe(buf_fx, &fx_sw_user)) + return false; fx_only = !fx_sw_user.magic1; state_size = fx_sw_user.xstate_size; user_xfeatures = fx_sw_user.xfeatures; } else { user_xfeatures = XFEATURE_MASK_FPSSE; + state_size = fpu->fpstate->user_size; } if (likely(!ia32_fxstate)) { - /* - * Attempt to restore the FPU registers directly from user - * memory. For that to succeed, the user access cannot cause page - * faults. If it does, fall back to the slow path below, going - * through the kernel buffer with the enabled pagefault handler. - */ + /* Restore the FPU registers directly from user memory. */ return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only, state_size); } @@ -342,9 +367,8 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx, * to be ignored for histerical raisins. The legacy state is folded * in once the larger state has been copied. */ - ret = __copy_from_user(&env, buf, sizeof(env)); - if (ret) - return ret; + if (__copy_from_user(&env, buf, sizeof(env))) + return false; /* * By setting TIF_NEED_FPU_LOAD it is ensured that our xstate is @@ -363,38 +387,38 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx, * the right place in memory. It's ia32 mode. Shrug. */ if (xfeatures_mask_supervisor()) - os_xsave(&fpu->state.xsave); + os_xsave(fpu->fpstate); set_thread_flag(TIF_NEED_FPU_LOAD); } __fpu_invalidate_fpregs_state(fpu); __cpu_invalidate_fpregs_state(); fpregs_unlock(); + fpregs = &fpu->fpstate->regs; if (use_xsave() && !fx_only) { - ret = copy_sigframe_from_user_to_xstate(&fpu->state.xsave, buf_fx); - if (ret) - return ret; + if (copy_sigframe_from_user_to_xstate(fpu->fpstate, buf_fx)) + return false; } else { - if (__copy_from_user(&fpu->state.fxsave, buf_fx, - sizeof(fpu->state.fxsave))) - return -EFAULT; + if (__copy_from_user(&fpregs->fxsave, buf_fx, + sizeof(fpregs->fxsave))) + return false; if (IS_ENABLED(CONFIG_X86_64)) { /* Reject invalid MXCSR values. */ - if (fpu->state.fxsave.mxcsr & ~mxcsr_feature_mask) - return -EINVAL; + if (fpregs->fxsave.mxcsr & ~mxcsr_feature_mask) + return false; } else { /* Mask invalid bits out for historical reasons (broken hardware). */ - fpu->state.fxsave.mxcsr &= mxcsr_feature_mask; + fpregs->fxsave.mxcsr &= mxcsr_feature_mask; } /* Enforce XFEATURE_MASK_FPSSE when XSAVE is enabled */ if (use_xsave()) - fpu->state.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE; + fpregs->xsave.header.xfeatures |= XFEATURE_MASK_FPSSE; } /* Fold the legacy FP storage */ - convert_to_fxsr(&fpu->state.fxsave, &env); + convert_to_fxsr(&fpregs->fxsave, &env); fpregs_lock(); if (use_xsave()) { @@ -409,40 +433,45 @@ static int __fpu_restore_sig(void __user *buf, void __user *buf_fx, */ u64 mask = user_xfeatures | xfeatures_mask_supervisor(); - fpu->state.xsave.header.xfeatures &= mask; - ret = os_xrstor_safe(&fpu->state.xsave, xfeatures_mask_all); + fpregs->xsave.header.xfeatures &= mask; + success = !os_xrstor_safe(fpu->fpstate, + fpu_kernel_cfg.max_features); } else { - ret = fxrstor_safe(&fpu->state.fxsave); + success = !fxrstor_safe(&fpregs->fxsave); } - if (likely(!ret)) + if (likely(success)) fpregs_mark_activate(); fpregs_unlock(); - return ret; + return success; } -static inline int xstate_sigframe_size(void) + +static inline unsigned int xstate_sigframe_size(struct fpstate *fpstate) { - return use_xsave() ? fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE : - fpu_user_xstate_size; + unsigned int size = fpstate->user_size; + + return use_xsave() ? size + FP_XSTATE_MAGIC2_SIZE : size; } /* * Restore FPU state from a sigframe: */ -int fpu__restore_sig(void __user *buf, int ia32_frame) +bool fpu__restore_sig(void __user *buf, int ia32_frame) { - unsigned int size = xstate_sigframe_size(); struct fpu *fpu = ¤t->thread.fpu; void __user *buf_fx = buf; bool ia32_fxstate = false; - int ret; + bool success = false; + unsigned int size; if (unlikely(!buf)) { fpu__clear_user_states(fpu); - return 0; + return true; } + size = xstate_sigframe_size(fpu->fpstate); + ia32_frame &= (IS_ENABLED(CONFIG_X86_32) || IS_ENABLED(CONFIG_IA32_EMULATION)); @@ -456,30 +485,28 @@ int fpu__restore_sig(void __user *buf, int ia32_frame) ia32_fxstate = true; } - if (!access_ok(buf, size)) { - ret = -EACCES; + if (!access_ok(buf, size)) goto out; - } if (!IS_ENABLED(CONFIG_X86_64) && !cpu_feature_enabled(X86_FEATURE_FPU)) { - ret = fpregs_soft_set(current, NULL, 0, - sizeof(struct user_i387_ia32_struct), - NULL, buf); + success = !fpregs_soft_set(current, NULL, 0, + sizeof(struct user_i387_ia32_struct), + NULL, buf); } else { - ret = __fpu_restore_sig(buf, buf_fx, ia32_fxstate); + success = __fpu_restore_sig(buf, buf_fx, ia32_fxstate); } out: - if (unlikely(ret)) + if (unlikely(!success)) fpu__clear_user_states(fpu); - return ret; + return success; } unsigned long fpu__alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long *buf_fx, unsigned long *size) { - unsigned long frame_size = xstate_sigframe_size(); + unsigned long frame_size = xstate_sigframe_size(current->thread.fpu.fpstate); *buf_fx = sp = round_down(sp - frame_size, 64); if (ia32_frame && use_fxsr()) { @@ -492,9 +519,12 @@ fpu__alloc_mathframe(unsigned long sp, int ia32_frame, return sp; } -unsigned long fpu__get_fpstate_size(void) +unsigned long __init fpu__get_fpstate_size(void) { - unsigned long ret = xstate_sigframe_size(); + unsigned long ret = fpu_user_cfg.max_size; + + if (use_xsave()) + ret += FP_XSTATE_MAGIC2_SIZE; /* * This space is needed on (most) 32-bit kernels, or when a 32-bit @@ -510,28 +540,3 @@ unsigned long fpu__get_fpstate_size(void) return ret; } -/* - * Prepare the SW reserved portion of the fxsave memory layout, indicating - * the presence of the extended state information in the memory layout - * pointed by the fpstate pointer in the sigcontext. - * This will be saved when ever the FP and extended state context is - * saved on the user stack during the signal handler delivery to the user. - */ -void fpu__init_prepare_fx_sw_frame(void) -{ - int size = fpu_user_xstate_size + FP_XSTATE_MAGIC2_SIZE; - - fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1; - fx_sw_reserved.extended_size = size; - fx_sw_reserved.xfeatures = xfeatures_mask_uabi(); - fx_sw_reserved.xstate_size = fpu_user_xstate_size; - - if (IS_ENABLED(CONFIG_IA32_EMULATION) || - IS_ENABLED(CONFIG_X86_32)) { - int fsave_header_size = sizeof(struct fregs_state); - - fx_sw_reserved_ia32 = fx_sw_reserved; - fx_sw_reserved_ia32.extended_size = size + fsave_header_size; - } -} - diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index c8def1b7f8..7c7824ae78 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -4,21 +4,33 @@ * * Author: Suresh Siddha */ +#include #include #include #include +#include #include #include #include +#include #include -#include -#include #include -#include +#include +#include #include -#include +#include +#include + +#include "context.h" +#include "internal.h" +#include "legacy.h" +#include "xstate.h" + +#define for_each_extended_xfeature(bit, mask) \ + (bit) = FIRST_EXTENDED_XFEATURE; \ + for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask)) /* * Although we spell it out in here, the Processor Trace @@ -39,29 +51,32 @@ static const char *xfeature_names[] = "Protection Keys User registers", "PASID state", "unknown xstate feature" , + "unknown xstate feature" , + "unknown xstate feature" , + "unknown xstate feature" , + "unknown xstate feature" , + "unknown xstate feature" , + "AMX Tile config" , + "AMX Tile data" , + "unknown xstate feature" , }; -static short xsave_cpuid_features[] __initdata = { - X86_FEATURE_FPU, - X86_FEATURE_XMM, - X86_FEATURE_AVX, - X86_FEATURE_MPX, - X86_FEATURE_MPX, - X86_FEATURE_AVX512F, - X86_FEATURE_AVX512F, - X86_FEATURE_AVX512F, - X86_FEATURE_INTEL_PT, - X86_FEATURE_PKU, - X86_FEATURE_ENQCMD, +static unsigned short xsave_cpuid_features[] __initdata = { + [XFEATURE_FP] = X86_FEATURE_FPU, + [XFEATURE_SSE] = X86_FEATURE_XMM, + [XFEATURE_YMM] = X86_FEATURE_AVX, + [XFEATURE_BNDREGS] = X86_FEATURE_MPX, + [XFEATURE_BNDCSR] = X86_FEATURE_MPX, + [XFEATURE_OPMASK] = X86_FEATURE_AVX512F, + [XFEATURE_ZMM_Hi256] = X86_FEATURE_AVX512F, + [XFEATURE_Hi16_ZMM] = X86_FEATURE_AVX512F, + [XFEATURE_PT_UNIMPLEMENTED_SO_FAR] = X86_FEATURE_INTEL_PT, + [XFEATURE_PKRU] = X86_FEATURE_PKU, + [XFEATURE_PASID] = X86_FEATURE_ENQCMD, + [XFEATURE_XTILE_CFG] = X86_FEATURE_AMX_TILE, + [XFEATURE_XTILE_DATA] = X86_FEATURE_AMX_TILE, }; -/* - * This represents the full set of bits that should ever be set in a kernel - * XSAVE buffer, both supervisor and user xstates. - */ -u64 xfeatures_mask_all __ro_after_init; -EXPORT_SYMBOL_GPL(xfeatures_mask_all); - static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init = { [ 0 ... XFEATURE_MAX - 1] = -1}; static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init = @@ -71,13 +86,6 @@ static unsigned int xstate_comp_offsets[XFEATURE_MAX] __ro_after_init = static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] __ro_after_init = { [ 0 ... XFEATURE_MAX - 1] = -1}; -/* - * The XSAVE area of kernel can be in standard or compacted format; - * it is always in standard format for user mode. This is the user - * mode standard format size used for signal and ptrace frames. - */ -unsigned int fpu_user_xstate_size __ro_after_init; - /* * Return whether the system supports a given xfeature. * @@ -85,7 +93,7 @@ unsigned int fpu_user_xstate_size __ro_after_init; */ int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) { - u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all; + u64 xfeatures_missing = xfeatures_needed & ~fpu_kernel_cfg.max_features; if (unlikely(feature_name)) { long xfeature_idx, max_idx; @@ -135,17 +143,26 @@ static bool xfeature_is_supervisor(int xfeature_nr) */ void fpu__init_cpu_xstate(void) { - if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all) + if (!boot_cpu_has(X86_FEATURE_XSAVE) || !fpu_kernel_cfg.max_features) return; cr4_set_bits(X86_CR4_OSXSAVE); + /* + * Must happen after CR4 setup and before xsetbv() to allow KVM + * lazy passthrough. Write independent of the dynamic state static + * key as that does not work on the boot CPU. This also ensures + * that any stale state is wiped out from XFD. + */ + if (cpu_feature_enabled(X86_FEATURE_XFD)) + wrmsrl(MSR_IA32_XFD, init_fpstate.xfd); + /* * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features * managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user * states can be set here. */ - xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_uabi()); + xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features); /* * MSR_IA32_XSS sets supervisor states managed by XSAVES. @@ -158,7 +175,7 @@ void fpu__init_cpu_xstate(void) static bool xfeature_enabled(enum xfeature xfeature) { - return xfeatures_mask_all & BIT_ULL(xfeature); + return fpu_kernel_cfg.max_features & BIT_ULL(xfeature); } /* @@ -184,10 +201,7 @@ static void __init setup_xstate_features(void) xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, xmm_space); - for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { - if (!xfeature_enabled(i)) - continue; - + for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); xstate_sizes[i] = eax; @@ -236,6 +250,8 @@ static void __init print_xstate_features(void) print_xstate_feature(XFEATURE_MASK_Hi16_ZMM); print_xstate_feature(XFEATURE_MASK_PKRU); print_xstate_feature(XFEATURE_MASK_PASID); + print_xstate_feature(XFEATURE_MASK_XTILE_CFG); + print_xstate_feature(XFEATURE_MASK_XTILE_DATA); } /* @@ -291,20 +307,15 @@ static void __init setup_xstate_comp_offsets(void) xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state, xmm_space); - if (!boot_cpu_has(X86_FEATURE_XSAVES)) { - for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { - if (xfeature_enabled(i)) - xstate_comp_offsets[i] = xstate_offsets[i]; - } + if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) { + for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) + xstate_comp_offsets[i] = xstate_offsets[i]; return; } next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; - for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { - if (!xfeature_enabled(i)) - continue; - + for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { if (xfeature_is_aligned(i)) next_offset = ALIGN(next_offset, 64); @@ -328,8 +339,8 @@ static void __init setup_supervisor_only_offsets(void) next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; - for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { - if (!xfeature_enabled(i) || !xfeature_is_supervisor(i)) + for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { + if (!xfeature_is_supervisor(i)) continue; if (xfeature_is_aligned(i)) @@ -347,14 +358,35 @@ static void __init print_xstate_offset_size(void) { int i; - for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { - if (!xfeature_enabled(i)) - continue; + for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", i, xstate_comp_offsets[i], i, xstate_sizes[i]); } } +/* + * This function is called only during boot time when x86 caps are not set + * up and alternative can not be used yet. + */ +static __init void os_xrstor_booting(struct xregs_state *xstate) +{ + u64 mask = fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSTATE; + u32 lmask = mask; + u32 hmask = mask >> 32; + int err; + + if (cpu_feature_enabled(X86_FEATURE_XSAVES)) + XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); + else + XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); + + /* + * We should never fault when copying from a kernel buffer, and the FPU + * state we set at boot time should be valid. + */ + WARN_ON_FPU(err); +} + /* * All supported features have either init state all zeros or are * handled in setup_init_fpu() individually. This is an explicit @@ -372,36 +404,30 @@ static void __init print_xstate_offset_size(void) XFEATURE_MASK_PKRU | \ XFEATURE_MASK_BNDREGS | \ XFEATURE_MASK_BNDCSR | \ - XFEATURE_MASK_PASID) + XFEATURE_MASK_PASID | \ + XFEATURE_MASK_XTILE) /* * setup the xstate image representing the init state */ static void __init setup_init_fpu_buf(void) { - static int on_boot_cpu __initdata = 1; - BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED | XFEATURE_MASK_SUPERVISOR_SUPPORTED) != XFEATURES_INIT_FPSTATE_HANDLED); - WARN_ON_FPU(!on_boot_cpu); - on_boot_cpu = 0; - if (!boot_cpu_has(X86_FEATURE_XSAVE)) return; setup_xstate_features(); print_xstate_features(); - if (boot_cpu_has(X86_FEATURE_XSAVES)) - init_fpstate.xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT | - xfeatures_mask_all; + xstate_init_xcomp_bv(&init_fpstate.regs.xsave, fpu_kernel_cfg.max_features); /* * Init all the features state with header.xfeatures being 0x0 */ - os_xrstor_booting(&init_fpstate.xsave); + os_xrstor_booting(&init_fpstate.regs.xsave); /* * All components are now in init state. Read the state back so @@ -419,7 +445,7 @@ static void __init setup_init_fpu_buf(void) * state is all zeroes or if not to add the necessary handling * here. */ - fxsave(&init_fpstate.fxsave); + fxsave(&init_fpstate.regs.fxsave); } static int xfeature_uncompacted_offset(int xfeature_nr) @@ -451,10 +477,11 @@ int xfeature_size(int xfeature_nr) } /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */ -static int validate_user_xstate_header(const struct xstate_header *hdr) +static int validate_user_xstate_header(const struct xstate_header *hdr, + struct fpstate *fpstate) { /* No unknown or supervisor features may be set */ - if (hdr->xfeatures & ~xfeatures_mask_uabi()) + if (hdr->xfeatures & ~fpstate->user_xfeatures) return -EINVAL; /* Userspace must use the uncompacted format */ @@ -474,7 +501,7 @@ static int validate_user_xstate_header(const struct xstate_header *hdr) return 0; } -static void __xstate_dump_leaves(void) +static void __init __xstate_dump_leaves(void) { int i; u32 eax, ebx, ecx, edx; @@ -509,12 +536,73 @@ static void __xstate_dump_leaves(void) } \ } while (0) +/** + * check_xtile_data_against_struct - Check tile data state size. + * + * Calculate the state size by multiplying the single tile size which is + * recorded in a C struct, and the number of tiles that the CPU informs. + * Compare the provided size with the calculation. + * + * @size: The tile data state size + * + * Returns: 0 on success, -EINVAL on mismatch. + */ +static int __init check_xtile_data_against_struct(int size) +{ + u32 max_palid, palid, state_size; + u32 eax, ebx, ecx, edx; + u16 max_tile; + + /* + * Check the maximum palette id: + * eax: the highest numbered palette subleaf. + */ + cpuid_count(TILE_CPUID, 0, &max_palid, &ebx, &ecx, &edx); + + /* + * Cross-check each tile size and find the maximum number of + * supported tiles. + */ + for (palid = 1, max_tile = 0; palid <= max_palid; palid++) { + u16 tile_size, max; + + /* + * Check the tile size info: + * eax[31:16]: bytes per title + * ebx[31:16]: the max names (or max number of tiles) + */ + cpuid_count(TILE_CPUID, palid, &eax, &ebx, &edx, &edx); + tile_size = eax >> 16; + max = ebx >> 16; + + if (tile_size != sizeof(struct xtile_data)) { + pr_err("%s: struct is %zu bytes, cpu xtile %d bytes\n", + __stringify(XFEATURE_XTILE_DATA), + sizeof(struct xtile_data), tile_size); + __xstate_dump_leaves(); + return -EINVAL; + } + + if (max > max_tile) + max_tile = max; + } + + state_size = sizeof(struct xtile_data) * max_tile; + if (size != state_size) { + pr_err("%s: calculated size is %u bytes, cpu state %d bytes\n", + __stringify(XFEATURE_XTILE_DATA), state_size, size); + __xstate_dump_leaves(); + return -EINVAL; + } + return 0; +} + /* * We have a C struct for each 'xstate'. We need to ensure * that our software representation matches what the CPU * tells us about the state's size. */ -static void check_xstate_against_struct(int nr) +static bool __init check_xstate_against_struct(int nr) { /* * Ask the CPU for the size of the state. @@ -532,6 +620,11 @@ static void check_xstate_against_struct(int nr) XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state); XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state); XCHECK_SZ(sz, nr, XFEATURE_PASID, struct ia32_pasid_state); + XCHECK_SZ(sz, nr, XFEATURE_XTILE_CFG, struct xtile_cfg); + + /* The tile data size varies between implementations. */ + if (nr == XFEATURE_XTILE_DATA) + check_xtile_data_against_struct(sz); /* * Make *SURE* to add any feature numbers in below if @@ -541,10 +634,39 @@ static void check_xstate_against_struct(int nr) if ((nr < XFEATURE_YMM) || (nr >= XFEATURE_MAX) || (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) || - ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) { + ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_RSRVD_COMP_16))) { WARN_ONCE(1, "no structure for xstate: %d\n", nr); XSTATE_WARN_ON(1); + return false; } + return true; +} + +static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted) +{ + unsigned int size = FXSAVE_SIZE + XSAVE_HDR_SIZE; + int i; + + for_each_extended_xfeature(i, xfeatures) { + /* Align from the end of the previous feature */ + if (xfeature_is_aligned(i)) + size = ALIGN(size, 64); + /* + * In compacted format the enabled features are packed, + * i.e. disabled features do not occupy space. + * + * In non-compacted format the offsets are fixed and + * disabled states still occupy space in the memory buffer. + */ + if (!compacted) + size = xfeature_uncompacted_offset(i); + /* + * Add the feature size even for non-compacted format + * to make the end result correct + */ + size += xfeature_size(i); + } + return size; } /* @@ -556,44 +678,29 @@ static void check_xstate_against_struct(int nr) * covered by these checks. Only the size of the buffer for task->fpu * is checked here. */ -static void do_extra_xstate_size_checks(void) +static bool __init paranoid_xstate_size_valid(unsigned int kernel_size) { - int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE; + bool compacted = cpu_feature_enabled(X86_FEATURE_XSAVES); + unsigned int size = FXSAVE_SIZE + XSAVE_HDR_SIZE; int i; - for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { - if (!xfeature_enabled(i)) - continue; - - check_xstate_against_struct(i); + for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { + if (!check_xstate_against_struct(i)) + return false; /* * Supervisor state components can be managed only by * XSAVES. */ - if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) - XSTATE_WARN_ON(xfeature_is_supervisor(i)); - - /* Align from the end of the previous feature */ - if (xfeature_is_aligned(i)) - paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64); - /* - * The offset of a given state in the non-compacted - * format is given to us in a CPUID leaf. We check - * them for being ordered (increasing offsets) in - * setup_xstate_features(). XSAVES uses compacted format. - */ - if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) - paranoid_xstate_size = xfeature_uncompacted_offset(i); - /* - * The compacted-format offset always depends on where - * the previous state ended. - */ - paranoid_xstate_size += xfeature_size(i); + if (!compacted && xfeature_is_supervisor(i)) { + XSTATE_WARN_ON(1); + return false; + } } - XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size); + size = xstate_calculate_size(fpu_kernel_cfg.max_features, compacted); + XSTATE_WARN_ON(size != kernel_size); + return size == kernel_size; } - /* * Get total size of enabled xstates in XCR0 | IA32_XSS. * @@ -644,7 +751,7 @@ static unsigned int __init get_xsaves_size_no_independent(void) return size; } -static unsigned int __init get_xsave_size(void) +static unsigned int __init get_xsave_size_user(void) { unsigned int eax, ebx, ecx, edx; /* @@ -662,44 +769,54 @@ static unsigned int __init get_xsave_size(void) * Will the runtime-enumerated 'xstate_size' fit in the init * task's statically-allocated buffer? */ -static bool is_supported_xstate_size(unsigned int test_xstate_size) +static bool __init is_supported_xstate_size(unsigned int test_xstate_size) { - if (test_xstate_size <= sizeof(union fpregs_state)) + if (test_xstate_size <= sizeof(init_fpstate.regs)) return true; pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n", - sizeof(union fpregs_state), test_xstate_size); + sizeof(init_fpstate.regs), test_xstate_size); return false; } static int __init init_xstate_size(void) { /* Recompute the context size for enabled features: */ - unsigned int possible_xstate_size; - unsigned int xsave_size; + unsigned int user_size, kernel_size, kernel_default_size; + bool compacted = cpu_feature_enabled(X86_FEATURE_XSAVES); - xsave_size = get_xsave_size(); + /* Uncompacted user space size */ + user_size = get_xsave_size_user(); - if (boot_cpu_has(X86_FEATURE_XSAVES)) - possible_xstate_size = get_xsaves_size_no_independent(); + /* + * XSAVES kernel size includes supervisor states and + * uses compacted format when available. + * + * XSAVE does not support supervisor states so + * kernel and user size is identical. + */ + if (compacted) + kernel_size = get_xsaves_size_no_independent(); else - possible_xstate_size = xsave_size; + kernel_size = user_size; - /* Ensure we have the space to store all enabled: */ - if (!is_supported_xstate_size(possible_xstate_size)) + kernel_default_size = + xstate_calculate_size(fpu_kernel_cfg.default_features, compacted); + + /* Ensure we have the space to store all default enabled features. */ + if (!is_supported_xstate_size(kernel_default_size)) return -EINVAL; - /* - * The size is OK, we are definitely going to use xsave, - * make it known to the world that we need more space. - */ - fpu_kernel_xstate_size = possible_xstate_size; - do_extra_xstate_size_checks(); + if (!paranoid_xstate_size_valid(kernel_size)) + return -EINVAL; + + fpu_kernel_cfg.max_size = kernel_size; + fpu_user_cfg.max_size = user_size; + + fpu_kernel_cfg.default_size = kernel_default_size; + fpu_user_cfg.default_size = + xstate_calculate_size(fpu_user_cfg.default_features, false); - /* - * User space is always in standard format. - */ - fpu_user_xstate_size = xsave_size; return 0; } @@ -707,28 +824,38 @@ static int __init init_xstate_size(void) * We enabled the XSAVE hardware, but something went wrong and * we can not use it. Disable it. */ -static void fpu__init_disable_system_xstate(void) +static void __init fpu__init_disable_system_xstate(unsigned int legacy_size) { - xfeatures_mask_all = 0; + fpu_kernel_cfg.max_features = 0; cr4_clear_bits(X86_CR4_OSXSAVE); setup_clear_cpu_cap(X86_FEATURE_XSAVE); + + /* Restore the legacy size.*/ + fpu_kernel_cfg.max_size = legacy_size; + fpu_kernel_cfg.default_size = legacy_size; + fpu_user_cfg.max_size = legacy_size; + fpu_user_cfg.default_size = legacy_size; + + /* + * Prevent enabling the static branch which enables writes to the + * XFD MSR. + */ + init_fpstate.xfd = 0; + + fpstate_reset(¤t->thread.fpu); } /* * Enable and initialize the xsave feature. * Called once per system bootup. */ -void __init fpu__init_system_xstate(void) +void __init fpu__init_system_xstate(unsigned int legacy_size) { unsigned int eax, ebx, ecx, edx; - static int on_boot_cpu __initdata = 1; u64 xfeatures; int err; int i; - WARN_ON_FPU(!on_boot_cpu); - on_boot_cpu = 0; - if (!boot_cpu_has(X86_FEATURE_FPU)) { pr_info("x86/fpu: No FPU detected\n"); return; @@ -749,22 +876,22 @@ void __init fpu__init_system_xstate(void) * Find user xstates supported by the processor. */ cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); - xfeatures_mask_all = eax + ((u64)edx << 32); + fpu_kernel_cfg.max_features = eax + ((u64)edx << 32); /* * Find supervisor xstates supported by the processor. */ cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); - xfeatures_mask_all |= ecx + ((u64)edx << 32); + fpu_kernel_cfg.max_features |= ecx + ((u64)edx << 32); - if ((xfeatures_mask_uabi() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) { + if ((fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) { /* * This indicates that something really unexpected happened * with the enumeration. Disable XSAVE and try to continue * booting without it. This is too early to BUG(). */ pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", - xfeatures_mask_all); + fpu_kernel_cfg.max_features); goto out_disable; } @@ -772,15 +899,39 @@ void __init fpu__init_system_xstate(void) * Clear XSAVE features that are disabled in the normal CPUID. */ for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) { - if (!boot_cpu_has(xsave_cpuid_features[i])) - xfeatures_mask_all &= ~BIT_ULL(i); + unsigned short cid = xsave_cpuid_features[i]; + + /* Careful: X86_FEATURE_FPU is 0! */ + if ((i != XFEATURE_FP && !cid) || !boot_cpu_has(cid)) + fpu_kernel_cfg.max_features &= ~BIT_ULL(i); } - xfeatures_mask_all &= XFEATURE_MASK_USER_SUPPORTED | + if (!cpu_feature_enabled(X86_FEATURE_XFD)) + fpu_kernel_cfg.max_features &= ~XFEATURE_MASK_USER_DYNAMIC; + + fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED | XFEATURE_MASK_SUPERVISOR_SUPPORTED; + fpu_user_cfg.max_features = fpu_kernel_cfg.max_features; + fpu_user_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED; + + /* Clean out dynamic features from default */ + fpu_kernel_cfg.default_features = fpu_kernel_cfg.max_features; + fpu_kernel_cfg.default_features &= ~XFEATURE_MASK_USER_DYNAMIC; + + fpu_user_cfg.default_features = fpu_user_cfg.max_features; + fpu_user_cfg.default_features &= ~XFEATURE_MASK_USER_DYNAMIC; + /* Store it for paranoia check at the end */ - xfeatures = xfeatures_mask_all; + xfeatures = fpu_kernel_cfg.max_features; + + /* + * Initialize the default XFD state in initfp_state and enable the + * dynamic sizing mechanism if dynamic states are available. The + * static key cannot be enabled here because this runs before + * jump_label_init(). This is delayed to an initcall. + */ + init_fpstate.xfd = fpu_user_cfg.max_features & XFEATURE_MASK_USER_DYNAMIC; /* Enable xstate instructions to be able to continue with initialization: */ fpu__init_cpu_xstate(); @@ -788,13 +939,16 @@ void __init fpu__init_system_xstate(void) if (err) goto out_disable; + /* Reset the state for the current task */ + fpstate_reset(¤t->thread.fpu); + /* * Update info used for ptrace frames; use standard-format size and no * supervisor xstates: */ - update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_uabi()); + update_regset_xstate_info(fpu_user_cfg.max_size, + fpu_user_cfg.max_features); - fpu__init_prepare_fx_sw_frame(); setup_init_fpu_buf(); setup_xstate_comp_offsets(); setup_supervisor_only_offsets(); @@ -803,22 +957,22 @@ void __init fpu__init_system_xstate(void) * Paranoia check whether something in the setup modified the * xfeatures mask. */ - if (xfeatures != xfeatures_mask_all) { + if (xfeatures != fpu_kernel_cfg.max_features) { pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n", - xfeatures, xfeatures_mask_all); + xfeatures, fpu_kernel_cfg.max_features); goto out_disable; } print_xstate_offset_size(); pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", - xfeatures_mask_all, - fpu_kernel_xstate_size, + fpu_kernel_cfg.max_features, + fpu_kernel_cfg.max_size, boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard"); return; out_disable: /* something went wrong, try to boot without any XSAVE support */ - fpu__init_disable_system_xstate(); + fpu__init_disable_system_xstate(legacy_size); } /* @@ -830,7 +984,7 @@ void fpu__resume_cpu(void) * Restore XCR0 on xsave capable CPUs: */ if (cpu_feature_enabled(X86_FEATURE_XSAVE)) - xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_uabi()); + xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features); /* * Restore IA32_XSS. The same CPUID bit enumerates support @@ -840,6 +994,9 @@ void fpu__resume_cpu(void) wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | xfeatures_mask_independent()); } + + if (fpu_state_size_dynamic()) + wrmsrl(MSR_IA32_XFD, current->thread.fpu.fpstate->xfd); } /* @@ -886,7 +1043,7 @@ void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) * We should not ever be requesting features that we * have not enabled. */ - WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)), + WARN_ONCE(!(fpu_kernel_cfg.max_features & BIT_ULL(xfeature_nr)), "get of unsupported state"); /* * This assumes the last 'xsave*' instruction to @@ -904,7 +1061,6 @@ void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) return __raw_xsave_addr(xsave, xfeature_nr); } -EXPORT_SYMBOL_GPL(get_xsave_addr); #ifdef CONFIG_ARCH_HAS_PKEYS @@ -961,9 +1117,10 @@ static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, } /** - * copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer + * __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer * @to: membuf descriptor - * @tsk: The task from which to copy the saved xstate + * @fpstate: The fpstate buffer from which to copy + * @pkru_val: The PKRU value to store in the PKRU component * @copy_mode: The requested copy mode * * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming @@ -972,14 +1129,15 @@ static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, * * It supports partial copy but @to.pos always starts from zero. */ -void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, - enum xstate_copy_mode copy_mode) +void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate, + u32 pkru_val, enum xstate_copy_mode copy_mode) { const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr); - struct xregs_state *xsave = &tsk->thread.fpu.state.xsave; - struct xregs_state *xinit = &init_fpstate.xsave; + struct xregs_state *xinit = &init_fpstate.regs.xsave; + struct xregs_state *xsave = &fpstate->regs.xsave; struct xstate_header header; unsigned int zerofrom; + u64 mask; int i; memset(&header, 0, sizeof(header)); @@ -996,7 +1154,7 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, break; case XSTATE_COPY_XSAVE: - header.xfeatures &= xfeatures_mask_uabi(); + header.xfeatures &= fpstate->user_xfeatures; break; } @@ -1033,17 +1191,15 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, zerofrom = offsetof(struct xregs_state, extended_state_area); - for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { - /* - * The ptrace buffer is in non-compacted XSAVE format. - * In non-compacted format disabled features still occupy - * state space, but there is no state to copy from in the - * compacted init_fpstate. The gap tracking will zero this - * later. - */ - if (!(xfeatures_mask_uabi() & BIT_ULL(i))) - continue; + /* + * The ptrace buffer is in non-compacted XSAVE format. In + * non-compacted format disabled features still occupy state space, + * but there is no state to copy from in the compacted + * init_fpstate. The gap tracking will zero these states. + */ + mask = fpstate->user_xfeatures; + for_each_extended_xfeature(i, mask) { /* * If there was a feature or alignment gap, zero the space * in the destination buffer. @@ -1055,10 +1211,9 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, struct pkru_state pkru = {0}; /* * PKRU is not necessarily up to date in the - * thread's XSAVE buffer. Fill this part from the - * per-thread storage. + * XSAVE buffer. Use the provided value. */ - pkru.pkru = tsk->thread.pkru; + pkru.pkru = pkru_val; membuf_write(&to, &pkru, sizeof(pkru)); } else { copy_feature(header.xfeatures & BIT_ULL(i), &to, @@ -1078,6 +1233,25 @@ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, membuf_zero(&to, to.left); } +/** + * copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer + * @to: membuf descriptor + * @tsk: The task from which to copy the saved xstate + * @copy_mode: The requested copy mode + * + * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming + * format, i.e. from the kernel internal hardware dependent storage format + * to the requested @mode. UABI XSTATE is always uncompacted! + * + * It supports partial copy but @to.pos always starts from zero. + */ +void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, + enum xstate_copy_mode copy_mode) +{ + __copy_xstate_to_uabi_buf(to, tsk->thread.fpu.fpstate, + tsk->thread.pkru, copy_mode); +} + static int copy_from_buffer(void *dst, unsigned int offset, unsigned int size, const void *kbuf, const void __user *ubuf) { @@ -1091,9 +1265,10 @@ static int copy_from_buffer(void *dst, unsigned int offset, unsigned int size, } -static int copy_uabi_to_xstate(struct xregs_state *xsave, const void *kbuf, +static int copy_uabi_to_xstate(struct fpstate *fpstate, const void *kbuf, const void __user *ubuf) { + struct xregs_state *xsave = &fpstate->regs.xsave; unsigned int offset, size; struct xstate_header hdr; u64 mask; @@ -1103,7 +1278,7 @@ static int copy_uabi_to_xstate(struct xregs_state *xsave, const void *kbuf, if (copy_from_buffer(&hdr, offset, sizeof(hdr), kbuf, ubuf)) return -EFAULT; - if (validate_user_xstate_header(&hdr)) + if (validate_user_xstate_header(&hdr, fpstate)) return -EINVAL; /* Validate MXCSR when any of the related features is in use */ @@ -1156,12 +1331,11 @@ static int copy_uabi_to_xstate(struct xregs_state *xsave, const void *kbuf, /* * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S] - * format and copy to the target thread. This is called from - * xstateregs_set(). + * format and copy to the target thread. Used by ptrace and KVM. */ -int copy_uabi_from_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf) +int copy_uabi_from_kernel_to_xstate(struct fpstate *fpstate, const void *kbuf) { - return copy_uabi_to_xstate(xsave, kbuf, NULL); + return copy_uabi_to_xstate(fpstate, kbuf, NULL); } /* @@ -1169,26 +1343,20 @@ int copy_uabi_from_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf) * XSAVE[S] format and copy to the target thread. This is called from the * sigreturn() and rt_sigreturn() system calls. */ -int copy_sigframe_from_user_to_xstate(struct xregs_state *xsave, +int copy_sigframe_from_user_to_xstate(struct fpstate *fpstate, const void __user *ubuf) { - return copy_uabi_to_xstate(xsave, NULL, ubuf); + return copy_uabi_to_xstate(fpstate, NULL, ubuf); } -static bool validate_xsaves_xrstors(u64 mask) +static bool validate_independent_components(u64 mask) { u64 xchk; if (WARN_ON_FPU(!cpu_feature_enabled(X86_FEATURE_XSAVES))) return false; - /* - * Validate that this is either a task->fpstate related component - * subset or an independent one. - */ - if (mask & xfeatures_mask_independent()) - xchk = ~xfeatures_mask_independent(); - else - xchk = ~xfeatures_mask_all; + + xchk = ~xfeatures_mask_independent(); if (WARN_ON_ONCE(!mask || mask & xchk)) return false; @@ -1206,14 +1374,13 @@ static bool validate_xsaves_xrstors(u64 mask) * buffer should be zeroed otherwise a consecutive XRSTORS from that buffer * can #GP. * - * The feature mask must either be a subset of the independent features or - * a subset of the task->fpstate related features. + * The feature mask must be a subset of the independent features. */ void xsaves(struct xregs_state *xstate, u64 mask) { int err; - if (!validate_xsaves_xrstors(mask)) + if (!validate_independent_components(mask)) return; XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err); @@ -1231,20 +1398,423 @@ void xsaves(struct xregs_state *xstate, u64 mask) * Proper usage is to restore the state which was saved with * xsaves() into @xstate. * - * The feature mask must either be a subset of the independent features or - * a subset of the task->fpstate related features. + * The feature mask must be a subset of the independent features. */ void xrstors(struct xregs_state *xstate, u64 mask) { int err; - if (!validate_xsaves_xrstors(mask)) + if (!validate_independent_components(mask)) return; XSTATE_OP(XRSTORS, xstate, (u32)mask, (u32)(mask >> 32), err); WARN_ON_ONCE(err); } +#if IS_ENABLED(CONFIG_KVM) +void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature) +{ + void *addr = get_xsave_addr(&fps->regs.xsave, xfeature); + + if (addr) + memset(addr, 0, xstate_sizes[xfeature]); +} +EXPORT_SYMBOL_GPL(fpstate_clear_xstate_component); +#endif + +#ifdef CONFIG_X86_64 + +#ifdef CONFIG_X86_DEBUG_FPU +/* + * Ensure that a subsequent XSAVE* or XRSTOR* instruction with RFBM=@mask + * can safely operate on the @fpstate buffer. + */ +static bool xstate_op_valid(struct fpstate *fpstate, u64 mask, bool rstor) +{ + u64 xfd = __this_cpu_read(xfd_state); + + if (fpstate->xfd == xfd) + return true; + + /* + * The XFD MSR does not match fpstate->xfd. That's invalid when + * the passed in fpstate is current's fpstate. + */ + if (fpstate->xfd == current->thread.fpu.fpstate->xfd) + return false; + + /* + * XRSTOR(S) from init_fpstate are always correct as it will just + * bring all components into init state and not read from the + * buffer. XSAVE(S) raises #PF after init. + */ + if (fpstate == &init_fpstate) + return rstor; + + /* + * XSAVE(S): clone(), fpu_swap_kvm_fpu() + * XRSTORS(S): fpu_swap_kvm_fpu() + */ + + /* + * No XSAVE/XRSTOR instructions (except XSAVE itself) touch + * the buffer area for XFD-disabled state components. + */ + mask &= ~xfd; + + /* + * Remove features which are valid in fpstate. They + * have space allocated in fpstate. + */ + mask &= ~fpstate->xfeatures; + + /* + * Any remaining state components in 'mask' might be written + * by XSAVE/XRSTOR. Fail validation it found. + */ + return !mask; +} + +void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor) +{ + WARN_ON_ONCE(!xstate_op_valid(fpstate, mask, rstor)); +} +#endif /* CONFIG_X86_DEBUG_FPU */ + +static int __init xfd_update_static_branch(void) +{ + /* + * If init_fpstate.xfd has bits set then dynamic features are + * available and the dynamic sizing must be enabled. + */ + if (init_fpstate.xfd) + static_branch_enable(&__fpu_state_size_dynamic); + return 0; +} +arch_initcall(xfd_update_static_branch) + +void fpstate_free(struct fpu *fpu) +{ + if (fpu->fpstate && fpu->fpstate != &fpu->__fpstate) + vfree(fpu->fpstate); +} + +/** + * fpstate_realloc - Reallocate struct fpstate for the requested new features + * + * @xfeatures: A bitmap of xstate features which extend the enabled features + * of that task + * @ksize: The required size for the kernel buffer + * @usize: The required size for user space buffers + * @guest_fpu: Pointer to a guest FPU container. NULL for host allocations + * + * Note vs. vmalloc(): If the task with a vzalloc()-allocated buffer + * terminates quickly, vfree()-induced IPIs may be a concern, but tasks + * with large states are likely to live longer. + * + * Returns: 0 on success, -ENOMEM on allocation error. + */ +static int fpstate_realloc(u64 xfeatures, unsigned int ksize, + unsigned int usize, struct fpu_guest *guest_fpu) +{ + struct fpu *fpu = ¤t->thread.fpu; + struct fpstate *curfps, *newfps = NULL; + unsigned int fpsize; + bool in_use; + + fpsize = ksize + ALIGN(offsetof(struct fpstate, regs), 64); + + newfps = vzalloc(fpsize); + if (!newfps) + return -ENOMEM; + newfps->size = ksize; + newfps->user_size = usize; + newfps->is_valloc = true; + + /* + * When a guest FPU is supplied, use @guest_fpu->fpstate + * as reference independent whether it is in use or not. + */ + curfps = guest_fpu ? guest_fpu->fpstate : fpu->fpstate; + + /* Determine whether @curfps is the active fpstate */ + in_use = fpu->fpstate == curfps; + + if (guest_fpu) { + newfps->is_guest = true; + newfps->is_confidential = curfps->is_confidential; + newfps->in_use = curfps->in_use; + guest_fpu->xfeatures |= xfeatures; + guest_fpu->uabi_size = usize; + } + + fpregs_lock(); + /* + * If @curfps is in use, ensure that the current state is in the + * registers before swapping fpstate as that might invalidate it + * due to layout changes. + */ + if (in_use && test_thread_flag(TIF_NEED_FPU_LOAD)) + fpregs_restore_userregs(); + + newfps->xfeatures = curfps->xfeatures | xfeatures; + + if (!guest_fpu) + newfps->user_xfeatures = curfps->user_xfeatures | xfeatures; + + newfps->xfd = curfps->xfd & ~xfeatures; + + /* Do the final updates within the locked region */ + xstate_init_xcomp_bv(&newfps->regs.xsave, newfps->xfeatures); + + if (guest_fpu) { + guest_fpu->fpstate = newfps; + /* If curfps is active, update the FPU fpstate pointer */ + if (in_use) + fpu->fpstate = newfps; + } else { + fpu->fpstate = newfps; + } + + if (in_use) + xfd_update_state(fpu->fpstate); + fpregs_unlock(); + + /* Only free valloc'ed state */ + if (curfps && curfps->is_valloc) + vfree(curfps); + + return 0; +} + +static int validate_sigaltstack(unsigned int usize) +{ + struct task_struct *thread, *leader = current->group_leader; + unsigned long framesize = get_sigframe_size(); + + lockdep_assert_held(¤t->sighand->siglock); + + /* get_sigframe_size() is based on fpu_user_cfg.max_size */ + framesize -= fpu_user_cfg.max_size; + framesize += usize; + for_each_thread(leader, thread) { + if (thread->sas_ss_size && thread->sas_ss_size < framesize) + return -ENOSPC; + } + return 0; +} + +static int __xstate_request_perm(u64 permitted, u64 requested, bool guest) +{ + /* + * This deliberately does not exclude !XSAVES as we still might + * decide to optionally context switch XCR0 or talk the silicon + * vendors into extending XFD for the pre AMX states, especially + * AVX512. + */ + bool compacted = cpu_feature_enabled(X86_FEATURE_XSAVES); + struct fpu *fpu = ¤t->group_leader->thread.fpu; + struct fpu_state_perm *perm; + unsigned int ksize, usize; + u64 mask; + int ret = 0; + + /* Check whether fully enabled */ + if ((permitted & requested) == requested) + return 0; + + /* Calculate the resulting kernel state size */ + mask = permitted | requested; + ksize = xstate_calculate_size(mask, compacted); + + /* Calculate the resulting user state size */ + mask &= XFEATURE_MASK_USER_SUPPORTED; + usize = xstate_calculate_size(mask, false); + + if (!guest) { + ret = validate_sigaltstack(usize); + if (ret) + return ret; + } + + perm = guest ? &fpu->guest_perm : &fpu->perm; + /* Pairs with the READ_ONCE() in xstate_get_group_perm() */ + WRITE_ONCE(perm->__state_perm, requested); + /* Protected by sighand lock */ + perm->__state_size = ksize; + perm->__user_state_size = usize; + return ret; +} + +/* + * Permissions array to map facilities with more than one component + */ +static const u64 xstate_prctl_req[XFEATURE_MAX] = { + [XFEATURE_XTILE_DATA] = XFEATURE_MASK_XTILE_DATA, +}; + +static int xstate_request_perm(unsigned long idx, bool guest) +{ + u64 permitted, requested; + int ret; + + if (idx >= XFEATURE_MAX) + return -EINVAL; + + /* + * Look up the facility mask which can require more than + * one xstate component. + */ + idx = array_index_nospec(idx, ARRAY_SIZE(xstate_prctl_req)); + requested = xstate_prctl_req[idx]; + if (!requested) + return -EOPNOTSUPP; + + if ((fpu_user_cfg.max_features & requested) != requested) + return -EOPNOTSUPP; + + /* Lockless quick check */ + permitted = xstate_get_group_perm(guest); + if ((permitted & requested) == requested) + return 0; + + /* Protect against concurrent modifications */ + spin_lock_irq(¤t->sighand->siglock); + permitted = xstate_get_group_perm(guest); + + /* First vCPU allocation locks the permissions. */ + if (guest && (permitted & FPU_GUEST_PERM_LOCKED)) + ret = -EBUSY; + else + ret = __xstate_request_perm(permitted, requested, guest); + spin_unlock_irq(¤t->sighand->siglock); + return ret; +} + +int __xfd_enable_feature(u64 xfd_err, struct fpu_guest *guest_fpu) +{ + u64 xfd_event = xfd_err & XFEATURE_MASK_USER_DYNAMIC; + struct fpu_state_perm *perm; + unsigned int ksize, usize; + struct fpu *fpu; + + if (!xfd_event) { + if (!guest_fpu) + pr_err_once("XFD: Invalid xfd error: %016llx\n", xfd_err); + return 0; + } + + /* Protect against concurrent modifications */ + spin_lock_irq(¤t->sighand->siglock); + + /* If not permitted let it die */ + if ((xstate_get_group_perm(!!guest_fpu) & xfd_event) != xfd_event) { + spin_unlock_irq(¤t->sighand->siglock); + return -EPERM; + } + + fpu = ¤t->group_leader->thread.fpu; + perm = guest_fpu ? &fpu->guest_perm : &fpu->perm; + ksize = perm->__state_size; + usize = perm->__user_state_size; + + /* + * The feature is permitted. State size is sufficient. Dropping + * the lock is safe here even if more features are added from + * another task, the retrieved buffer sizes are valid for the + * currently requested feature(s). + */ + spin_unlock_irq(¤t->sighand->siglock); + + /* + * Try to allocate a new fpstate. If that fails there is no way + * out. + */ + if (fpstate_realloc(xfd_event, ksize, usize, guest_fpu)) + return -EFAULT; + return 0; +} + +int xfd_enable_feature(u64 xfd_err) +{ + return __xfd_enable_feature(xfd_err, NULL); +} + +#else /* CONFIG_X86_64 */ +static inline int xstate_request_perm(unsigned long idx, bool guest) +{ + return -EPERM; +} +#endif /* !CONFIG_X86_64 */ + +u64 xstate_get_guest_group_perm(void) +{ + return xstate_get_group_perm(true); +} +EXPORT_SYMBOL_GPL(xstate_get_guest_group_perm); + +/** + * fpu_xstate_prctl - xstate permission operations + * @tsk: Redundant pointer to current + * @option: A subfunction of arch_prctl() + * @arg2: option argument + * Return: 0 if successful; otherwise, an error code + * + * Option arguments: + * + * ARCH_GET_XCOMP_SUPP: Pointer to user space u64 to store the info + * ARCH_GET_XCOMP_PERM: Pointer to user space u64 to store the info + * ARCH_REQ_XCOMP_PERM: Facility number requested + * + * For facilities which require more than one XSTATE component, the request + * must be the highest state component number related to that facility, + * e.g. for AMX which requires XFEATURE_XTILE_CFG(17) and + * XFEATURE_XTILE_DATA(18) this would be XFEATURE_XTILE_DATA(18). + */ +long fpu_xstate_prctl(struct task_struct *tsk, int option, unsigned long arg2) +{ + u64 __user *uptr = (u64 __user *)arg2; + u64 permitted, supported; + unsigned long idx = arg2; + bool guest = false; + + if (tsk != current) + return -EPERM; + + switch (option) { + case ARCH_GET_XCOMP_SUPP: + supported = fpu_user_cfg.max_features | fpu_user_cfg.legacy_features; + return put_user(supported, uptr); + + case ARCH_GET_XCOMP_PERM: + /* + * Lockless snapshot as it can also change right after the + * dropping the lock. + */ + permitted = xstate_get_host_group_perm(); + permitted &= XFEATURE_MASK_USER_SUPPORTED; + return put_user(permitted, uptr); + + case ARCH_GET_XCOMP_GUEST_PERM: + permitted = xstate_get_guest_group_perm(); + permitted &= XFEATURE_MASK_USER_SUPPORTED; + return put_user(permitted, uptr); + + case ARCH_REQ_XCOMP_GUEST_PERM: + guest = true; + fallthrough; + + case ARCH_REQ_XCOMP_PERM: + if (!IS_ENABLED(CONFIG_X86_64)) + return -EOPNOTSUPP; + + return xstate_request_perm(idx, guest); + + default: + return -EINVAL; + } +} + #ifdef CONFIG_PROC_PID_ARCH_STATUS /* * Report the amount of time elapsed in millisecond since last AVX512 diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 1b3ce3b4a2..7cc540e6de 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -252,11 +252,6 @@ void arch_ftrace_update_code(int command) ftrace_modify_all_code(command); } -int __init ftrace_dyn_arch_init(void) -{ - return 0; -} - /* Currently only x86_64 supports dynamic trampolines */ #ifdef CONFIG_X86_64 @@ -308,7 +303,7 @@ union ftrace_op_code_union { } __attribute__((packed)); }; -#define RET_SIZE 1 +#define RET_SIZE 1 + IS_ENABLED(CONFIG_SLS) static unsigned long create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) @@ -527,7 +522,7 @@ static void *addr_from_call(void *ptr) return ptr + CALL_INSN_SIZE + call.disp; } -void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent, +void prepare_ftrace_return(unsigned long ip, unsigned long *parent, unsigned long frame_pointer); /* @@ -541,7 +536,8 @@ static void *static_tramp_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) void *ptr; if (ops && ops->trampoline) { -#ifdef CONFIG_FUNCTION_GRAPH_TRACER +#if !defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS) && \ + defined(CONFIG_FUNCTION_GRAPH_TRACER) /* * We only know about function graph tracer setting as static * trampoline. @@ -589,8 +585,9 @@ void arch_ftrace_trampoline_free(struct ftrace_ops *ops) #ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifdef CONFIG_DYNAMIC_FTRACE -extern void ftrace_graph_call(void); +#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS +extern void ftrace_graph_call(void); static const char *ftrace_jmp_replace(unsigned long ip, unsigned long addr) { return text_gen_insn(JMP32_INSN_OPCODE, (void *)ip, (void *)addr); @@ -618,19 +615,28 @@ int ftrace_disable_ftrace_graph_caller(void) return ftrace_mod_jmp(ip, &ftrace_stub); } +#else /* !CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */ +int ftrace_enable_ftrace_graph_caller(void) +{ + return 0; +} +int ftrace_disable_ftrace_graph_caller(void) +{ + return 0; +} +#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */ #endif /* !CONFIG_DYNAMIC_FTRACE */ /* * Hook the return address and push it in the stack of return addrs * in current thread info. */ -void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent, +void prepare_ftrace_return(unsigned long ip, unsigned long *parent, unsigned long frame_pointer) { unsigned long return_hooker = (unsigned long)&return_to_handler; - unsigned long old; - int faulted; + int bit; /* * When resuming from suspend-to-ram, this function can be indirectly @@ -650,37 +656,25 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent, if (unlikely(atomic_read(¤t->tracing_graph_pause))) return; - /* - * Protect against fault, even if it shouldn't - * happen. This tool is too much intrusive to - * ignore such a protection. - */ - asm volatile( - "1: " _ASM_MOV " (%[parent]), %[old]\n" - "2: " _ASM_MOV " %[return_hooker], (%[parent])\n" - " movl $0, %[faulted]\n" - "3:\n" - - ".section .fixup, \"ax\"\n" - "4: movl $1, %[faulted]\n" - " jmp 3b\n" - ".previous\n" - - _ASM_EXTABLE(1b, 4b) - _ASM_EXTABLE(2b, 4b) - - : [old] "=&r" (old), [faulted] "=r" (faulted) - : [parent] "r" (parent), [return_hooker] "r" (return_hooker) - : "memory" - ); - - if (unlikely(faulted)) { - ftrace_graph_stop(); - WARN_ON(1); + bit = ftrace_test_recursion_trylock(ip, *parent); + if (bit < 0) return; - } - if (function_graph_enter(old, self_addr, frame_pointer, parent)) - *parent = old; + if (!function_graph_enter(*parent, ip, frame_pointer, parent)) + *parent = return_hooker; + + ftrace_test_recursion_unlock(bit); } + +#ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS +void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *op, struct ftrace_regs *fregs) +{ + struct pt_regs *regs = &fregs->regs; + unsigned long *stack = (unsigned long *)kernel_stack_pointer(regs); + + prepare_ftrace_return(ip, (unsigned long *)stack, 0); +} +#endif + #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 7c273846c6..11ac028e30 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -132,7 +132,7 @@ #ifdef CONFIG_DYNAMIC_FTRACE SYM_FUNC_START(__fentry__) - retq + RET SYM_FUNC_END(__fentry__) EXPORT_SYMBOL(__fentry__) @@ -174,18 +174,13 @@ SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL) SYM_FUNC_END(ftrace_caller); SYM_FUNC_START(ftrace_epilogue) -#ifdef CONFIG_FUNCTION_GRAPH_TRACER -SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) - jmp ftrace_stub -#endif - /* * This is weak to keep gas from relaxing the jumps. - * It is also used to copy the retq for trampolines. + * It is also used to copy the RET for trampolines. */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) UNWIND_HINT_FUNC - retq + RET SYM_FUNC_END(ftrace_epilogue) SYM_FUNC_START(ftrace_regs_caller) @@ -251,7 +246,6 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) * If ORIG_RAX is anything but zero, make this a call to that. * See arch_ftrace_set_direct_caller(). */ - movq ORIG_RAX(%rsp), %rax testq %rax, %rax SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL) jnz 1f @@ -289,17 +283,8 @@ SYM_FUNC_START(__fentry__) cmpq $ftrace_stub, ftrace_trace_function jnz trace -fgraph_trace: -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - cmpq $ftrace_stub, ftrace_graph_return - jnz ftrace_graph_caller - - cmpq $ftrace_graph_entry_stub, ftrace_graph_entry - jnz ftrace_graph_caller -#endif - SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL) - retq + RET trace: /* save_mcount_regs fills in first two parameters */ @@ -315,25 +300,12 @@ trace: CALL_NOSPEC r8 restore_mcount_regs - jmp fgraph_trace + jmp ftrace_stub SYM_FUNC_END(__fentry__) EXPORT_SYMBOL(__fentry__) #endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_FUNCTION_GRAPH_TRACER -SYM_FUNC_START(ftrace_graph_caller) - /* Saves rbp into %rdx and fills first parameter */ - save_mcount_regs - - leaq MCOUNT_REG_SIZE+8(%rsp), %rsi - movq $0, %rdx /* No framepointers needed */ - call prepare_ftrace_return - - restore_mcount_regs - - retq -SYM_FUNC_END(ftrace_graph_caller) - SYM_FUNC_START(return_to_handler) subq $24, %rsp diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index de01903c37..de563db9cd 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include @@ -126,6 +126,36 @@ static bool __head check_la57_support(unsigned long physaddr) } #endif +static unsigned long sme_postprocess_startup(struct boot_params *bp, pmdval_t *pmd) +{ + unsigned long vaddr, vaddr_end; + int i; + + /* Encrypt the kernel and related (if SME is active) */ + sme_encrypt_kernel(bp); + + /* + * Clear the memory encryption mask from the .bss..decrypted section. + * The bss section will be memset to zero later in the initialization so + * there is no need to zero it after changing the memory encryption + * attribute. + */ + if (sme_get_me_mask()) { + vaddr = (unsigned long)__start_bss_decrypted; + vaddr_end = (unsigned long)__end_bss_decrypted; + for (; vaddr < vaddr_end; vaddr += PMD_SIZE) { + i = pmd_index(vaddr); + pmd[i] -= sme_get_me_mask(); + } + } + + /* + * Return the SME encryption mask (if SME is active) to be used as a + * modifier for the initial pgdir entry programmed into CR3. + */ + return sme_get_me_mask(); +} + /* Code in __startup_64() can be relocated during execution, but the compiler * doesn't have to generate PC-relative relocations when accessing globals from * that function. Clang actually does not generate them, which leads to @@ -135,7 +165,6 @@ static bool __head check_la57_support(unsigned long physaddr) unsigned long __head __startup_64(unsigned long physaddr, struct boot_params *bp) { - unsigned long vaddr, vaddr_end; unsigned long load_delta, *p; unsigned long pgtable_flags; pgdval_t *pgd; @@ -276,29 +305,7 @@ unsigned long __head __startup_64(unsigned long physaddr, */ *fixup_long(&phys_base, physaddr) += load_delta - sme_get_me_mask(); - /* Encrypt the kernel and related (if SME is active) */ - sme_encrypt_kernel(bp); - - /* - * Clear the memory encryption mask from the .bss..decrypted section. - * The bss section will be memset to zero later in the initialization so - * there is no need to zero it after changing the memory encryption - * attribute. - */ - if (mem_encrypt_active()) { - vaddr = (unsigned long)__start_bss_decrypted; - vaddr_end = (unsigned long)__end_bss_decrypted; - for (; vaddr < vaddr_end; vaddr += PMD_SIZE) { - i = pmd_index(vaddr); - pmd[i] -= sme_get_me_mask(); - } - } - - /* - * Return the SME encryption mask (if SME is active) to be used as a - * modifier for the initial pgdir entry programmed into CR3. - */ - return sme_get_me_mask(); + return sme_postprocess_startup(bp, pmd); } unsigned long __startup_secondary_64(void) @@ -480,6 +487,10 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data) clear_bss(); + /* + * This needs to happen *before* kasan_early_init() because latter maps stuff + * into that page. + */ clear_page(init_top_pgt); /* @@ -491,6 +502,16 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data) kasan_early_init(); + /* + * Flush global TLB entries which could be left over from the trampoline page + * table. + * + * This needs to happen *after* kasan_early_init() as KASAN-enabled .configs + * instrument native_write_cr4() so KASAN must be initialized for that + * instrumentation to work. + */ + __native_tlb_flush_global(this_cpu_read(cpu_tlbstate.cr4)); + idt_setup_early_handler(); copy_bootdata(__va(real_mode_data)); diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index d8b3ebd2bb..9c63fc5988 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -166,9 +166,26 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) call sev_verify_cbit popq %rsi - /* Switch to new page-table */ + /* + * Switch to new page-table + * + * For the boot CPU this switches to early_top_pgt which still has the + * indentity mappings present. The secondary CPUs will switch to the + * init_top_pgt here, away from the trampoline_pgd and unmap the + * indentity mapped ranges. + */ movq %rax, %cr3 + /* + * Do a global TLB flush after the CR3 switch to make sure the TLB + * entries from the identity mapping are flushed. + */ + movq %cr4, %rcx + movq %rcx, %rax + xorq $X86_CR4_PGE, %rcx + movq %rcx, %cr4 + movq %rax, %cr4 + /* Ensure I am executing from virtual addresses */ movq $1f, %rax ANNOTATE_RETPOLINE_SAFE diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 882213df37..71f336425e 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -1435,8 +1435,12 @@ irqreturn_t hpet_rtc_interrupt(int irq, void *dev_id) hpet_rtc_timer_reinit(); memset(&curr_time, 0, sizeof(struct rtc_time)); - if (hpet_rtc_flags & (RTC_UIE | RTC_AIE)) - mc146818_get_time(&curr_time); + if (hpet_rtc_flags & (RTC_UIE | RTC_AIE)) { + if (unlikely(mc146818_get_time(&curr_time) < 0)) { + pr_err_ratelimited("unable to read current time from RTC\n"); + return IRQ_HANDLED; + } + } if (hpet_rtc_flags & RTC_UIE && curr_time.tm_sec != hpet_prev_update_sec) { diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index 044902d5a3..e5dd6da787 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -132,6 +132,7 @@ int irq_init_percpu_irqstack(unsigned int cpu) return 0; } +#ifndef CONFIG_PREEMPT_RT void do_softirq_own_stack(void) { struct irq_stack *irqstk; @@ -148,6 +149,7 @@ void do_softirq_own_stack(void) call_on_stack(__do_softirq, isp); } +#endif void __handle_irq(struct irq_desc *desc, struct pt_regs *regs) { diff --git a/arch/x86/kernel/irqflags.S b/arch/x86/kernel/irqflags.S index 8ef3506396..aaf9e776f3 100644 --- a/arch/x86/kernel/irqflags.S +++ b/arch/x86/kernel/irqflags.S @@ -7,9 +7,11 @@ /* * unsigned long native_save_fl(void) */ +.pushsection .noinstr.text, "ax" SYM_FUNC_START(native_save_fl) pushf pop %_ASM_AX - ret + RET SYM_FUNC_END(native_save_fl) +.popsection EXPORT_SYMBOL(native_save_fl) diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c index 1afbdd1dd7..9ff480e945 100644 --- a/arch/x86/kernel/itmt.c +++ b/arch/x86/kernel/itmt.c @@ -198,7 +198,7 @@ void sched_set_itmt_core_prio(int prio, int core_cpu) * of the priority chain and only used when * all other high priority cpus are out of capacity. */ - smt_prio = prio * smp_num_siblings / i; + smt_prio = prio * smp_num_siblings / (i * i); per_cpu(sched_core_priority, cpu) = smt_prio; i++; } diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index b6e046e4b2..6290712cb3 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -809,7 +809,7 @@ void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) ri->fp = sara; /* Replace the return addr with trampoline addr */ - *sara = (unsigned long) &kretprobe_trampoline; + *sara = (unsigned long) &__kretprobe_trampoline; } NOKPROBE_SYMBOL(arch_prepare_kretprobe); @@ -1019,52 +1019,91 @@ NOKPROBE_SYMBOL(kprobe_int3_handler); */ asm( ".text\n" - ".global kretprobe_trampoline\n" - ".type kretprobe_trampoline, @function\n" - "kretprobe_trampoline:\n" - /* We don't bother saving the ss register */ + ".global __kretprobe_trampoline\n" + ".type __kretprobe_trampoline, @function\n" + "__kretprobe_trampoline:\n" #ifdef CONFIG_X86_64 + /* Push a fake return address to tell the unwinder it's a kretprobe. */ + " pushq $__kretprobe_trampoline\n" + UNWIND_HINT_FUNC + /* Save the 'sp - 8', this will be fixed later. */ " pushq %rsp\n" " pushfq\n" SAVE_REGS_STRING " movq %rsp, %rdi\n" " call trampoline_handler\n" - /* Replace saved sp with true return address. */ - " movq %rax, 19*8(%rsp)\n" RESTORE_REGS_STRING + /* In trampoline_handler(), 'regs->flags' is copied to 'regs->sp'. */ + " addq $8, %rsp\n" " popfq\n" #else + /* Push a fake return address to tell the unwinder it's a kretprobe. */ + " pushl $__kretprobe_trampoline\n" + UNWIND_HINT_FUNC + /* Save the 'sp - 4', this will be fixed later. */ " pushl %esp\n" " pushfl\n" SAVE_REGS_STRING " movl %esp, %eax\n" " call trampoline_handler\n" - /* Replace saved sp with true return address. */ - " movl %eax, 15*4(%esp)\n" RESTORE_REGS_STRING + /* In trampoline_handler(), 'regs->flags' is copied to 'regs->sp'. */ + " addl $4, %esp\n" " popfl\n" #endif - " ret\n" - ".size kretprobe_trampoline, .-kretprobe_trampoline\n" + ASM_RET + ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n" ); -NOKPROBE_SYMBOL(kretprobe_trampoline); -STACK_FRAME_NON_STANDARD(kretprobe_trampoline); +NOKPROBE_SYMBOL(__kretprobe_trampoline); +/* + * __kretprobe_trampoline() skips updating frame pointer. The frame pointer + * saved in trampoline_handler() points to the real caller function's + * frame pointer. Thus the __kretprobe_trampoline() doesn't have a + * standard stack frame with CONFIG_FRAME_POINTER=y. + * Let's mark it non-standard function. Anyway, FP unwinder can correctly + * unwind without the hint. + */ +STACK_FRAME_NON_STANDARD_FP(__kretprobe_trampoline); +/* This is called from kretprobe_trampoline_handler(). */ +void arch_kretprobe_fixup_return(struct pt_regs *regs, + kprobe_opcode_t *correct_ret_addr) +{ + unsigned long *frame_pointer = ®s->sp + 1; + + /* Replace fake return address with real one. */ + *frame_pointer = (unsigned long)correct_ret_addr; +} /* - * Called from kretprobe_trampoline + * Called from __kretprobe_trampoline */ -__used __visible void *trampoline_handler(struct pt_regs *regs) +__used __visible void trampoline_handler(struct pt_regs *regs) { + unsigned long *frame_pointer; + /* fixup registers */ regs->cs = __KERNEL_CS; #ifdef CONFIG_X86_32 regs->gs = 0; #endif - regs->ip = (unsigned long)&kretprobe_trampoline; + regs->ip = (unsigned long)&__kretprobe_trampoline; regs->orig_ax = ~0UL; + regs->sp += sizeof(long); + frame_pointer = ®s->sp + 1; - return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, ®s->sp); + /* + * The return address at 'frame_pointer' is recovered by the + * arch_kretprobe_fixup_return() which called from the + * kretprobe_trampoline_handler(). + */ + kretprobe_trampoline_handler(regs, frame_pointer); + + /* + * Copy FLAGS to 'pt_regs::sp' so that __kretprobe_trapmoline() + * can do RET right after POPF. + */ + regs->sp = regs->flags; } NOKPROBE_SYMBOL(trampoline_handler); diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c index 596de2f6d3..dd2ec14adb 100644 --- a/arch/x86/kernel/kprobes/ftrace.c +++ b/arch/x86/kernel/kprobes/ftrace.c @@ -25,7 +25,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, if (bit < 0) return; - preempt_disable_notrace(); p = get_kprobe((kprobe_opcode_t *)ip); if (unlikely(!p) || kprobe_disabled(p)) goto out; @@ -59,7 +58,6 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, __this_cpu_write(current_kprobe, NULL); } out: - preempt_enable_notrace(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(kprobe_ftrace_handler); diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 71425ebba9..b4a54a52aa 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -367,10 +367,10 @@ int arch_check_optimized_kprobe(struct optimized_kprobe *op) /* Check the addr is within the optimized instructions. */ int arch_within_optimized_kprobe(struct optimized_kprobe *op, - unsigned long addr) + kprobe_opcode_t *addr) { - return ((unsigned long)op->kp.addr <= addr && - (unsigned long)op->kp.addr + op->optinsn.size > addr); + return (op->kp.addr <= addr && + op->kp.addr + op->optinsn.size > addr); } /* Free optimized instruction slot */ diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 811c7aaf23..d77481ecb0 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include #include #include @@ -40,6 +42,7 @@ #include #include #include +#include DEFINE_STATIC_KEY_FALSE(kvm_async_pf_enabled); @@ -310,7 +313,7 @@ static void kvm_register_steal_time(void) return; wrmsrl(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED)); - pr_info("stealtime: cpu %d, msr %llx\n", cpu, + pr_debug("stealtime: cpu %d, msr %llx\n", cpu, (unsigned long long) slow_virt_to_phys(st)); } @@ -347,7 +350,7 @@ static void kvm_guest_cpu_init(void) wrmsrl(MSR_KVM_ASYNC_PF_EN, pa); __this_cpu_write(apf_reason.enabled, 1); - pr_info("setup async PF for cpu %d\n", smp_processor_id()); + pr_debug("setup async PF for cpu %d\n", smp_processor_id()); } if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) { @@ -373,7 +376,7 @@ static void kvm_pv_disable_apf(void) wrmsrl(MSR_KVM_ASYNC_PF_EN, 0); __this_cpu_write(apf_reason.enabled, 0); - pr_info("disable async PF for cpu %d\n", smp_processor_id()); + pr_debug("disable async PF for cpu %d\n", smp_processor_id()); } static void kvm_disable_steal_time(void) @@ -418,7 +421,7 @@ static void __init sev_map_percpu_data(void) { int cpu; - if (!sev_active()) + if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) return; for_each_possible_cpu(cpu) { @@ -433,6 +436,8 @@ static void kvm_guest_cpu_offline(bool shutdown) kvm_disable_steal_time(); if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) wrmsrl(MSR_KVM_PV_EOI_EN, 0); + if (kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) + wrmsrl(MSR_KVM_MIGRATION_CONTROL, 0); kvm_pv_disable_apf(); if (!shutdown) apf_task_wake_all(); @@ -458,6 +463,7 @@ static bool pv_tlb_flush_supported(void) return (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) && !kvm_para_has_hint(KVM_HINTS_REALTIME) && kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) && + !boot_cpu_has(X86_FEATURE_MWAIT) && (num_possible_cpus() != 1)); } @@ -472,6 +478,7 @@ static bool pv_sched_yield_supported(void) return (kvm_para_has_feature(KVM_FEATURE_PV_SCHED_YIELD) && !kvm_para_has_hint(KVM_HINTS_REALTIME) && kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) && + !boot_cpu_has(X86_FEATURE_MWAIT) && (num_possible_cpus() != 1)); } @@ -550,6 +557,55 @@ static void kvm_send_ipi_mask_allbutself(const struct cpumask *mask, int vector) __send_ipi_mask(local_mask, vector); } +static int __init setup_efi_kvm_sev_migration(void) +{ + efi_char16_t efi_sev_live_migration_enabled[] = L"SevLiveMigrationEnabled"; + efi_guid_t efi_variable_guid = AMD_SEV_MEM_ENCRYPT_GUID; + efi_status_t status; + unsigned long size; + bool enabled; + + if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) || + !kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) + return 0; + + if (!efi_enabled(EFI_BOOT)) + return 0; + + if (!efi_enabled(EFI_RUNTIME_SERVICES)) { + pr_info("%s : EFI runtime services are not enabled\n", __func__); + return 0; + } + + size = sizeof(enabled); + + /* Get variable contents into buffer */ + status = efi.get_variable(efi_sev_live_migration_enabled, + &efi_variable_guid, NULL, &size, &enabled); + + if (status == EFI_NOT_FOUND) { + pr_info("%s : EFI live migration variable not found\n", __func__); + return 0; + } + + if (status != EFI_SUCCESS) { + pr_info("%s : EFI variable retrieval failed\n", __func__); + return 0; + } + + if (enabled == 0) { + pr_info("%s: live migration disabled in EFI\n", __func__); + return 0; + } + + pr_info("%s : live migration enabled in EFI\n", __func__); + wrmsrl(MSR_KVM_MIGRATION_CONTROL, KVM_MIGRATION_READY); + + return 1; +} + +late_initcall(setup_efi_kvm_sev_migration); + /* * Set the IPI entry points */ @@ -568,7 +624,7 @@ static void kvm_smp_send_call_func_ipi(const struct cpumask *mask) /* Make sure other vCPUs get a chance to run if they need to. */ for_each_cpu(cpu, mask) { - if (vcpu_is_preempted(cpu)) { + if (!idle_cpu(cpu) && vcpu_is_preempted(cpu)) { kvm_hypercall1(KVM_HC_SCHED_YIELD, per_cpu(x86_cpu_to_apicid, cpu)); break; } @@ -758,7 +814,7 @@ static noinline uint32_t __kvm_cpuid_base(void) return 0; /* So we don't blow up on old processors */ if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) - return hypervisor_cpuid_base("KVMKVMKVM\0\0\0", 0); + return hypervisor_cpuid_base(KVM_SIGNATURE, 0); return 0; } @@ -808,8 +864,62 @@ static bool __init kvm_msi_ext_dest_id(void) return kvm_para_has_feature(KVM_FEATURE_MSI_EXT_DEST_ID); } +static void kvm_sev_hc_page_enc_status(unsigned long pfn, int npages, bool enc) +{ + kvm_sev_hypercall3(KVM_HC_MAP_GPA_RANGE, pfn << PAGE_SHIFT, npages, + KVM_MAP_GPA_RANGE_ENC_STAT(enc) | KVM_MAP_GPA_RANGE_PAGE_SZ_4K); +} + static void __init kvm_init_platform(void) { + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) && + kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) { + unsigned long nr_pages; + int i; + + pv_ops.mmu.notify_page_enc_status_changed = + kvm_sev_hc_page_enc_status; + + /* + * Reset the host's shared pages list related to kernel + * specific page encryption status settings before we load a + * new kernel by kexec. Reset the page encryption status + * during early boot intead of just before kexec to avoid SMP + * races during kvm_pv_guest_cpu_reboot(). + * NOTE: We cannot reset the complete shared pages list + * here as we need to retain the UEFI/OVMF firmware + * specific settings. + */ + + for (i = 0; i < e820_table->nr_entries; i++) { + struct e820_entry *entry = &e820_table->entries[i]; + + if (entry->type != E820_TYPE_RAM) + continue; + + nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE); + + kvm_sev_hypercall3(KVM_HC_MAP_GPA_RANGE, entry->addr, + nr_pages, + KVM_MAP_GPA_RANGE_ENCRYPTED | KVM_MAP_GPA_RANGE_PAGE_SZ_4K); + } + + /* + * Ensure that _bss_decrypted section is marked as decrypted in the + * shared pages list. + */ + nr_pages = DIV_ROUND_UP(__end_bss_decrypted - __start_bss_decrypted, + PAGE_SIZE); + early_set_mem_enc_dec_hypercall((unsigned long)__start_bss_decrypted, + nr_pages, 0); + + /* + * If not booted using EFI, enable Live migration support. + */ + if (!efi_enabled(EFI_BOOT)) + wrmsrl(MSR_KVM_MIGRATION_CONTROL, + KVM_MIGRATION_READY); + } kvmclock_init(); x86_platform.apic_post_init = kvm_apic_init; } diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 73c74b961d..c5caa7311b 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -16,9 +16,9 @@ #include #include #include +#include #include -#include #include #include @@ -174,7 +174,7 @@ static void kvm_register_clock(char *txt) pa = slow_virt_to_phys(&src->pvti) | 0x01ULL; wrmsrl(msr_kvm_system_time, pa); - pr_info("kvm-clock: cpu %d, msr %llx, %s", smp_processor_id(), pa, txt); + pr_debug("kvm-clock: cpu %d, msr %llx, %s", smp_processor_id(), pa, txt); } static void kvm_save_sched_clock_state(void) @@ -223,7 +223,7 @@ static void __init kvmclock_init_mem(void) * hvclock is shared between the guest and the hypervisor, must * be mapped decrypted. */ - if (sev_active()) { + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { r = set_memory_decrypted((unsigned long) hvclock_mem, 1UL << order); if (r) { @@ -239,6 +239,9 @@ static void __init kvmclock_init_mem(void) static int __init kvm_setup_vsyscall_timeinfo(void) { + if (!kvm_para_available() || !kvmclock) + return 0; + kvmclock_init_mem(); #ifdef CONFIG_X86_64 diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c index 131f30fdcf..f5da4a1807 100644 --- a/arch/x86/kernel/machine_kexec_64.c +++ b/arch/x86/kernel/machine_kexec_64.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -166,7 +167,7 @@ static int init_transition_pgtable(struct kimage *image, pgd_t *pgd) } pte = pte_offset_kernel(pmd, vaddr); - if (sev_active()) + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) prot = PAGE_KERNEL_EXEC; set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot)); @@ -206,7 +207,7 @@ static int init_pgtable(struct kimage *image, unsigned long start_pgtable) level4p = (pgd_t *)__va(start_pgtable); clear_page(level4p); - if (sev_active()) { + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { info.page_flag |= _PAGE_ENC; info.kernpg_flag |= _PAGE_ENC; } @@ -358,7 +359,7 @@ void machine_kexec(struct kimage *image) (unsigned long)page_list, image->start, image->preserve_context, - sme_active()); + cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)); #ifdef CONFIG_KEXEC_JUMP if (image->preserve_context) @@ -569,12 +570,12 @@ void arch_kexec_unprotect_crashkres(void) */ int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp) { - if (sev_active()) + if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) return 0; /* - * If SME is active we need to be sure that kexec pages are - * not encrypted because when we boot to the new kernel the + * If host memory encryption is active we need to be sure that kexec + * pages are not encrypted because when we boot to the new kernel the * pages won't be accessed encrypted (initially). */ return set_memory_decrypted((unsigned long)vaddr, pages); @@ -582,12 +583,12 @@ int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp) void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages) { - if (sev_active()) + if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) return; /* - * If SME is active we need to reset the pages back to being - * an encrypted mapping before freeing them. + * If host memory encryption is active we need to reset the pages back + * to being an encrypted mapping before freeing them. */ set_memory_encrypted((unsigned long)vaddr, pages); } diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index 32e546e416..96d7c27b70 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -252,7 +252,8 @@ int module_finalize(const Elf_Ehdr *hdr, struct module *me) { const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL, - *para = NULL, *orc = NULL, *orc_ip = NULL; + *para = NULL, *orc = NULL, *orc_ip = NULL, + *retpolines = NULL; char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { @@ -268,6 +269,8 @@ int module_finalize(const Elf_Ehdr *hdr, orc = s; if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name)) orc_ip = s; + if (!strcmp(".retpoline_sites", secstrings + s->sh_name)) + retpolines = s; } /* @@ -278,6 +281,10 @@ int module_finalize(const Elf_Ehdr *hdr, void *pseg = (void *)para->sh_addr; apply_paravirt(pseg, pseg + para->sh_size); } + if (retpolines) { + void *rseg = (void *)retpolines->sh_addr; + apply_retpolines(rseg, rseg + retpolines->sh_size); + } if (alt) { /* patch .altinstructions */ void *aseg = (void *)alt->sh_addr; diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 04cafc057b..4420499f7b 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -41,11 +41,22 @@ extern void _paravirt_nop(void); asm (".pushsection .entry.text, \"ax\"\n" ".global _paravirt_nop\n" "_paravirt_nop:\n\t" - "ret\n\t" + ASM_RET ".size _paravirt_nop, . - _paravirt_nop\n\t" ".type _paravirt_nop, @function\n\t" ".popsection"); +/* stub always returning 0. */ +asm (".pushsection .entry.text, \"ax\"\n" + ".global paravirt_ret0\n" + "paravirt_ret0:\n\t" + "xor %" _ASM_AX ", %" _ASM_AX ";\n\t" + ASM_RET + ".size paravirt_ret0, . - paravirt_ret0\n\t" + ".type paravirt_ret0, @function\n\t" + ".popsection"); + + void __init default_banner(void) { printk(KERN_INFO "Booting paravirtualized kernel on %s\n", @@ -53,7 +64,7 @@ void __init default_banner(void) } /* Undefined instruction for dealing with missing ops pointers. */ -static void paravirt_BUG(void) +noinstr void paravirt_BUG(void) { BUG(); } @@ -218,6 +229,36 @@ void paravirt_end_context_switch(struct task_struct *next) if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES)) arch_enter_lazy_mmu_mode(); } + +static noinstr unsigned long pv_native_read_cr2(void) +{ + return native_read_cr2(); +} + +static noinstr void pv_native_write_cr2(unsigned long val) +{ + native_write_cr2(val); +} + +static noinstr unsigned long pv_native_get_debugreg(int regno) +{ + return native_get_debugreg(regno); +} + +static noinstr void pv_native_set_debugreg(int regno, unsigned long val) +{ + native_set_debugreg(regno, val); +} + +static noinstr void pv_native_irq_enable(void) +{ + native_irq_enable(); +} + +static noinstr void pv_native_irq_disable(void) +{ + native_irq_disable(); +} #endif enum paravirt_lazy_mode paravirt_get_lazy_mode(void) @@ -244,8 +285,8 @@ struct paravirt_patch_template pv_ops = { #ifdef CONFIG_PARAVIRT_XXL .cpu.cpuid = native_cpuid, - .cpu.get_debugreg = native_get_debugreg, - .cpu.set_debugreg = native_set_debugreg, + .cpu.get_debugreg = pv_native_get_debugreg, + .cpu.set_debugreg = pv_native_set_debugreg, .cpu.read_cr0 = native_read_cr0, .cpu.write_cr0 = native_write_cr0, .cpu.write_cr4 = native_write_cr4, @@ -281,8 +322,8 @@ struct paravirt_patch_template pv_ops = { /* Irq ops. */ .irq.save_fl = __PV_IS_CALLEE_SAVE(native_save_fl), - .irq.irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable), - .irq.irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable), + .irq.irq_disable = __PV_IS_CALLEE_SAVE(pv_native_irq_disable), + .irq.irq_enable = __PV_IS_CALLEE_SAVE(pv_native_irq_enable), .irq.safe_halt = native_safe_halt, .irq.halt = native_halt, #endif /* CONFIG_PARAVIRT_XXL */ @@ -296,10 +337,11 @@ struct paravirt_patch_template pv_ops = { (void (*)(struct mmu_gather *, void *))tlb_remove_page, .mmu.exit_mmap = paravirt_nop, + .mmu.notify_page_enc_status_changed = paravirt_nop, #ifdef CONFIG_PARAVIRT_XXL - .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(native_read_cr2), - .mmu.write_cr2 = native_write_cr2, + .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(pv_native_read_cr2), + .mmu.write_cr2 = pv_native_write_cr2, .mmu.read_cr3 = __native_read_cr3, .mmu.write_cr3 = native_write_cr3, @@ -371,9 +413,6 @@ struct paravirt_patch_template pv_ops = { }; #ifdef CONFIG_PARAVIRT_XXL -/* At this point, native_get/set_debugreg has real function entries */ -NOKPROBE_SYMBOL(native_get_debugreg); -NOKPROBE_SYMBOL(native_set_debugreg); NOKPROBE_SYMBOL(native_load_idt); void (*paravirt_iret)(void) = native_iret; diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c index c2cfa5e7c1..814ab46a0d 100644 --- a/arch/x86/kernel/pci-swiotlb.c +++ b/arch/x86/kernel/pci-swiotlb.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include @@ -45,11 +45,10 @@ int __init pci_swiotlb_detect_4gb(void) swiotlb = 1; /* - * If SME is active then swiotlb will be set to 1 so that bounce - * buffers are allocated and used for devices that do not support - * the addressing range required for the encryption mask. + * Set swiotlb to 1 so that bounce buffers are allocated and used for + * devices that can't support DMA to encrypted memory. */ - if (sme_active()) + if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) swiotlb = 1; return swiotlb; diff --git a/arch/x86/kernel/probe_roms.c b/arch/x86/kernel/probe_roms.c index 9e1def3744..36e84d9042 100644 --- a/arch/x86/kernel/probe_roms.c +++ b/arch/x86/kernel/probe_roms.c @@ -80,7 +80,7 @@ static struct resource video_rom_resource = { */ static bool match_id(struct pci_dev *pdev, unsigned short vendor, unsigned short device) { - struct pci_driver *drv = pdev->driver; + struct pci_driver *drv = to_pci_driver(pdev->dev.driver); const struct pci_device_id *id; if (pdev->vendor == vendor && pdev->device == device) diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index f2f733bcb2..81d8ef0366 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -30,7 +30,9 @@ #include #include #include -#include +#include +#include +#include #include #include #include @@ -43,6 +45,7 @@ #include #include #include +#include #include "process.h" @@ -87,9 +90,20 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) #ifdef CONFIG_VM86 dst->thread.vm86 = NULL; #endif - return fpu_clone(dst); + /* Drop the copied pointer to current's fpstate */ + dst->thread.fpu.fpstate = NULL; + + return 0; } +#ifdef CONFIG_X86_64 +void arch_release_task_struct(struct task_struct *tsk) +{ + if (fpu_state_size_dynamic()) + fpstate_free(&tsk->thread.fpu); +} +#endif + /* * Free thread data structures etc.. */ @@ -155,6 +169,8 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, frame->flags = X86_EFLAGS_FIXED; #endif + fpu_clone(p, clone_flags); + /* Kernel thread ? */ if (unlikely(p->flags & PF_KTHREAD)) { p->thread.pkru = pkru_get_init_value(); @@ -349,7 +365,7 @@ void arch_setup_new_exec(void) clear_thread_flag(TIF_SSBD); task_clear_spec_ssb_disable(current); task_clear_spec_ssb_noexec(current); - speculation_ctrl_update(task_thread_info(current)->flags); + speculation_ctrl_update(read_thread_flags()); } } @@ -601,7 +617,7 @@ static unsigned long speculation_ctrl_update_tif(struct task_struct *tsk) clear_tsk_thread_flag(tsk, TIF_SPEC_IB); } /* Return the updated threadinfo flags*/ - return task_thread_info(tsk)->flags; + return read_task_thread_flags(tsk); } void speculation_ctrl_update(unsigned long tif) @@ -637,8 +653,8 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p) { unsigned long tifp, tifn; - tifn = READ_ONCE(task_thread_info(next_p)->flags); - tifp = READ_ONCE(task_thread_info(prev_p)->flags); + tifn = read_task_thread_flags(next_p); + tifp = read_task_thread_flags(prev_p); switch_to_bitmap(tifp); @@ -943,70 +959,43 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) * because the task might wake up and we might look at a stack * changing under us. */ -unsigned long get_wchan(struct task_struct *p) +unsigned long __get_wchan(struct task_struct *p) { - unsigned long start, bottom, top, sp, fp, ip, ret = 0; - int count = 0; - - if (p == current || task_is_running(p)) - return 0; + struct unwind_state state; + unsigned long addr = 0; if (!try_get_task_stack(p)) return 0; - start = (unsigned long)task_stack_page(p); - if (!start) - goto out; + for (unwind_start(&state, p, NULL, NULL); !unwind_done(&state); + unwind_next_frame(&state)) { + addr = unwind_get_return_address(&state); + if (!addr) + break; + if (in_sched_functions(addr)) + continue; + break; + } - /* - * Layout of the stack page: - * - * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long) - * PADDING - * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING - * stack - * ----------- bottom = start - * - * The tasks stack pointer points at the location where the - * framepointer is stored. The data on the stack is: - * ... IP FP ... IP FP - * - * We need to read FP and IP, so we need to adjust the upper - * bound by another unsigned long. - */ - top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING; - top -= 2 * sizeof(unsigned long); - bottom = start; - - sp = READ_ONCE(p->thread.sp); - if (sp < bottom || sp > top) - goto out; - - fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp); - do { - if (fp < bottom || fp > top) - goto out; - ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long))); - if (!in_sched_functions(ip)) { - ret = ip; - goto out; - } - fp = READ_ONCE_NOCHECK(*(unsigned long *)fp); - } while (count++ < 16 && !task_is_running(p)); - -out: put_task_stack(p); - return ret; + + return addr; } long do_arch_prctl_common(struct task_struct *task, int option, - unsigned long cpuid_enabled) + unsigned long arg2) { switch (option) { case ARCH_GET_CPUID: return get_cpuid_mode(); case ARCH_SET_CPUID: - return set_cpuid_mode(task, cpuid_enabled); + return set_cpuid_mode(task, arg2); + case ARCH_GET_XCOMP_SUPP: + case ARCH_GET_XCOMP_PERM: + case ARCH_REQ_XCOMP_PERM: + case ARCH_GET_XCOMP_GUEST_PERM: + case ARCH_REQ_XCOMP_GUEST_PERM: + return fpu_xstate_prctl(task, option, arg2); } return -EINVAL; diff --git a/arch/x86/kernel/process.h b/arch/x86/kernel/process.h index 1d0797b233..76b547b832 100644 --- a/arch/x86/kernel/process.h +++ b/arch/x86/kernel/process.h @@ -13,8 +13,8 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p); static inline void switch_to_extra(struct task_struct *prev, struct task_struct *next) { - unsigned long next_tif = task_thread_info(next)->flags; - unsigned long prev_tif = task_thread_info(prev)->flags; + unsigned long next_tif = read_task_thread_flags(next); + unsigned long prev_tif = read_task_thread_flags(prev); if (IS_ENABLED(CONFIG_SMP)) { /* diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index 4f2f54e128..26edb1cd07 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -41,7 +41,7 @@ #include #include -#include +#include #include #include @@ -160,7 +160,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) struct thread_struct *prev = &prev_p->thread, *next = &next_p->thread; struct fpu *prev_fpu = &prev->fpu; - struct fpu *next_fpu = &next->fpu; int cpu = smp_processor_id(); /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */ @@ -213,7 +212,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) this_cpu_write(current_task, next_p); - switch_fpu_finish(next_fpu); + switch_fpu_finish(); /* Load the Intel cache allocation PQR MSR. */ resctrl_sched_in(); diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index ec0d836a13..3402edec23 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -42,7 +42,7 @@ #include #include -#include +#include #include #include #include @@ -559,7 +559,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) struct thread_struct *prev = &prev_p->thread; struct thread_struct *next = &next_p->thread; struct fpu *prev_fpu = &prev->fpu; - struct fpu *next_fpu = &next->fpu; int cpu = smp_processor_id(); WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) && @@ -620,7 +619,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) this_cpu_write(current_task, next_p); this_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p)); - switch_fpu_finish(next_fpu); + switch_fpu_finish(); /* Reload sp0. */ update_task_stack(next_p); diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index 033d9c6a94..8d2f2f9955 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c @@ -29,9 +29,9 @@ #include #include -#include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index c53271aebb..399f075ccd 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -47,7 +47,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) * %rsi page_list * %rdx start address * %rcx preserve_context - * %r8 sme_active + * %r8 host_mem_enc_active */ /* Save the CPU context, used for jumping back */ @@ -104,7 +104,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) /* jump to identity mapped page */ addq $(identity_mapped - relocate_kernel), %r8 pushq %r8 - ret + RET SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) @@ -191,7 +191,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) xorl %r14d, %r14d xorl %r15d, %r15d - ret + RET 1: popq %rdx @@ -210,7 +210,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) call swap_pages movq $virtual_mapped, %rax pushq %rax - ret + RET SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) @@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) popq %r12 popq %rbp popq %rbx - ret + RET SYM_CODE_END(virtual_mapped) /* Do the copies */ @@ -288,7 +288,7 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) lea PAGE_SIZE(%rax), %rsi jmp 0b 3: - ret + RET SYM_CODE_END(swap_pages) .globl kexec_control_code_size diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 8e56c4de00..90d7e1788c 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -322,7 +323,7 @@ static void __init reserve_initrd(void) relocate_initrd(); - memblock_free(ramdisk_image, ramdisk_end - ramdisk_image); + memblock_phys_free(ramdisk_image, ramdisk_end - ramdisk_image); } #else @@ -541,7 +542,7 @@ static void __init reserve_crashkernel(void) } if (crash_base >= (1ULL << 32) && reserve_crashkernel_low()) { - memblock_free(crash_base, crash_size); + memblock_phys_free(crash_base, crash_size); return; } @@ -987,7 +988,11 @@ void __init setup_arch(char **cmdline_p) max_pfn = e820__end_of_ram_pfn(); /* update e820 for memory not covered by WB MTRRs */ - mtrr_bp_init(); + if (IS_ENABLED(CONFIG_MTRR)) + mtrr_bp_init(); + else + pat_disable("PAT support disabled because CONFIG_MTRR is disabled in the kernel."); + if (mtrr_trim_uncached_memory(max_pfn)) max_pfn = e820__end_of_ram_pfn(); diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index 5afd985591..49325caa73 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -84,60 +84,6 @@ static bool __init pcpu_need_numa(void) } #endif -/** - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu - * @cpu: cpu to allocate for - * @size: size allocation in bytes - * @align: alignment - * - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper - * does the right thing for NUMA regardless of the current - * configuration. - * - * RETURNS: - * Pointer to the allocated area on success, NULL on failure. - */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, - unsigned long align) -{ - const unsigned long goal = __pa(MAX_DMA_ADDRESS); -#ifdef CONFIG_NUMA - int node = early_cpu_to_node(cpu); - void *ptr; - - if (!node_online(node) || !NODE_DATA(node)) { - ptr = memblock_alloc_from(size, align, goal); - pr_info("cpu %d has no node %d or node-local memory\n", - cpu, node); - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", - cpu, size, __pa(ptr)); - } else { - ptr = memblock_alloc_try_nid(size, align, goal, - MEMBLOCK_ALLOC_ACCESSIBLE, - node); - - pr_debug("per cpu data for cpu%d %lu bytes on node%d at %016lx\n", - cpu, size, node, __pa(ptr)); - } - return ptr; -#else - return memblock_alloc_from(size, align, goal); -#endif -} - -/* - * Helpers for first chunk memory allocation - */ -static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align) -{ - return pcpu_alloc_bootmem(cpu, size, align); -} - -static void __init pcpu_fc_free(void *ptr, size_t size) -{ - memblock_free_ptr(ptr, size); -} - static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) { #ifdef CONFIG_NUMA @@ -150,7 +96,12 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) #endif } -static void __init pcpup_populate_pte(unsigned long addr) +static int __init pcpu_cpu_to_node(int cpu) +{ + return early_cpu_to_node(cpu); +} + +void __init pcpu_populate_pte(unsigned long addr) { populate_extra_pte(addr); } @@ -205,15 +156,14 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, dyn_size, atom_size, pcpu_cpu_distance, - pcpu_fc_alloc, pcpu_fc_free); + pcpu_cpu_to_node); if (rc < 0) pr_warn("%s allocator failed (%d), falling back to page size\n", pcpu_fc_names[pcpu_chosen_fc], rc); } if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, - pcpu_fc_alloc, pcpu_fc_free, - pcpup_populate_pte); + pcpu_cpu_to_node); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/arch/x86/kernel/sev-shared.c b/arch/x86/kernel/sev-shared.c index bf1033a62e..ce987688bb 100644 --- a/arch/x86/kernel/sev-shared.c +++ b/arch/x86/kernel/sev-shared.c @@ -64,7 +64,7 @@ static bool sev_es_negotiate_protocol(void) static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) { ghcb->save.sw_exit_code = 0; - memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); + __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); } static bool vc_decoding_needed(unsigned long exit_code) @@ -94,25 +94,15 @@ static void vc_finish_insn(struct es_em_ctxt *ctxt) ctxt->regs->ip += ctxt->insn.length; } -static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, - struct es_em_ctxt *ctxt, - u64 exit_code, u64 exit_info_1, - u64 exit_info_2) +static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { - enum es_result ret; + u32 ret; - /* Fill in protocol and format specifiers */ - ghcb->protocol_version = GHCB_PROTOCOL_MAX; - ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; + ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); + if (!ret) + return ES_OK; - ghcb_set_sw_exit_code(ghcb, exit_code); - ghcb_set_sw_exit_info_1(ghcb, exit_info_1); - ghcb_set_sw_exit_info_2(ghcb, exit_info_2); - - sev_es_wr_ghcb_msr(__pa(ghcb)); - VMGEXIT(); - - if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) { + if (ret == 1) { u64 info = ghcb->save.sw_exit_info_2; unsigned long v; @@ -124,19 +114,40 @@ static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { ctxt->fi.vector = v; + if (info & SVM_EVTINJ_VALID_ERR) ctxt->fi.error_code = info >> 32; - ret = ES_EXCEPTION; - } else { - ret = ES_VMM_ERROR; + + return ES_EXCEPTION; } - } else if (ghcb->save.sw_exit_info_1 & 0xffffffff) { - ret = ES_VMM_ERROR; - } else { - ret = ES_OK; } - return ret; + return ES_VMM_ERROR; +} + +enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, bool set_ghcb_msr, + struct es_em_ctxt *ctxt, u64 exit_code, + u64 exit_info_1, u64 exit_info_2) +{ + /* Fill in protocol and format specifiers */ + ghcb->protocol_version = GHCB_PROTOCOL_MAX; + ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; + + ghcb_set_sw_exit_code(ghcb, exit_code); + ghcb_set_sw_exit_info_1(ghcb, exit_info_1); + ghcb_set_sw_exit_info_2(ghcb, exit_info_2); + + /* + * Hyper-V unenlightened guests use a paravisor for communicating and + * GHCB pages are being allocated and set up by that paravisor. Linux + * should not change the GHCB page's physical address. + */ + if (set_ghcb_msr) + sev_es_wr_ghcb_msr(__pa(ghcb)); + + VMGEXIT(); + + return verify_exception_info(ghcb, ctxt); } /* @@ -210,7 +221,7 @@ void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) fail: /* Terminate the guest */ - sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + sev_es_terminate(GHCB_SEV_ES_GEN_REQ); } static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, @@ -413,7 +424,7 @@ static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) */ sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); ghcb_set_sw_scratch(ghcb, sw_scratch); - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, + ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_IOIO, exit_info_1, exit_info_2); if (ret != ES_OK) return ret; @@ -455,7 +466,8 @@ static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) ghcb_set_rax(ghcb, rax); - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); + ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, + SVM_EXIT_IOIO, exit_info_1, 0); if (ret != ES_OK) return ret; @@ -486,7 +498,7 @@ static enum es_result vc_handle_cpuid(struct ghcb *ghcb, /* xgetbv will cause #GP - use reset value for xcr0 */ ghcb_set_xcr0(ghcb, 1); - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); + ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_CPUID, 0, 0); if (ret != ES_OK) return ret; @@ -511,7 +523,7 @@ static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); enum es_result ret; - ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); + ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, exit_code, 0, 0); if (ret != ES_OK) return ret; diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c index a0064cf77e..e6d316a01f 100644 --- a/arch/x86/kernel/sev.c +++ b/arch/x86/kernel/sev.c @@ -11,7 +11,7 @@ #include /* For show_regs() */ #include -#include +#include #include #include #include @@ -23,9 +23,10 @@ #include #include #include -#include +#include #include #include +#include #include #include #include @@ -86,9 +87,6 @@ struct ghcb_state { static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); DEFINE_STATIC_KEY_FALSE(sev_es_enable_key); -/* Needed in vc_early_forward_exception */ -void do_early_exception(struct pt_regs *regs, int trapnr); - static __always_inline bool on_vc_stack(struct pt_regs *regs) { unsigned long sp = regs->sp; @@ -209,9 +207,6 @@ static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) return ghcb; } -/* Needed in vc_early_forward_exception */ -void do_early_exception(struct pt_regs *regs, int trapnr); - static inline u64 sev_es_rd_ghcb_msr(void) { return __rdmsr(MSR_AMD64_SEV_ES_GHCB); @@ -605,7 +600,7 @@ int __init sev_es_efi_map_ghcbs(pgd_t *pgd) int cpu; u64 pfn; - if (!sev_es_active()) + if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) return 0; pflags = _PAGE_NX | _PAGE_RW; @@ -638,7 +633,8 @@ static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) ghcb_set_rdx(ghcb, regs->dx); } - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0); + ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_MSR, + exit_info_1, 0); if ((ret == ES_OK) && (!exit_info_1)) { regs->ax = ghcb->save.rax; @@ -764,7 +760,7 @@ void __init sev_es_init_vc_handling(void) BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE); - if (!sev_es_active()) + if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) return; if (!sev_es_check_cpu_features()) @@ -796,22 +792,6 @@ static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) do_early_exception(ctxt->regs, trapnr); } -static long *vc_insn_get_reg(struct es_em_ctxt *ctxt) -{ - long *reg_array; - int offset; - - reg_array = (long *)ctxt->regs; - offset = insn_get_modrm_reg_off(&ctxt->insn, ctxt->regs); - - if (offset < 0) - return NULL; - - offset /= sizeof(long); - - return reg_array + offset; -} - static long *vc_insn_get_rm(struct es_em_ctxt *ctxt) { long *reg_array; @@ -856,77 +836,7 @@ static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer)); - return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); -} - -static enum es_result vc_handle_mmio_twobyte_ops(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) -{ - struct insn *insn = &ctxt->insn; - unsigned int bytes = 0; - enum es_result ret; - int sign_byte; - long *reg_data; - - switch (insn->opcode.bytes[1]) { - /* MMIO Read w/ zero-extension */ - case 0xb6: - bytes = 1; - fallthrough; - case 0xb7: - if (!bytes) - bytes = 2; - - ret = vc_do_mmio(ghcb, ctxt, bytes, true); - if (ret) - break; - - /* Zero extend based on operand size */ - reg_data = vc_insn_get_reg(ctxt); - if (!reg_data) - return ES_DECODE_FAILED; - - memset(reg_data, 0, insn->opnd_bytes); - - memcpy(reg_data, ghcb->shared_buffer, bytes); - break; - - /* MMIO Read w/ sign-extension */ - case 0xbe: - bytes = 1; - fallthrough; - case 0xbf: - if (!bytes) - bytes = 2; - - ret = vc_do_mmio(ghcb, ctxt, bytes, true); - if (ret) - break; - - /* Sign extend based on operand size */ - reg_data = vc_insn_get_reg(ctxt); - if (!reg_data) - return ES_DECODE_FAILED; - - if (bytes == 1) { - u8 *val = (u8 *)ghcb->shared_buffer; - - sign_byte = (*val & 0x80) ? 0xff : 0x00; - } else { - u16 *val = (u16 *)ghcb->shared_buffer; - - sign_byte = (*val & 0x8000) ? 0xff : 0x00; - } - memset(reg_data, sign_byte, insn->opnd_bytes); - - memcpy(reg_data, ghcb->shared_buffer, bytes); - break; - - default: - ret = ES_UNSUPPORTED; - } - - return ret; + return sev_es_ghcb_hv_call(ghcb, true, ctxt, exit_code, exit_info_1, exit_info_2); } /* @@ -996,83 +906,79 @@ static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt, return ES_RETRY; } -static enum es_result vc_handle_mmio(struct ghcb *ghcb, - struct es_em_ctxt *ctxt) +static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { struct insn *insn = &ctxt->insn; unsigned int bytes = 0; + enum mmio_type mmio; enum es_result ret; + u8 sign_byte; long *reg_data; - switch (insn->opcode.bytes[0]) { - /* MMIO Write */ - case 0x88: - bytes = 1; - fallthrough; - case 0x89: - if (!bytes) - bytes = insn->opnd_bytes; + mmio = insn_decode_mmio(insn, &bytes); + if (mmio == MMIO_DECODE_FAILED) + return ES_DECODE_FAILED; - reg_data = vc_insn_get_reg(ctxt); + if (mmio != MMIO_WRITE_IMM && mmio != MMIO_MOVS) { + reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs); if (!reg_data) return ES_DECODE_FAILED; + } + switch (mmio) { + case MMIO_WRITE: memcpy(ghcb->shared_buffer, reg_data, bytes); - ret = vc_do_mmio(ghcb, ctxt, bytes, false); break; - - case 0xc6: - bytes = 1; - fallthrough; - case 0xc7: - if (!bytes) - bytes = insn->opnd_bytes; - + case MMIO_WRITE_IMM: memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes); - ret = vc_do_mmio(ghcb, ctxt, bytes, false); break; - - /* MMIO Read */ - case 0x8a: - bytes = 1; - fallthrough; - case 0x8b: - if (!bytes) - bytes = insn->opnd_bytes; - + case MMIO_READ: ret = vc_do_mmio(ghcb, ctxt, bytes, true); if (ret) break; - reg_data = vc_insn_get_reg(ctxt); - if (!reg_data) - return ES_DECODE_FAILED; - /* Zero-extend for 32-bit operation */ if (bytes == 4) *reg_data = 0; memcpy(reg_data, ghcb->shared_buffer, bytes); break; + case MMIO_READ_ZERO_EXTEND: + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; - /* MOVS instruction */ - case 0xa4: - bytes = 1; - fallthrough; - case 0xa5: - if (!bytes) - bytes = insn->opnd_bytes; - - ret = vc_handle_mmio_movs(ctxt, bytes); + /* Zero extend based on operand size */ + memset(reg_data, 0, insn->opnd_bytes); + memcpy(reg_data, ghcb->shared_buffer, bytes); break; - /* Two-Byte Opcodes */ - case 0x0f: - ret = vc_handle_mmio_twobyte_ops(ghcb, ctxt); + case MMIO_READ_SIGN_EXTEND: + ret = vc_do_mmio(ghcb, ctxt, bytes, true); + if (ret) + break; + + if (bytes == 1) { + u8 *val = (u8 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x80) ? 0xff : 0x00; + } else { + u16 *val = (u16 *)ghcb->shared_buffer; + + sign_byte = (*val & 0x8000) ? 0xff : 0x00; + } + + /* Sign extend based on operand size */ + memset(reg_data, sign_byte, insn->opnd_bytes); + memcpy(reg_data, ghcb->shared_buffer, bytes); + break; + case MMIO_MOVS: + ret = vc_handle_mmio_movs(ctxt, bytes); break; default: ret = ES_UNSUPPORTED; + break; } return ret; @@ -1106,7 +1012,7 @@ static enum es_result vc_handle_dr7_write(struct ghcb *ghcb, /* Using a value of 0 for ExitInfo1 means RAX holds the value */ ghcb_set_rax(ghcb, val); - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); + ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); if (ret != ES_OK) return ret; @@ -1136,7 +1042,7 @@ static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, struct es_em_ctxt *ctxt) { - return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); + return sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_WBINVD, 0, 0); } static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt) @@ -1145,7 +1051,7 @@ static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt ghcb_set_rcx(ghcb, ctxt->regs->cx); - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0); + ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_RDPMC, 0, 0); if (ret != ES_OK) return ret; @@ -1186,7 +1092,7 @@ static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, if (x86_platform.hyper.sev_es_hcall_prepare) x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs); - ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); + ret = sev_es_ghcb_hv_call(ghcb, true, ctxt, SVM_EXIT_VMMCALL, 0, 0); if (ret != ES_OK) return ret; @@ -1308,13 +1214,26 @@ static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) } } -static __always_inline bool on_vc_fallback_stack(struct pt_regs *regs) +static __always_inline bool is_vc2_stack(unsigned long sp) { - unsigned long sp = (unsigned long)regs; - return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2)); } +static __always_inline bool vc_from_invalid_context(struct pt_regs *regs) +{ + unsigned long sp, prev_sp; + + sp = (unsigned long)regs; + prev_sp = regs->sp; + + /* + * If the code was already executing on the VC2 stack when the #VC + * happened, let it proceed to the normal handling routine. This way the + * code executing on the VC2 stack can cause #VC exceptions to get handled. + */ + return is_vc2_stack(sp) && !is_vc2_stack(prev_sp); +} + static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code) { struct ghcb_state state; @@ -1395,7 +1314,7 @@ DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) * But keep this here in case the noinstr annotations are violated due * to bug elsewhere. */ - if (unlikely(on_vc_fallback_stack(regs))) { + if (unlikely(vc_from_invalid_context(regs))) { instrumentation_begin(); panic("Can't handle #VC exception from unsupported context\n"); instrumentation_end(); @@ -1418,7 +1337,7 @@ DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) show_regs(regs); /* Ask hypervisor to sev_es_terminate */ - sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + sev_es_terminate(GHCB_SEV_ES_GEN_REQ); /* If that fails and we get here - just panic */ panic("Returned from Terminate-Request to Hypervisor\n"); @@ -1466,7 +1385,7 @@ bool __init handle_vc_boot_ghcb(struct pt_regs *regs) /* Do initial setup or terminate the guest */ if (unlikely(boot_ghcb == NULL && !sev_es_setup_ghcb())) - sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST); + sev_es_terminate(GHCB_SEV_ES_GEN_REQ); vc_ghcb_invalidate(boot_ghcb); diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index f4d21e4700..ec71e06ae3 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -30,8 +31,8 @@ #include #include -#include #include +#include #include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include #endif /* CONFIG_X86_64 */ #include @@ -79,9 +81,9 @@ static void force_valid_ss(struct pt_regs *regs) # define CONTEXT_COPY_SIZE sizeof(struct sigcontext) #endif -static int restore_sigcontext(struct pt_regs *regs, - struct sigcontext __user *usc, - unsigned long uc_flags) +static bool restore_sigcontext(struct pt_regs *regs, + struct sigcontext __user *usc, + unsigned long uc_flags) { struct sigcontext sc; @@ -89,7 +91,7 @@ static int restore_sigcontext(struct pt_regs *regs, current->restart_block.fn = do_no_restart_syscall; if (copy_from_user(&sc, usc, CONTEXT_COPY_SIZE)) - return -EFAULT; + return false; #ifdef CONFIG_X86_32 set_user_gs(regs, sc.gs); @@ -244,7 +246,6 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size, unsigned long math_size = 0; unsigned long sp = regs->sp; unsigned long buf_fx = 0; - int ret; /* redzone */ if (IS_ENABLED(CONFIG_X86_64)) @@ -292,8 +293,7 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size, } /* save i387 and extended state */ - ret = copy_fpstate_to_sigframe(*fpstate, (void __user *)buf_fx, math_size); - if (ret < 0) + if (!copy_fpstate_to_sigframe(*fpstate, (void __user *)buf_fx, math_size)) return (void __user *)-1L; return (void __user *)sp; @@ -643,7 +643,7 @@ SYSCALL_DEFINE0(sigreturn) * x86_32 has no uc_flags bits relevant to restore_sigcontext. * Save a few cycles by skipping the __get_user. */ - if (restore_sigcontext(regs, &frame->sc, 0)) + if (!restore_sigcontext(regs, &frame->sc, 0)) goto badframe; return regs->ax; @@ -671,7 +671,7 @@ SYSCALL_DEFINE0(rt_sigreturn) set_current_blocked(&set); - if (restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags)) + if (!restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags)) goto badframe; if (restore_altstack(&frame->uc.uc_stack)) @@ -721,12 +721,15 @@ SYSCALL_DEFINE0(rt_sigreturn) /* max_frame_size tells userspace the worst case signal stack size. */ static unsigned long __ro_after_init max_frame_size; +static unsigned int __ro_after_init fpu_default_state_size; void __init init_sigframe_size(void) { + fpu_default_state_size = fpu__get_fpstate_size(); + max_frame_size = MAX_FRAME_SIGINFO_UCTXT_SIZE + MAX_FRAME_PADDING; - max_frame_size += fpu__get_fpstate_size() + MAX_XSAVE_PADDING; + max_frame_size += fpu_default_state_size + MAX_XSAVE_PADDING; /* Userspace expects an aligned size. */ max_frame_size = round_up(max_frame_size, FRAME_ALIGNMENT); @@ -910,6 +913,62 @@ void signal_fault(struct pt_regs *regs, void __user *frame, char *where) force_sig(SIGSEGV); } +#ifdef CONFIG_DYNAMIC_SIGFRAME +#ifdef CONFIG_STRICT_SIGALTSTACK_SIZE +static bool strict_sigaltstack_size __ro_after_init = true; +#else +static bool strict_sigaltstack_size __ro_after_init = false; +#endif + +static int __init strict_sas_size(char *arg) +{ + return kstrtobool(arg, &strict_sigaltstack_size); +} +__setup("strict_sas_size", strict_sas_size); + +/* + * MINSIGSTKSZ is 2048 and can't be changed despite the fact that AVX512 + * exceeds that size already. As such programs might never use the + * sigaltstack they just continued to work. While always checking against + * the real size would be correct, this might be considered a regression. + * + * Therefore avoid the sanity check, unless enforced by kernel + * configuration or command line option. + * + * When dynamic FPU features are supported, the check is also enforced when + * the task has permissions to use dynamic features. Tasks which have no + * permission are checked against the size of the non-dynamic feature set + * if strict checking is enabled. This avoids forcing all tasks on the + * system to allocate large sigaltstacks even if they are never going + * to use a dynamic feature. As this is serialized via sighand::siglock + * any permission request for a dynamic feature either happened already + * or will see the newly install sigaltstack size in the permission checks. + */ +bool sigaltstack_size_valid(size_t ss_size) +{ + unsigned long fsize = max_frame_size - fpu_default_state_size; + u64 mask; + + lockdep_assert_held(¤t->sighand->siglock); + + if (!fpu_state_size_dynamic() && !strict_sigaltstack_size) + return true; + + fsize += current->group_leader->thread.fpu.perm.__user_state_size; + if (likely(ss_size > fsize)) + return true; + + if (strict_sigaltstack_size) + return ss_size > fsize; + + mask = current->group_leader->thread.fpu.perm.__state_perm; + if (mask & XFEATURE_MASK_USER_DYNAMIC) + return ss_size > fsize; + + return true; +} +#endif /* CONFIG_DYNAMIC_SIGFRAME */ + #ifdef CONFIG_X86_X32_ABI COMPAT_SYSCALL_DEFINE0(x32_rt_sigreturn) { @@ -929,7 +988,7 @@ COMPAT_SYSCALL_DEFINE0(x32_rt_sigreturn) set_current_blocked(&set); - if (restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags)) + if (!restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags)) goto badframe; if (compat_restore_altstack(&frame->uc.uc_stack)) diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 85f6e242b6..617012f461 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -70,7 +70,7 @@ #include #include #include -#include +#include #include #include #include @@ -101,6 +101,8 @@ EXPORT_PER_CPU_SYMBOL(cpu_die_map); DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map); +DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_l2c_shared_map); + /* Per CPU bogomips and other parameters */ DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info); EXPORT_PER_CPU_SYMBOL(cpu_info); @@ -464,6 +466,21 @@ static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) return false; } +static bool match_l2c(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) +{ + int cpu1 = c->cpu_index, cpu2 = o->cpu_index; + + /* If the arch didn't set up l2c_id, fall back to SMT */ + if (per_cpu(cpu_l2c_id, cpu1) == BAD_APICID) + return match_smt(c, o); + + /* Do not match if L2 cache id does not match: */ + if (per_cpu(cpu_l2c_id, cpu1) != per_cpu(cpu_l2c_id, cpu2)) + return false; + + return topology_sane(c, o, "l2c"); +} + /* * Unlike the other levels, we do not enforce keeping a * multicore group inside a NUMA node. If this happens, we will @@ -523,7 +540,7 @@ static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) } -#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_MC) +#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_CLUSTER) || defined(CONFIG_SCHED_MC) static inline int x86_sched_itmt_flags(void) { return sysctl_sched_itmt_enabled ? SD_ASYM_PACKING : 0; @@ -541,22 +558,45 @@ static int x86_smt_flags(void) return cpu_smt_flags() | x86_sched_itmt_flags(); } #endif +#ifdef CONFIG_SCHED_CLUSTER +static int x86_cluster_flags(void) +{ + return cpu_cluster_flags() | x86_sched_itmt_flags(); +} +#endif #endif static struct sched_domain_topology_level x86_numa_in_package_topology[] = { #ifdef CONFIG_SCHED_SMT { cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) }, #endif +#ifdef CONFIG_SCHED_CLUSTER + { cpu_clustergroup_mask, x86_cluster_flags, SD_INIT_NAME(CLS) }, +#endif #ifdef CONFIG_SCHED_MC { cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) }, #endif { NULL, }, }; +static struct sched_domain_topology_level x86_hybrid_topology[] = { +#ifdef CONFIG_SCHED_SMT + { cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) }, +#endif +#ifdef CONFIG_SCHED_MC + { cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) }, +#endif + { cpu_cpu_mask, SD_INIT_NAME(DIE) }, + { NULL, }, +}; + static struct sched_domain_topology_level x86_topology[] = { #ifdef CONFIG_SCHED_SMT { cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) }, #endif +#ifdef CONFIG_SCHED_CLUSTER + { cpu_clustergroup_mask, x86_cluster_flags, SD_INIT_NAME(CLS) }, +#endif #ifdef CONFIG_SCHED_MC { cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) }, #endif @@ -584,6 +624,7 @@ void set_cpu_sibling_map(int cpu) if (!has_mp) { cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu)); cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu)); + cpumask_set_cpu(cpu, cpu_l2c_shared_mask(cpu)); cpumask_set_cpu(cpu, topology_core_cpumask(cpu)); cpumask_set_cpu(cpu, topology_die_cpumask(cpu)); c->booted_cores = 1; @@ -602,6 +643,9 @@ void set_cpu_sibling_map(int cpu) if ((i == cpu) || (has_mp && match_llc(c, o))) link_mask(cpu_llc_shared_mask, cpu, i); + if ((i == cpu) || (has_mp && match_l2c(c, o))) + link_mask(cpu_l2c_shared_mask, cpu, i); + if ((i == cpu) || (has_mp && match_die(c, o))) link_mask(topology_die_cpumask, cpu, i); } @@ -652,6 +696,11 @@ const struct cpumask *cpu_coregroup_mask(int cpu) return cpu_llc_shared_mask(cpu); } +const struct cpumask *cpu_clustergroup_mask(int cpu) +{ + return cpu_l2c_shared_mask(cpu); +} + static void impress_friends(void) { int cpu; @@ -1312,12 +1361,7 @@ static void __init smp_get_logical_apicid(void) cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR)); } -/* - * Prepare for SMP bootup. - * @max_cpus: configured maximum number of CPUs, It is a legacy parameter - * for common interface support. - */ -void __init native_smp_prepare_cpus(unsigned int max_cpus) +void __init smp_prepare_cpus_common(void) { unsigned int i; @@ -1335,6 +1379,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL); zalloc_cpumask_var(&per_cpu(cpu_die_map, i), GFP_KERNEL); zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL); + zalloc_cpumask_var(&per_cpu(cpu_l2c_shared_map, i), GFP_KERNEL); } /* @@ -1347,6 +1392,17 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) set_sched_topology(x86_topology); set_cpu_sibling_map(0); +} + +/* + * Prepare for SMP bootup. + * @max_cpus: configured maximum number of CPUs, It is a legacy parameter + * for common interface support. + */ +void __init native_smp_prepare_cpus(unsigned int max_cpus) +{ + smp_prepare_cpus_common(); + init_freq_invariance(false, false); smp_sanity_check(); @@ -1424,8 +1480,11 @@ void __init native_smp_cpus_done(unsigned int max_cpus) calculate_max_logical_packages(); + /* XXX for now assume numa-in-package and hybrid don't overlap */ if (x86_has_numa_in_package) set_sched_topology(x86_numa_in_package_topology); + if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) + set_sched_topology(x86_hybrid_topology); nmi_selftest(); impress_friends(); @@ -1564,7 +1623,10 @@ static void remove_siblinginfo(int cpu) for_each_cpu(sibling, cpu_llc_shared_mask(cpu)) cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling)); + for_each_cpu(sibling, cpu_l2c_shared_mask(cpu)) + cpumask_clear_cpu(cpu, cpu_l2c_shared_mask(sibling)); cpumask_clear(cpu_llc_shared_mask(cpu)); + cpumask_clear(cpu_l2c_shared_mask(cpu)); cpumask_clear(topology_sibling_cpumask(cpu)); cpumask_clear(topology_core_cpumask(cpu)); cpumask_clear(topology_die_cpumask(cpu)); @@ -2166,7 +2228,7 @@ DEFINE_PER_CPU(unsigned long, arch_freq_scale) = SCHED_CAPACITY_SCALE; void arch_scale_freq_tick(void) { - u64 freq_scale = SCHED_CAPACITY_SCALE; + u64 freq_scale; u64 aperf, mperf; u64 acnt, mcnt; diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index ea028e7368..531fb4cbb6 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -17,6 +17,8 @@ enum insn_type { */ static const u8 xor5rax[] = { 0x66, 0x66, 0x48, 0x31, 0xc0 }; +static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc }; + static void __ref __static_call_transform(void *insn, enum insn_type type, void *func) { const void *emulate = NULL; @@ -42,8 +44,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, void break; case RET: - code = text_gen_insn(RET_INSN_OPCODE, insn, func); - size = RET_INSN_SIZE; + code = &retinsn; break; } @@ -56,10 +57,15 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, void text_poke_bp(insn, code, size, emulate); } -static void __static_call_validate(void *insn, bool tail) +static void __static_call_validate(void *insn, bool tail, bool tramp) { u8 opcode = *(u8 *)insn; + if (tramp && memcmp(insn+5, "SCT", 3)) { + pr_err("trampoline signature fail"); + BUG(); + } + if (tail) { if (opcode == JMP32_INSN_OPCODE || opcode == RET_INSN_OPCODE) @@ -74,7 +80,8 @@ static void __static_call_validate(void *insn, bool tail) /* * If we ever trigger this, our text is corrupt, we'll probably not live long. */ - WARN_ONCE(1, "unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn); + pr_err("unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn); + BUG(); } static inline enum insn_type __sc_insn(bool null, bool tail) @@ -97,12 +104,12 @@ void arch_static_call_transform(void *site, void *tramp, void *func, bool tail) mutex_lock(&text_mutex); if (tramp) { - __static_call_validate(tramp, true); + __static_call_validate(tramp, true, true); __static_call_transform(tramp, __sc_insn(!func, true), func); } if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site) { - __static_call_validate(site, tail); + __static_call_validate(site, tail, false); __static_call_transform(site, __sc_insn(!func, tail), func); } diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 928e1ac820..8143693a7e 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -48,7 +48,7 @@ #include #include #include -#include +#include #include #include #include @@ -1143,10 +1143,48 @@ DEFINE_IDTENTRY(exc_spurious_interrupt_bug) */ } +static bool handle_xfd_event(struct pt_regs *regs) +{ + u64 xfd_err; + int err; + + if (!IS_ENABLED(CONFIG_X86_64) || !cpu_feature_enabled(X86_FEATURE_XFD)) + return false; + + rdmsrl(MSR_IA32_XFD_ERR, xfd_err); + if (!xfd_err) + return false; + + wrmsrl(MSR_IA32_XFD_ERR, 0); + + /* Die if that happens in kernel space */ + if (WARN_ON(!user_mode(regs))) + return false; + + local_irq_enable(); + + err = xfd_enable_feature(xfd_err); + + switch (err) { + case -EPERM: + force_sig_fault(SIGILL, ILL_ILLOPC, error_get_trap_addr(regs)); + break; + case -EFAULT: + force_sig(SIGSEGV); + break; + } + + local_irq_disable(); + return true; +} + DEFINE_IDTENTRY(exc_device_not_available) { unsigned long cr0 = read_cr0(); + if (handle_xfd_event(regs)) + return; + #ifdef CONFIG_MATH_EMULATION if (!boot_cpu_has(X86_FEATURE_FPU) && (cr0 & X86_CR0_EM)) { struct math_emu_info info = { }; diff --git a/arch/x86/kernel/umip.c b/arch/x86/kernel/umip.c index 576b47e752..5a4b21389b 100644 --- a/arch/x86/kernel/umip.c +++ b/arch/x86/kernel/umip.c @@ -92,8 +92,8 @@ static const char * const umip_insns[5] = { #define umip_pr_err(regs, fmt, ...) \ umip_printk(regs, KERN_ERR, fmt, ##__VA_ARGS__) -#define umip_pr_warn(regs, fmt, ...) \ - umip_printk(regs, KERN_WARNING, fmt, ##__VA_ARGS__) +#define umip_pr_debug(regs, fmt, ...) \ + umip_printk(regs, KERN_DEBUG, fmt, ##__VA_ARGS__) /** * umip_printk() - Print a rate-limited message @@ -361,10 +361,10 @@ bool fixup_umip_exception(struct pt_regs *regs) if (umip_inst < 0) return false; - umip_pr_warn(regs, "%s instruction cannot be used by applications.\n", + umip_pr_debug(regs, "%s instruction cannot be used by applications.\n", umip_insns[umip_inst]); - umip_pr_warn(regs, "For now, expensive software emulation returns the result.\n"); + umip_pr_debug(regs, "For now, expensive software emulation returns the result.\n"); if (emulate_umip_insn(&insn, umip_inst, dummy_data, &dummy_data_size, user_64bit_mode(regs))) diff --git a/arch/x86/kernel/unwind_frame.c b/arch/x86/kernel/unwind_frame.c index d7c44b257f..8e1c50c86e 100644 --- a/arch/x86/kernel/unwind_frame.c +++ b/arch/x86/kernel/unwind_frame.c @@ -240,8 +240,7 @@ static bool update_stack_state(struct unwind_state *state, else { addr_p = unwind_get_return_address_ptr(state); addr = READ_ONCE_TASK_STACK(state->task, *addr_p); - state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, - addr, addr_p); + state->ip = unwind_recover_ret_addr(state, addr, addr_p); } /* Save the original stack pointer for unwind_dump(): */ diff --git a/arch/x86/kernel/unwind_guess.c b/arch/x86/kernel/unwind_guess.c index c49f10ffd8..884d68a6e7 100644 --- a/arch/x86/kernel/unwind_guess.c +++ b/arch/x86/kernel/unwind_guess.c @@ -15,8 +15,7 @@ unsigned long unwind_get_return_address(struct unwind_state *state) addr = READ_ONCE_NOCHECK(*state->sp); - return ftrace_graph_ret_addr(state->task, &state->graph_idx, - addr, state->sp); + return unwind_recover_ret_addr(state, addr, state->sp); } EXPORT_SYMBOL_GPL(unwind_get_return_address); diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index a1202536fc..2de3c8c5eb 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -175,7 +175,7 @@ static struct orc_entry *orc_find(unsigned long ip) } /* vmlinux .init slow lookup: */ - if (init_kernel_text(ip)) + if (is_kernel_inittext(ip)) return __orc_find(__start_orc_unwind_ip, __start_orc_unwind, __stop_orc_unwind_ip - __start_orc_unwind_ip, ip); @@ -534,9 +534,8 @@ bool unwind_next_frame(struct unwind_state *state) if (!deref_stack_reg(state, ip_p, &state->ip)) goto err; - state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, - state->ip, (void *)ip_p); - + state->ip = unwind_recover_ret_addr(state, state->ip, + (unsigned long *)ip_p); state->sp = sp; state->regs = NULL; state->prev_regs = NULL; @@ -549,7 +548,18 @@ bool unwind_next_frame(struct unwind_state *state) (void *)orig_ip); goto err; } - + /* + * There is a small chance to interrupt at the entry of + * __kretprobe_trampoline() where the ORC info doesn't exist. + * That point is right after the RET to __kretprobe_trampoline() + * which was modified return address. + * At that point, the @addr_p of the unwind_recover_kretprobe() + * (this has to point the address of the stack entry storing + * the modified return address) must be "SP - (a stack entry)" + * because SP is incremented by the RET. + */ + state->ip = unwind_recover_kretprobe(state, state->ip, + (unsigned long *)(state->sp - sizeof(long))); state->regs = (struct pt_regs *)sp; state->prev_regs = NULL; state->full_regs = true; @@ -562,6 +572,9 @@ bool unwind_next_frame(struct unwind_state *state) (void *)orig_ip); goto err; } + /* See UNWIND_HINT_TYPE_REGS case comment. */ + state->ip = unwind_recover_kretprobe(state, state->ip, + (unsigned long *)(state->sp - sizeof(long))); if (state->full_regs) state->prev_regs = state->regs; diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c index 17d5874089..c21bcd6682 100644 --- a/arch/x86/kernel/vm86_32.c +++ b/arch/x86/kernel/vm86_32.c @@ -106,10 +106,8 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval) */ local_irq_enable(); - if (!vm86 || !vm86->user_vm86) { - pr_alert("no user_vm86: BAD\n"); - do_exit(SIGSEGV); - } + BUG_ON(!vm86); + set_flags(regs->pt.flags, VEFLAGS, X86_EFLAGS_VIF | vm86->veflags_mask); user = vm86->user_vm86; diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index efd9e9ea17..27f830345b 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -137,7 +137,6 @@ SECTIONS ALIGN_ENTRY_TEXT_END SOFTIRQENTRY_TEXT STATIC_CALL_TEXT - *(.fixup) *(.gnu.warning) #ifdef CONFIG_RETPOLINE @@ -272,6 +271,20 @@ SECTIONS __parainstructions_end = .; } +#ifdef CONFIG_RETPOLINE + /* + * List of instructions that call/jmp/jcc to retpoline thunks + * __x86_indirect_thunk_*(). These instructions can be patched along + * with alternatives, after which the section can be freed. + */ + . = ALIGN(8); + .retpoline_sites : AT(ADDR(.retpoline_sites) - LOAD_OFFSET) { + __retpoline_sites = .; + *(.retpoline_sites) + __retpoline_sites_end = .; + } +#endif + /* * struct alt_inst entries. From the header (alternative.h): * "Alternative instructions for different CPU types or capabilities" diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index 8b395821cb..7d20c1d34a 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -145,18 +145,6 @@ struct x86_platform_ops x86_platform __ro_after_init = { EXPORT_SYMBOL_GPL(x86_platform); -#if defined(CONFIG_PCI_MSI) -struct x86_msi_ops x86_msi __ro_after_init = { - .restore_msi_irqs = default_restore_msi_irqs, -}; - -/* MSI arch specific hooks */ -void arch_restore_msi_irqs(struct pci_dev *dev) -{ - x86_msi.restore_msi_irqs(dev); -} -#endif - struct x86_apic_ops x86_apic_ops __ro_after_init = { .io_apic_read = native_io_apic_read, .restore = native_restore_boot_irq_mode, diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index ac69894eab..2b1548da00 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -26,7 +26,9 @@ config KVM select PREEMPT_NOTIFIERS select MMU_NOTIFIER select HAVE_KVM_IRQCHIP + select HAVE_KVM_PFNCACHE select HAVE_KVM_IRQFD + select HAVE_KVM_DIRTY_RING select IRQ_BYPASS_MANAGER select HAVE_KVM_IRQ_BYPASS select HAVE_KVM_IRQ_ROUTING @@ -36,6 +38,7 @@ config KVM select KVM_MMIO select SCHED_INFO select PERF_EVENTS + select GUEST_PERF_EVENTS select HAVE_KVM_MSI select HAVE_KVM_CPU_RELAX_INTERCEPT select HAVE_KVM_NO_POLL @@ -43,6 +46,7 @@ config KVM select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_VFIO select SRCU + select INTERVAL_TREE select HAVE_KVM_PM_NOTIFIER if PM help Support hosting fully virtualized guest machines using hardware @@ -129,4 +133,7 @@ config KVM_MMU_AUDIT This option adds a R/W kVM module parameter 'mmu_audit', which allows auditing of KVM MMU events at runtime. +config KVM_EXTERNAL_WRITE_TRACKING + bool + endif # VIRTUALIZATION diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 75dfd27b6e..30f244b645 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -7,12 +7,7 @@ ifeq ($(CONFIG_FRAME_POINTER),y) OBJECT_FILES_NON_STANDARD_vmenter.o := y endif -KVM := ../../../virt/kvm - -kvm-y += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o \ - $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o \ - $(KVM)/dirty_ring.o $(KVM)/binary_stats.o -kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o +include $(srctree)/virt/kvm/Makefile.kvm kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 5f1d4a5aa8..b8f8d268d0 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -32,7 +32,7 @@ u32 kvm_cpu_caps[NR_KVM_CPU_CAPS] __read_mostly; EXPORT_SYMBOL_GPL(kvm_cpu_caps); -static u32 xstate_required_size(u64 xstate_bv, bool compacted) +u32 xstate_required_size(u64 xstate_bv, bool compacted) { int feature_bit = 0; u32 ret = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; @@ -42,7 +42,11 @@ static u32 xstate_required_size(u64 xstate_bv, bool compacted) if (xstate_bv & 0x1) { u32 eax, ebx, ecx, edx, offset; cpuid_count(0xD, feature_bit, &eax, &ebx, &ecx, &edx); - offset = compacted ? ret : ebx; + /* ECX[1]: 64B alignment in compacted form */ + if (compacted) + offset = (ecx & 0x2) ? ALIGN(ret, 64) : ret; + else + offset = ebx; ret = max(ret, offset + eax); } @@ -53,9 +57,16 @@ static u32 xstate_required_size(u64 xstate_bv, bool compacted) return ret; } +/* + * This one is tied to SSB in the user API, and not + * visible in /proc/cpuinfo. + */ +#define KVM_X86_FEATURE_PSFD (13*32+28) /* Predictive Store Forwarding Disable */ + #define F feature_bit #define SF(name) (boot_cpu_has(X86_FEATURE_##name) ? F(name) : 0) + static inline struct kvm_cpuid_entry2 *cpuid_entry2_find( struct kvm_cpuid_entry2 *entries, int nent, u32 function, u32 index) { @@ -73,9 +84,12 @@ static inline struct kvm_cpuid_entry2 *cpuid_entry2_find( return NULL; } -static int kvm_check_cpuid(struct kvm_cpuid_entry2 *entries, int nent) +static int kvm_check_cpuid(struct kvm_vcpu *vcpu, + struct kvm_cpuid_entry2 *entries, + int nent) { struct kvm_cpuid_entry2 *best; + u64 xfeatures; /* * The existing code assumes virtual address is 48-bit or 57-bit in the @@ -89,14 +103,91 @@ static int kvm_check_cpuid(struct kvm_cpuid_entry2 *entries, int nent) return -EINVAL; } + /* + * Exposing dynamic xfeatures to the guest requires additional + * enabling in the FPU, e.g. to expand the guest XSAVE state size. + */ + best = cpuid_entry2_find(entries, nent, 0xd, 0); + if (!best) + return 0; + + xfeatures = best->eax | ((u64)best->edx << 32); + xfeatures &= XFEATURE_MASK_USER_DYNAMIC; + if (!xfeatures) + return 0; + + return fpu_enable_guest_xfd_features(&vcpu->arch.guest_fpu, xfeatures); +} + +/* Check whether the supplied CPUID data is equal to what is already set for the vCPU. */ +static int kvm_cpuid_check_equal(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2, + int nent) +{ + struct kvm_cpuid_entry2 *orig; + int i; + + if (nent != vcpu->arch.cpuid_nent) + return -EINVAL; + + for (i = 0; i < nent; i++) { + orig = &vcpu->arch.cpuid_entries[i]; + if (e2[i].function != orig->function || + e2[i].index != orig->index || + e2[i].flags != orig->flags || + e2[i].eax != orig->eax || e2[i].ebx != orig->ebx || + e2[i].ecx != orig->ecx || e2[i].edx != orig->edx) + return -EINVAL; + } + return 0; } +static void kvm_update_kvm_cpuid_base(struct kvm_vcpu *vcpu) +{ + u32 function; + struct kvm_cpuid_entry2 *entry; + + vcpu->arch.kvm_cpuid_base = 0; + + for_each_possible_hypervisor_cpuid_base(function) { + entry = kvm_find_cpuid_entry(vcpu, function, 0); + + if (entry) { + u32 signature[3]; + + signature[0] = entry->ebx; + signature[1] = entry->ecx; + signature[2] = entry->edx; + + BUILD_BUG_ON(sizeof(signature) > sizeof(KVM_SIGNATURE)); + if (!memcmp(signature, KVM_SIGNATURE, sizeof(signature))) { + vcpu->arch.kvm_cpuid_base = function; + break; + } + } + } +} + +static struct kvm_cpuid_entry2 *__kvm_find_kvm_cpuid_features(struct kvm_vcpu *vcpu, + struct kvm_cpuid_entry2 *entries, int nent) +{ + u32 base = vcpu->arch.kvm_cpuid_base; + + if (!base) + return NULL; + + return cpuid_entry2_find(entries, nent, base | KVM_CPUID_FEATURES, 0); +} + +static struct kvm_cpuid_entry2 *kvm_find_kvm_cpuid_features(struct kvm_vcpu *vcpu) +{ + return __kvm_find_kvm_cpuid_features(vcpu, vcpu->arch.cpuid_entries, + vcpu->arch.cpuid_nent); +} + void kvm_update_pv_runtime(struct kvm_vcpu *vcpu) { - struct kvm_cpuid_entry2 *best; - - best = kvm_find_cpuid_entry(vcpu, KVM_CPUID_FEATURES, 0); + struct kvm_cpuid_entry2 *best = kvm_find_kvm_cpuid_features(vcpu); /* * save the feature bitmap to avoid cpuid lookup for every PV @@ -106,11 +197,28 @@ void kvm_update_pv_runtime(struct kvm_vcpu *vcpu) vcpu->arch.pv_cpuid.features = best->eax; } -void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu) +/* + * Calculate guest's supported XCR0 taking into account guest CPUID data and + * supported_xcr0 (comprised of host configuration and KVM_SUPPORTED_XCR0). + */ +static u64 cpuid_get_supported_xcr0(struct kvm_cpuid_entry2 *entries, int nent) { struct kvm_cpuid_entry2 *best; - best = kvm_find_cpuid_entry(vcpu, 1, 0); + best = cpuid_entry2_find(entries, nent, 0xd, 0); + if (!best) + return 0; + + return (best->eax | ((u64)best->edx << 32)) & supported_xcr0; +} + +static void __kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *entries, + int nent) +{ + struct kvm_cpuid_entry2 *best; + u64 guest_supported_xcr0 = cpuid_get_supported_xcr0(entries, nent); + + best = cpuid_entry2_find(entries, nent, 1, 0); if (best) { /* Update OSXSAVE bit */ if (boot_cpu_has(X86_FEATURE_XSAVE)) @@ -121,32 +229,52 @@ void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu) vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE); } - best = kvm_find_cpuid_entry(vcpu, 7, 0); + best = cpuid_entry2_find(entries, nent, 7, 0); if (best && boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7) cpuid_entry_change(best, X86_FEATURE_OSPKE, kvm_read_cr4_bits(vcpu, X86_CR4_PKE)); - best = kvm_find_cpuid_entry(vcpu, 0xD, 0); + best = cpuid_entry2_find(entries, nent, 0xD, 0); if (best) best->ebx = xstate_required_size(vcpu->arch.xcr0, false); - best = kvm_find_cpuid_entry(vcpu, 0xD, 1); + best = cpuid_entry2_find(entries, nent, 0xD, 1); if (best && (cpuid_entry_has(best, X86_FEATURE_XSAVES) || cpuid_entry_has(best, X86_FEATURE_XSAVEC))) best->ebx = xstate_required_size(vcpu->arch.xcr0, true); - best = kvm_find_cpuid_entry(vcpu, KVM_CPUID_FEATURES, 0); + best = __kvm_find_kvm_cpuid_features(vcpu, entries, nent); if (kvm_hlt_in_guest(vcpu->kvm) && best && (best->eax & (1 << KVM_FEATURE_PV_UNHALT))) best->eax &= ~(1 << KVM_FEATURE_PV_UNHALT); if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT)) { - best = kvm_find_cpuid_entry(vcpu, 0x1, 0); + best = cpuid_entry2_find(entries, nent, 0x1, 0); if (best) cpuid_entry_change(best, X86_FEATURE_MWAIT, vcpu->arch.ia32_misc_enable_msr & MSR_IA32_MISC_ENABLE_MWAIT); } + + /* + * Bits 127:0 of the allowed SECS.ATTRIBUTES (CPUID.0x12.0x1) enumerate + * the supported XSAVE Feature Request Mask (XFRM), i.e. the enclave's + * requested XCR0 value. The enclave's XFRM must be a subset of XCRO + * at the time of EENTER, thus adjust the allowed XFRM by the guest's + * supported XCR0. Similar to XCR0 handling, FP and SSE are forced to + * '1' even on CPUs that don't support XSAVE. + */ + best = cpuid_entry2_find(entries, nent, 0x12, 0x1); + if (best) { + best->ecx &= guest_supported_xcr0 & 0xffffffff; + best->edx &= guest_supported_xcr0 >> 32; + best->ecx |= XFEATURE_MASK_FPSSE; + } +} + +void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu) +{ + __kvm_update_cpuid_runtime(vcpu, vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent); } EXPORT_SYMBOL_GPL(kvm_update_cpuid_runtime); @@ -154,6 +282,7 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) { struct kvm_lapic *apic = vcpu->arch.apic; struct kvm_cpuid_entry2 *best; + u64 guest_supported_xcr0; best = kvm_find_cpuid_entry(vcpu, 1, 0); if (best && apic) { @@ -165,27 +294,10 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) kvm_apic_set_version(vcpu); } - best = kvm_find_cpuid_entry(vcpu, 0xD, 0); - if (!best) - vcpu->arch.guest_supported_xcr0 = 0; - else - vcpu->arch.guest_supported_xcr0 = - (best->eax | ((u64)best->edx << 32)) & supported_xcr0; + guest_supported_xcr0 = + cpuid_get_supported_xcr0(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent); - /* - * Bits 127:0 of the allowed SECS.ATTRIBUTES (CPUID.0x12.0x1) enumerate - * the supported XSAVE Feature Request Mask (XFRM), i.e. the enclave's - * requested XCR0 value. The enclave's XFRM must be a subset of XCRO - * at the time of EENTER, thus adjust the allowed XFRM by the guest's - * supported XCR0. Similar to XCR0 handling, FP and SSE are forced to - * '1' even on CPUs that don't support XSAVE. - */ - best = kvm_find_cpuid_entry(vcpu, 0x12, 0x1); - if (best) { - best->ecx &= vcpu->arch.guest_supported_xcr0 & 0xffffffff; - best->edx &= vcpu->arch.guest_supported_xcr0 >> 32; - best->ecx |= XFEATURE_MASK_FPSSE; - } + vcpu->arch.guest_fpu.fpstate->user_xfeatures = guest_supported_xcr0; kvm_update_pv_runtime(vcpu); @@ -235,20 +347,42 @@ u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu) static int kvm_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2, int nent) { - int r; + int r; - r = kvm_check_cpuid(e2, nent); - if (r) - return r; + __kvm_update_cpuid_runtime(vcpu, e2, nent); - kvfree(vcpu->arch.cpuid_entries); - vcpu->arch.cpuid_entries = e2; - vcpu->arch.cpuid_nent = nent; + /* + * KVM does not correctly handle changing guest CPUID after KVM_RUN, as + * MAXPHYADDR, GBPAGES support, AMD reserved bit behavior, etc.. aren't + * tracked in kvm_mmu_page_role. As a result, KVM may miss guest page + * faults due to reusing SPs/SPTEs. In practice no sane VMM mucks with + * the core vCPU model on the fly. It would've been better to forbid any + * KVM_SET_CPUID{,2} calls after KVM_RUN altogether but unfortunately + * some VMMs (e.g. QEMU) reuse vCPU fds for CPU hotplug/unplug and do + * KVM_SET_CPUID{,2} again. To support this legacy behavior, check + * whether the supplied CPUID data is equal to what's already set. + */ + if (vcpu->arch.last_vmentry_cpu != -1) { + r = kvm_cpuid_check_equal(vcpu, e2, nent); + if (r) + return r; - kvm_update_cpuid_runtime(vcpu); - kvm_vcpu_after_set_cpuid(vcpu); + kvfree(e2); + return 0; + } - return 0; + r = kvm_check_cpuid(vcpu, e2, nent); + if (r) + return r; + + kvfree(vcpu->arch.cpuid_entries); + vcpu->arch.cpuid_entries = e2; + vcpu->arch.cpuid_nent = nent; + + kvm_update_kvm_cpuid_base(vcpu); + kvm_vcpu_after_set_cpuid(vcpu); + + return 0; } /* when an old userspace process fills a new kernel module */ @@ -380,9 +514,11 @@ void kvm_set_cpu_caps(void) #ifdef CONFIG_X86_64 unsigned int f_gbpages = F(GBPAGES); unsigned int f_lm = F(LM); + unsigned int f_xfd = F(XFD); #else unsigned int f_gbpages = 0; unsigned int f_lm = 0; + unsigned int f_xfd = 0; #endif memset(kvm_cpu_caps, 0, sizeof(kvm_cpu_caps)); @@ -451,7 +587,8 @@ void kvm_set_cpu_caps(void) F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) | F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) | F(MD_CLEAR) | F(AVX512_VP2INTERSECT) | F(FSRM) | - F(SERIALIZE) | F(TSXLDTRK) | F(AVX512_FP16) + F(SERIALIZE) | F(TSXLDTRK) | F(AVX512_FP16) | + F(AMX_TILE) | F(AMX_INT8) | F(AMX_BF16) ); /* TSC_ADJUST and ARCH_CAPABILITIES are emulated in software. */ @@ -470,7 +607,7 @@ void kvm_set_cpu_caps(void) ); kvm_cpu_cap_mask(CPUID_D_1_EAX, - F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | F(XSAVES) + F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | F(XSAVES) | f_xfd ); kvm_cpu_cap_init_scattered(CPUID_12_EAX, @@ -482,7 +619,7 @@ void kvm_set_cpu_caps(void) F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) | F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) | 0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) | - F(TOPOEXT) | F(PERFCTR_CORE) + F(TOPOEXT) | 0 /* PERFCTR_CORE */ ); kvm_cpu_cap_mask(CPUID_8000_0001_EDX, @@ -502,7 +639,8 @@ void kvm_set_cpu_caps(void) kvm_cpu_cap_mask(CPUID_8000_0008_EBX, F(CLZERO) | F(XSAVEERPTR) | F(WBNOINVD) | F(AMD_IBPB) | F(AMD_IBRS) | F(AMD_SSBD) | F(VIRT_SSBD) | - F(AMD_SSB_NO) | F(AMD_STIBP) | F(AMD_STIBP_ALWAYS_ON) + F(AMD_SSB_NO) | F(AMD_STIBP) | F(AMD_STIBP_ALWAYS_ON) | + __feature_bit(KVM_X86_FEATURE_PSFD) ); /* @@ -595,6 +733,8 @@ static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array, case 0x14: case 0x17: case 0x18: + case 0x1d: + case 0x1e: case 0x1f: case 0x8000001d: entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; @@ -728,10 +868,10 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) perf_get_x86_pmu_capability(&cap); /* - * Only support guest architectural pmu on a host - * with architectural pmu. + * The guest architecture pmu is only supported if the architecture + * pmu exists on the host and the module parameters allow it. */ - if (!cap.version) + if (!cap.version || !enable_pmu) memset(&cap, 0, sizeof(cap)); eax.split.version_id = min(cap.version, 2); @@ -769,12 +909,15 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) goto out; } break; - case 0xd: - entry->eax &= supported_xcr0; - entry->ebx = xstate_required_size(supported_xcr0, false); + case 0xd: { + u64 permitted_xcr0 = supported_xcr0 & xstate_get_guest_group_perm(); + u64 permitted_xss = supported_xss; + + entry->eax &= permitted_xcr0; + entry->ebx = xstate_required_size(permitted_xcr0, false); entry->ecx = entry->ebx; - entry->edx &= supported_xcr0 >> 32; - if (!supported_xcr0) + entry->edx &= permitted_xcr0 >> 32; + if (!permitted_xcr0) break; entry = do_host_cpuid(array, function, 1); @@ -783,20 +926,20 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) cpuid_entry_override(entry, CPUID_D_1_EAX); if (entry->eax & (F(XSAVES)|F(XSAVEC))) - entry->ebx = xstate_required_size(supported_xcr0 | supported_xss, + entry->ebx = xstate_required_size(permitted_xcr0 | permitted_xss, true); else { - WARN_ON_ONCE(supported_xss != 0); + WARN_ON_ONCE(permitted_xss != 0); entry->ebx = 0; } - entry->ecx &= supported_xss; - entry->edx &= supported_xss >> 32; + entry->ecx &= permitted_xss; + entry->edx &= permitted_xss >> 32; for (i = 2; i < 64; ++i) { bool s_state; - if (supported_xcr0 & BIT_ULL(i)) + if (permitted_xcr0 & BIT_ULL(i)) s_state = false; - else if (supported_xss & BIT_ULL(i)) + else if (permitted_xss & BIT_ULL(i)) s_state = true; else continue; @@ -810,16 +953,20 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) * invalid sub-leafs. Only valid sub-leafs should * reach this point, and they should have a non-zero * save state size. Furthermore, check whether the - * processor agrees with supported_xcr0/supported_xss + * processor agrees with permitted_xcr0/permitted_xss * on whether this is an XCR0- or IA32_XSS-managed area. */ if (WARN_ON_ONCE(!entry->eax || (entry->ecx & 0x1) != s_state)) { --array->nent; continue; } + + if (!kvm_cpu_cap_has(X86_FEATURE_XFD)) + entry->ecx &= ~BIT_ULL(2); entry->edx = 0; } break; + } case 0x12: /* Intel SGX */ if (!kvm_cpu_cap_has(X86_FEATURE_SGX)) { @@ -864,9 +1011,26 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) goto out; } break; + /* Intel AMX TILE */ + case 0x1d: + if (!kvm_cpu_cap_has(X86_FEATURE_AMX_TILE)) { + entry->eax = entry->ebx = entry->ecx = entry->edx = 0; + break; + } + + for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) { + if (!do_host_cpuid(array, function, i)) + goto out; + } + break; + case 0x1e: /* TMUL information */ + if (!kvm_cpu_cap_has(X86_FEATURE_AMX_TILE)) { + entry->eax = entry->ebx = entry->ecx = entry->edx = 0; + break; + } + break; case KVM_CPUID_SIGNATURE: { - static const char signature[12] = "KVMKVMKVM\0\0"; - const u32 *sigptr = (const u32 *)signature; + const u32 *sigptr = (const u32 *)KVM_SIGNATURE; entry->eax = KVM_CPUID_FEATURES; entry->ebx = sigptr[0]; entry->ecx = sigptr[1]; diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index c99edfff7f..8a770b481d 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -30,6 +30,8 @@ int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, bool exact_only); +u32 xstate_required_size(u64 xstate_bv, bool compacted); + int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c index f33c804a92..9240b3b7f8 100644 --- a/arch/x86/kvm/debugfs.c +++ b/arch/x86/kvm/debugfs.c @@ -110,9 +110,10 @@ static int kvm_mmu_rmaps_stat_show(struct seq_file *m, void *v) write_lock(&kvm->mmu_lock); for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + int bkt; + slots = __kvm_memslots(kvm, i); - for (j = 0; j < slots->used_slots; j++) { - slot = &slots->memslots[j]; + kvm_for_each_memslot(slot, bkt, slots) for (k = 0; k < KVM_NR_PAGE_SIZES; k++) { rmap = slot->arch.rmap[k]; lpage_size = kvm_mmu_slot_lpages(slot, k + 1); @@ -124,7 +125,6 @@ static int kvm_mmu_rmaps_stat_show(struct seq_file *m, void *v) cur[index]++; } } - } } write_unlock(&kvm->mmu_lock); diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 9a144ca8e1..e86d610dc6 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -175,6 +175,7 @@ #define No16 ((u64)1 << 53) /* No 16 bit operand */ #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */ #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */ +#define IsBranch ((u64)1 << 56) /* Instruction is considered a branch. */ #define DstXacc (DstAccLo | SrcAccHi | SrcWrite) @@ -191,8 +192,9 @@ #define FASTOP_SIZE 8 struct opcode { - u64 flags : 56; - u64 intercept : 8; + u64 flags; + u8 intercept; + u8 pad[7]; union { int (*execute)(struct x86_emulate_ctxt *ctxt); const struct opcode *group; @@ -315,7 +317,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); __FOP_FUNC(#name) #define __FOP_RET(name) \ - "ret \n\t" \ + "11: " ASM_RET \ ".size " name ", .-" name "\n\t" #define FOP_RET(name) \ @@ -344,7 +346,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); __FOP_RET(#op "_" #dst) #define FOP1EEX(op, dst) \ - FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception) + FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi) #define FASTOP1(op) \ FOP_START(op) \ @@ -427,17 +429,28 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); FOP_END /* Special case for SETcc - 1 instruction per cc */ + +/* + * Depending on .config the SETcc functions look like: + * + * SETcc %al [3 bytes] + * RET [1 byte] + * INT3 [1 byte; CONFIG_SLS] + * + * Which gives possible sizes 4 or 5. When rounded up to the + * next power-of-two alignment they become 4 or 8. + */ +#define SETCC_LENGTH (4 + IS_ENABLED(CONFIG_SLS)) +#define SETCC_ALIGN (4 << IS_ENABLED(CONFIG_SLS)) +static_assert(SETCC_LENGTH <= SETCC_ALIGN); + #define FOP_SETCC(op) \ - ".align 4 \n\t" \ + ".align " __stringify(SETCC_ALIGN) " \n\t" \ ".type " #op ", @function \n\t" \ #op ": \n\t" \ #op " %al \n\t" \ __FOP_RET(#op) -asm(".pushsection .fixup, \"ax\"\n" - "kvm_fastop_exception: xor %esi, %esi; ret\n" - ".popsection"); - FOP_START(setcc) FOP_SETCC(seto) FOP_SETCC(setno) @@ -473,12 +486,8 @@ FOP_END; \ asm volatile("1:" insn "\n" \ "2:\n" \ - ".pushsection .fixup, \"ax\"\n" \ - "3: movl $1, %[_fault]\n" \ - " jmp 2b\n" \ - ".popsection\n" \ - _ASM_EXTABLE(1b, 3b) \ - : [_fault] "+qm"(_fault) inoutclob ); \ + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %[_fault]) \ + : [_fault] "+r"(_fault) inoutclob ); \ \ _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \ }) @@ -1053,7 +1062,7 @@ static int em_bsr_c(struct x86_emulate_ctxt *ctxt) static __always_inline u8 test_cc(unsigned int condition, unsigned long flags) { u8 rc; - void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf); + void (*fop)(void) = (void *)em_setcc + SETCC_ALIGN * (condition & 0xf); flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF; asm("push %[flags]; popf; " CALL_NOSPEC @@ -4222,6 +4231,11 @@ static int check_rdpmc(struct x86_emulate_ctxt *ctxt) if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx)) return X86EMUL_CONTINUE; + /* + * If CR4.PCE is set, the SDM requires CPL=0 or CR0.PE=0. The CR0.PE + * check however is unnecessary because CPL is always 0 outside + * protected mode. + */ if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) || ctxt->ops->check_pmc(ctxt, rcx)) return emulate_gp(ctxt, 0); @@ -4359,10 +4373,10 @@ static const struct opcode group4[] = { static const struct opcode group5[] = { F(DstMem | SrcNone | Lock, em_inc), F(DstMem | SrcNone | Lock, em_dec), - I(SrcMem | NearBranch, em_call_near_abs), - I(SrcMemFAddr | ImplicitOps, em_call_far), - I(SrcMem | NearBranch, em_jmp_abs), - I(SrcMemFAddr | ImplicitOps, em_jmp_far), + I(SrcMem | NearBranch | IsBranch, em_call_near_abs), + I(SrcMemFAddr | ImplicitOps | IsBranch, em_call_far), + I(SrcMem | NearBranch | IsBranch, em_jmp_abs), + I(SrcMemFAddr | ImplicitOps | IsBranch, em_jmp_far), I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined), }; @@ -4572,7 +4586,7 @@ static const struct opcode opcode_table[256] = { I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */ I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */ /* 0x70 - 0x7F */ - X16(D(SrcImmByte | NearBranch)), + X16(D(SrcImmByte | NearBranch | IsBranch)), /* 0x80 - 0x87 */ G(ByteOp | DstMem | SrcImm, group1), G(DstMem | SrcImm, group1), @@ -4591,7 +4605,7 @@ static const struct opcode opcode_table[256] = { DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)), /* 0x98 - 0x9F */ D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd), - I(SrcImmFAddr | No64, em_call_far), N, + I(SrcImmFAddr | No64 | IsBranch, em_call_far), N, II(ImplicitOps | Stack, em_pushf, pushf), II(ImplicitOps | Stack, em_popf, popf), I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf), @@ -4611,17 +4625,19 @@ static const struct opcode opcode_table[256] = { X8(I(DstReg | SrcImm64 | Mov, em_mov)), /* 0xC0 - 0xC7 */ G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2), - I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm), - I(ImplicitOps | NearBranch, em_ret), + I(ImplicitOps | NearBranch | SrcImmU16 | IsBranch, em_ret_near_imm), + I(ImplicitOps | NearBranch | IsBranch, em_ret), I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg), I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg), G(ByteOp, group11), G(0, group11), /* 0xC8 - 0xCF */ - I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave), - I(ImplicitOps | SrcImmU16, em_ret_far_imm), - I(ImplicitOps, em_ret_far), - D(ImplicitOps), DI(SrcImmByte, intn), - D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret), + I(Stack | SrcImmU16 | Src2ImmByte | IsBranch, em_enter), + I(Stack | IsBranch, em_leave), + I(ImplicitOps | SrcImmU16 | IsBranch, em_ret_far_imm), + I(ImplicitOps | IsBranch, em_ret_far), + D(ImplicitOps | IsBranch), DI(SrcImmByte | IsBranch, intn), + D(ImplicitOps | No64 | IsBranch), + II(ImplicitOps | IsBranch, em_iret, iret), /* 0xD0 - 0xD7 */ G(Src2One | ByteOp, group2), G(Src2One, group2), G(Src2CL | ByteOp, group2), G(Src2CL, group2), @@ -4632,14 +4648,15 @@ static const struct opcode opcode_table[256] = { /* 0xD8 - 0xDF */ N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N, /* 0xE0 - 0xE7 */ - X3(I(SrcImmByte | NearBranch, em_loop)), - I(SrcImmByte | NearBranch, em_jcxz), + X3(I(SrcImmByte | NearBranch | IsBranch, em_loop)), + I(SrcImmByte | NearBranch | IsBranch, em_jcxz), I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in), I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out), /* 0xE8 - 0xEF */ - I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch), - I(SrcImmFAddr | No64, em_jmp_far), - D(SrcImmByte | ImplicitOps | NearBranch), + I(SrcImm | NearBranch | IsBranch, em_call), + D(SrcImm | ImplicitOps | NearBranch | IsBranch), + I(SrcImmFAddr | No64 | IsBranch, em_jmp_far), + D(SrcImmByte | ImplicitOps | NearBranch | IsBranch), I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in), I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out), /* 0xF0 - 0xF7 */ @@ -4655,7 +4672,7 @@ static const struct opcode opcode_table[256] = { static const struct opcode twobyte_table[256] = { /* 0x00 - 0x0F */ G(0, group6), GD(0, &group7), N, N, - N, I(ImplicitOps | EmulateOnUD, em_syscall), + N, I(ImplicitOps | EmulateOnUD | IsBranch, em_syscall), II(ImplicitOps | Priv, em_clts, clts), N, DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N, @@ -4686,8 +4703,8 @@ static const struct opcode twobyte_table[256] = { IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc), II(ImplicitOps | Priv, em_rdmsr, rdmsr), IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc), - I(ImplicitOps | EmulateOnUD, em_sysenter), - I(ImplicitOps | Priv | EmulateOnUD, em_sysexit), + I(ImplicitOps | EmulateOnUD | IsBranch, em_sysenter), + I(ImplicitOps | Priv | EmulateOnUD | IsBranch, em_sysexit), N, N, N, N, N, N, N, N, N, N, /* 0x40 - 0x4F */ @@ -4705,7 +4722,7 @@ static const struct opcode twobyte_table[256] = { N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f), /* 0x80 - 0x8F */ - X16(D(SrcImm | NearBranch)), + X16(D(SrcImm | NearBranch | IsBranch)), /* 0x90 - 0x9F */ X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)), /* 0xA0 - 0xA7 */ @@ -5219,6 +5236,8 @@ int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len, int ctxt->d |= opcode.flags; } + ctxt->is_branch = opcode.flags & IsBranch; + /* Unrecognised? */ if (ctxt->d == 0) return EMULATION_FAILED; diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 2092834efb..6e38a7d22e 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -112,7 +112,7 @@ static void synic_update_vector(struct kvm_vcpu_hv_synic *synic, if (!!auto_eoi_old == !!auto_eoi_new) return; - mutex_lock(&vcpu->kvm->arch.apicv_update_lock); + down_write(&vcpu->kvm->arch.apicv_update_lock); if (auto_eoi_new) hv->synic_auto_eoi_used++; @@ -123,7 +123,7 @@ static void synic_update_vector(struct kvm_vcpu_hv_synic *synic, !hv->synic_auto_eoi_used, APICV_INHIBIT_REASON_HYPERV); - mutex_unlock(&vcpu->kvm->arch.apicv_update_lock); + up_write(&vcpu->kvm->arch.apicv_update_lock); } static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint, @@ -164,7 +164,7 @@ static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint, static struct kvm_vcpu *get_vcpu_by_vpidx(struct kvm *kvm, u32 vpidx) { struct kvm_vcpu *vcpu = NULL; - int i; + unsigned long i; if (vpidx >= KVM_MAX_VCPUS) return NULL; @@ -1472,7 +1472,7 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) if (!(data & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE)) { hv_vcpu->hv_vapic = data; - if (kvm_lapic_enable_pv_eoi(vcpu, 0, 0)) + if (kvm_lapic_set_pv_eoi(vcpu, 0, 0)) return 1; break; } @@ -1490,7 +1490,7 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) return 1; hv_vcpu->hv_vapic = data; kvm_vcpu_mark_page_dirty(vcpu, gfn); - if (kvm_lapic_enable_pv_eoi(vcpu, + if (kvm_lapic_set_pv_eoi(vcpu, gfn_to_gpa(gfn) | KVM_MSR_ENABLED, sizeof(struct hv_vp_assist_page))) return 1; @@ -1716,7 +1716,8 @@ static __always_inline unsigned long *sparse_set_to_vcpu_mask( { struct kvm_hv *hv = to_kvm_hv(kvm); struct kvm_vcpu *vcpu; - int i, bank, sbank = 0; + int bank, sbank = 0; + unsigned long i; memset(vp_bitmap, 0, KVM_HV_MAX_SPARSE_VCPU_SET_BITS * sizeof(*vp_bitmap)); @@ -1754,7 +1755,6 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool int i; gpa_t gpa; struct kvm *kvm = vcpu->kvm; - struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); struct hv_tlb_flush_ex flush_ex; struct hv_tlb_flush flush; u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS]; @@ -1836,18 +1836,19 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool } } - cpumask_clear(&hv_vcpu->tlb_flush); - - vcpu_mask = all_cpus ? NULL : - sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask, - vp_bitmap, vcpu_bitmap); - /* * vcpu->arch.cr3 may not be up-to-date for running vCPUs so we can't * analyze it here, flush TLB regardless of the specified address space. */ - kvm_make_vcpus_request_mask(kvm, KVM_REQ_TLB_FLUSH_GUEST, - NULL, vcpu_mask, &hv_vcpu->tlb_flush); + if (all_cpus) { + kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH_GUEST); + } else { + vcpu_mask = sparse_set_to_vcpu_mask(kvm, sparse_banks, valid_bank_mask, + vp_bitmap, vcpu_bitmap); + + kvm_make_vcpus_request_mask(kvm, KVM_REQ_TLB_FLUSH_GUEST, + vcpu_mask); + } ret_success: /* We always do full TLB flush, set 'Reps completed' = 'Rep Count' */ @@ -1863,7 +1864,7 @@ static void kvm_send_ipi_to_many(struct kvm *kvm, u32 vector, .vector = vector }; struct kvm_vcpu *vcpu; - int i; + unsigned long i; kvm_for_each_vcpu(i, vcpu, kvm) { if (vcpu_bitmap && !test_bit(i, vcpu_bitmap)) @@ -2519,6 +2520,8 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, case HYPERV_CPUID_NESTED_FEATURES: ent->eax = evmcs_ver; + if (evmcs_ver) + ent->eax |= HV_X64_NESTED_MSR_BITMAP; break; diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index 5a69cce4d7..0b65a764ed 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c @@ -242,7 +242,7 @@ static void pit_do_work(struct kthread_work *work) struct kvm_pit *pit = container_of(work, struct kvm_pit, expired); struct kvm *kvm = pit->kvm; struct kvm_vcpu *vcpu; - int i; + unsigned long i; struct kvm_kpit_state *ps = &pit->pit_state; if (atomic_read(&ps->reinject) && !atomic_xchg(&ps->irq_ack, 0)) diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c index 0b80263d46..814064d060 100644 --- a/arch/x86/kvm/i8259.c +++ b/arch/x86/kvm/i8259.c @@ -50,7 +50,7 @@ static void pic_unlock(struct kvm_pic *s) { bool wakeup = s->wakeup_needed; struct kvm_vcpu *vcpu; - int i; + unsigned long i; s->wakeup_needed = false; @@ -270,7 +270,8 @@ int kvm_pic_read_irq(struct kvm *kvm) static void kvm_pic_reset(struct kvm_kpic_state *s) { - int irq, i; + int irq; + unsigned long i; struct kvm_vcpu *vcpu; u8 edge_irr = s->irr & ~s->elcr; bool found = false; diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c index 4e0f526608..decfa36b78 100644 --- a/arch/x86/kvm/ioapic.c +++ b/arch/x86/kvm/ioapic.c @@ -96,7 +96,7 @@ static unsigned long ioapic_read_indirect(struct kvm_ioapic *ioapic, static void rtc_irq_eoi_tracking_reset(struct kvm_ioapic *ioapic) { ioapic->rtc_status.pending_eoi = 0; - bitmap_zero(ioapic->rtc_status.dest_map.map, KVM_MAX_VCPU_ID); + bitmap_zero(ioapic->rtc_status.dest_map.map, KVM_MAX_VCPU_IDS); } static void kvm_rtc_eoi_tracking_restore_all(struct kvm_ioapic *ioapic); @@ -149,7 +149,7 @@ void kvm_rtc_eoi_tracking_restore_one(struct kvm_vcpu *vcpu) static void kvm_rtc_eoi_tracking_restore_all(struct kvm_ioapic *ioapic) { struct kvm_vcpu *vcpu; - int i; + unsigned long i; if (RTC_GSI >= IOAPIC_NUM_PINS) return; @@ -184,7 +184,7 @@ static bool rtc_irq_check_coalesced(struct kvm_ioapic *ioapic) static void ioapic_lazy_update_eoi(struct kvm_ioapic *ioapic, int irq) { - int i; + unsigned long i; struct kvm_vcpu *vcpu; union kvm_ioapic_redirect_entry *entry = &ioapic->redirtbl[irq]; diff --git a/arch/x86/kvm/ioapic.h b/arch/x86/kvm/ioapic.h index f1b2b2a6ff..539333ac4b 100644 --- a/arch/x86/kvm/ioapic.h +++ b/arch/x86/kvm/ioapic.h @@ -39,13 +39,13 @@ struct kvm_vcpu; struct dest_map { /* vcpu bitmap where IRQ has been sent */ - DECLARE_BITMAP(map, KVM_MAX_VCPU_ID); + DECLARE_BITMAP(map, KVM_MAX_VCPU_IDS); /* * Vector sent to a given vcpu, only valid when * the vcpu's bit in map is set */ - u8 vectors[KVM_MAX_VCPU_ID]; + u8 vectors[KVM_MAX_VCPU_IDS]; }; diff --git a/arch/x86/kvm/irq_comm.c b/arch/x86/kvm/irq_comm.c index d5b72a08e5..6e0dab0432 100644 --- a/arch/x86/kvm/irq_comm.c +++ b/arch/x86/kvm/irq_comm.c @@ -24,6 +24,7 @@ #include "hyperv.h" #include "x86.h" +#include "xen.h" static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, @@ -45,9 +46,9 @@ static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, struct kvm_lapic_irq *irq, struct dest_map *dest_map) { - int i, r = -1; + int r = -1; struct kvm_vcpu *vcpu, *lowest = NULL; - unsigned long dest_vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)]; + unsigned long i, dest_vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)]; unsigned int dest_vcpus = 0; if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, dest_map)) @@ -175,6 +176,13 @@ int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, return r; break; +#ifdef CONFIG_KVM_XEN + case KVM_IRQ_ROUTING_XEN_EVTCHN: + if (!level) + return -1; + + return kvm_xen_set_evtchn_fast(e, kvm); +#endif default: break; } @@ -310,6 +318,10 @@ int kvm_set_routing_entry(struct kvm *kvm, e->hv_sint.vcpu = ue->u.hv_sint.vcpu; e->hv_sint.sint = ue->u.hv_sint.sint; break; +#ifdef CONFIG_KVM_XEN + case KVM_IRQ_ROUTING_XEN_EVTCHN: + return kvm_xen_setup_evtchn(kvm, e, ue); +#endif default: return -EINVAL; } @@ -320,7 +332,8 @@ int kvm_set_routing_entry(struct kvm *kvm, bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, struct kvm_vcpu **dest_vcpu) { - int i, r = 0; + int r = 0; + unsigned long i; struct kvm_vcpu *vcpu; if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu)) diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h index 90e1ffdc05..3febc34236 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/kvm_cache_regs.h @@ -9,6 +9,12 @@ (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR \ | X86_CR4_OSXMMEXCPT | X86_CR4_PGE | X86_CR4_TSD | X86_CR4_FSGSBASE) +#define X86_CR0_PDPTR_BITS (X86_CR0_CD | X86_CR0_NW | X86_CR0_PG) +#define X86_CR4_TLBFLUSH_BITS (X86_CR4_PGE | X86_CR4_PCIDE | X86_CR4_PAE | X86_CR4_SMEP) +#define X86_CR4_PDPTR_BITS (X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_SMEP) + +static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS)); + #define BUILD_KVM_GPR_ACCESSORS(lname, uname) \ static __always_inline unsigned long kvm_##lname##_read(struct kvm_vcpu *vcpu)\ { \ @@ -37,6 +43,13 @@ BUILD_KVM_GPR_ACCESSORS(r14, R14) BUILD_KVM_GPR_ACCESSORS(r15, R15) #endif +/* + * avail dirty + * 0 0 register in VMCS/VMCB + * 0 1 *INVALID* + * 1 0 register in vcpu->arch + * 1 1 register in vcpu->arch, needs to be stored back + */ static inline bool kvm_register_is_available(struct kvm_vcpu *vcpu, enum kvm_reg reg) { @@ -55,13 +68,6 @@ static inline void kvm_register_mark_available(struct kvm_vcpu *vcpu, __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail); } -static inline void kvm_register_clear_available(struct kvm_vcpu *vcpu, - enum kvm_reg reg) -{ - __clear_bit(reg, (unsigned long *)&vcpu->arch.regs_avail); - __clear_bit(reg, (unsigned long *)&vcpu->arch.regs_dirty); -} - static inline void kvm_register_mark_dirty(struct kvm_vcpu *vcpu, enum kvm_reg reg) { diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 68b420289d..39eded2426 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -369,6 +369,7 @@ struct x86_emulate_ctxt { struct fetch_cache fetch; struct read_cache io_read; struct read_cache mem_read; + bool is_branch; }; /* Repeat String Operation Prefix */ diff --git a/arch/x86/kvm/kvm_onhyperv.c b/arch/x86/kvm/kvm_onhyperv.c index c7db2df50a..b469f45e3f 100644 --- a/arch/x86/kvm/kvm_onhyperv.c +++ b/arch/x86/kvm/kvm_onhyperv.c @@ -33,7 +33,8 @@ int hv_remote_flush_tlb_with_range(struct kvm *kvm, { struct kvm_arch *kvm_arch = &kvm->arch; struct kvm_vcpu *vcpu; - int ret = 0, i, nr_unique_valid_roots; + int ret = 0, nr_unique_valid_roots; + unsigned long i; hpa_t root; spin_lock(&kvm_arch->hv_root_tdp_lock); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 91c2dc9f19..9322e6340a 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -185,7 +185,7 @@ void kvm_recalculate_apic_map(struct kvm *kvm) { struct kvm_apic_map *new, *old = NULL; struct kvm_vcpu *vcpu; - int i; + unsigned long i; u32 max_id = 255; /* enough space for any xAPIC ID */ /* Read kvm->arch.apic_map_dirty before kvm->arch.apic_map. */ @@ -673,15 +673,6 @@ static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu) return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED; } -static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu) -{ - u8 val; - if (pv_eoi_get_user(vcpu, &val) < 0) - return false; - - return val & KVM_PV_EOI_ENABLED; -} - static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) { if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) @@ -690,12 +681,26 @@ static void pv_eoi_set_pending(struct kvm_vcpu *vcpu) __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); } -static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu) +static bool pv_eoi_test_and_clr_pending(struct kvm_vcpu *vcpu) { - if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) - return; + u8 val; + if (pv_eoi_get_user(vcpu, &val) < 0) + return false; + + val &= KVM_PV_EOI_ENABLED; + + if (val && pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) + return false; + + /* + * Clear pending bit in any case: it will be set again on vmentry. + * While this might not be ideal from performance point of view, + * this makes sure pv eoi is only enabled when we know it's safe. + */ __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); + + return val; } static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) @@ -1091,11 +1096,8 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, apic->regs + APIC_TMR); } - if (static_call(kvm_x86_deliver_posted_interrupt)(vcpu, vector)) { - kvm_lapic_set_irr(vector, apic); - kvm_make_request(KVM_REQ_EVENT, vcpu); - kvm_vcpu_kick(vcpu); - } + static_call(kvm_x86_deliver_interrupt)(apic, delivery_mode, + trig_mode, vector); break; case APIC_DM_REMRD: @@ -1166,8 +1168,8 @@ void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq, struct kvm_lapic *src = NULL; struct kvm_apic_map *map; struct kvm_vcpu *vcpu; - unsigned long bitmap; - int i, vcpu_idx; + unsigned long bitmap, i; + int vcpu_idx; bool ret; rcu_read_lock(); @@ -1925,7 +1927,7 @@ void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) /* If the preempt notifier has already run, it also called apic_timer_expired */ if (!apic->lapic_timer.hv_timer_in_use) goto out; - WARN_ON(rcuwait_active(&vcpu->wait)); + WARN_ON(kvm_vcpu_is_blocking(vcpu)); apic_timer_expired(apic, false); cancel_hv_timer(apic); @@ -1942,7 +1944,6 @@ void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu) { restart_apic_timer(vcpu->arch.apic); } -EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer); void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) { @@ -1954,7 +1955,6 @@ void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu) start_sw_timer(apic); preempt_enable(); } -EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer); void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) { @@ -2306,7 +2306,12 @@ void kvm_apic_update_apicv(struct kvm_vcpu *vcpu) apic->irr_pending = true; apic->isr_count = 1; } else { - apic->irr_pending = (apic_search_irr(apic) != -1); + /* + * Don't clear irr_pending, searching the IRR can race with + * updates from the CPU as APICv is still active from hardware's + * perspective. The flag will be cleared as appropriate when + * KVM injects the interrupt. + */ apic->isr_count = count_vectors(apic->regs + APIC_ISR); } } @@ -2671,7 +2676,6 @@ void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, struct kvm_lapic *apic) { - bool pending; int vector; /* * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host @@ -2685,14 +2689,8 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, * -> host enabled PV EOI, guest executed EOI. */ BUG_ON(!pv_eoi_enabled(vcpu)); - pending = pv_eoi_get_pending(vcpu); - /* - * Clear pending bit in any case: it will be set again on vmentry. - * While this might not be ideal from performance point of view, - * this makes sure pv eoi is only enabled when we know it's safe. - */ - pv_eoi_clr_pending(vcpu); - if (pending) + + if (pv_eoi_test_and_clr_pending(vcpu)) return; vector = apic_set_eoi(apic); trace_kvm_pv_eoi(apic, vector); @@ -2850,25 +2848,30 @@ int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data) return 0; } -int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len) +int kvm_lapic_set_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len) { u64 addr = data & ~KVM_MSR_ENABLED; struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data; unsigned long new_len; + int ret; if (!IS_ALIGNED(addr, 4)) return 1; + if (data & KVM_MSR_ENABLED) { + if (addr == ghc->gpa && len <= ghc->len) + new_len = ghc->len; + else + new_len = len; + + ret = kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); + if (ret) + return ret; + } + vcpu->arch.pv_eoi.msr_val = data; - if (!pv_eoi_enabled(vcpu)) - return 0; - if (addr == ghc->gpa && len <= ghc->len) - new_len = ghc->len; - else - new_len = len; - - return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); + return 0; } int kvm_apic_accept_events(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index d7c25d0c13..2b44e533fc 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -127,7 +127,7 @@ int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data); int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data); int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data); -int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len); +int kvm_lapic_set_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len); void kvm_lapic_exit(void); #define VEC_POS(v) ((v) & (32 - 1)) diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index e9688a9f7b..e9fbb2c8bb 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -44,9 +44,8 @@ #define PT32_ROOT_LEVEL 2 #define PT32E_ROOT_LEVEL 3 -#define KVM_MMU_CR4_ROLE_BITS (X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE | \ - X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE | \ - X86_CR4_LA57) +#define KVM_MMU_CR4_ROLE_BITS (X86_CR4_PSE | X86_CR4_PAE | X86_CR4_LA57 | \ + X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE) #define KVM_MMU_CR0_ROLE_BITS (X86_CR0_PG | X86_CR0_WP) @@ -72,7 +71,8 @@ void kvm_init_mmu(struct kvm_vcpu *vcpu); void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, unsigned long cr4, u64 efer, gpa_t nested_cr3); void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, - bool accessed_dirty, gpa_t new_eptp); + int huge_page_level, bool accessed_dirty, + gpa_t new_eptp); bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu); int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, u64 fault_address, char *insn, int insn_len); @@ -80,6 +80,7 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, int kvm_mmu_load(struct kvm_vcpu *vcpu); void kvm_mmu_unload(struct kvm_vcpu *vcpu); void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu); +void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu); static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) { @@ -114,17 +115,91 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu) vcpu->arch.mmu->shadow_root_level); } -int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, - bool prefault); +struct kvm_page_fault { + /* arguments to kvm_mmu_do_page_fault. */ + const gpa_t addr; + const u32 error_code; + const bool prefetch; + + /* Derived from error_code. */ + const bool exec; + const bool write; + const bool present; + const bool rsvd; + const bool user; + + /* Derived from mmu and global state. */ + const bool is_tdp; + const bool nx_huge_page_workaround_enabled; + + /* + * Whether a >4KB mapping can be created or is forbidden due to NX + * hugepages. + */ + bool huge_page_disallowed; + + /* + * Maximum page size that can be created for this fault; input to + * FNAME(fetch), __direct_map and kvm_tdp_mmu_map. + */ + u8 max_level; + + /* + * Page size that can be created based on the max_level and the + * page size used by the host mapping. + */ + u8 req_level; + + /* + * Page size that will be created based on the req_level and + * huge_page_disallowed. + */ + u8 goal_level; + + /* Shifted addr, or result of guest page table walk if addr is a gva. */ + gfn_t gfn; + + /* The memslot containing gfn. May be NULL. */ + struct kvm_memory_slot *slot; + + /* Outputs of kvm_faultin_pfn. */ + kvm_pfn_t pfn; + hva_t hva; + bool map_writable; +}; + +int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); + +extern int nx_huge_pages; +static inline bool is_nx_huge_page_enabled(void) +{ + return READ_ONCE(nx_huge_pages); +} static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, - u32 err, bool prefault) + u32 err, bool prefetch) { + struct kvm_page_fault fault = { + .addr = cr2_or_gpa, + .error_code = err, + .exec = err & PFERR_FETCH_MASK, + .write = err & PFERR_WRITE_MASK, + .present = err & PFERR_PRESENT_MASK, + .rsvd = err & PFERR_RSVD_MASK, + .user = err & PFERR_USER_MASK, + .prefetch = prefetch, + .is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault), + .nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(), + + .max_level = KVM_MAX_HUGEPAGE_LEVEL, + .req_level = PG_LEVEL_4K, + .goal_level = PG_LEVEL_4K, + }; #ifdef CONFIG_RETPOLINE - if (likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault)) - return kvm_tdp_page_fault(vcpu, cr2_or_gpa, err, prefault); + if (fault.is_tdp) + return kvm_tdp_page_fault(vcpu, &fault); #endif - return vcpu->arch.mmu->page_fault(vcpu, cr2_or_gpa, err, prefault); + return vcpu->arch.mmu->page_fault(vcpu, &fault); } /* @@ -230,14 +305,26 @@ int kvm_arch_write_log_dirty(struct kvm_vcpu *vcpu); int kvm_mmu_post_init_vm(struct kvm *kvm); void kvm_mmu_pre_destroy_vm(struct kvm *kvm); -static inline bool kvm_memslots_have_rmaps(struct kvm *kvm) +static inline bool kvm_shadow_root_allocated(struct kvm *kvm) { /* - * Read memslot_have_rmaps before rmap pointers. Hence, threads reading - * memslots_have_rmaps in any lock context are guaranteed to see the - * pointers. Pairs with smp_store_release in alloc_all_memslots_rmaps. + * Read shadow_root_allocated before related pointers. Hence, threads + * reading shadow_root_allocated in any lock context are guaranteed to + * see the pointers. Pairs with smp_store_release in + * mmu_first_shadow_root_alloc. */ - return smp_load_acquire(&kvm->arch.memslots_have_rmaps); + return smp_load_acquire(&kvm->arch.shadow_root_allocated); +} + +#ifdef CONFIG_X86_64 +static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return kvm->arch.tdp_mmu_enabled; } +#else +static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return false; } +#endif + +static inline bool kvm_memslots_have_rmaps(struct kvm *kvm) +{ + return !is_tdp_mmu_enabled(kvm) || kvm_shadow_root_allocated(kvm); } static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level) @@ -265,4 +352,17 @@ static inline void kvm_update_page_stats(struct kvm *kvm, int level, int count) { atomic64_add(count, &kvm->stat.pages[level - 1]); } + +gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, + struct x86_exception *exception); + +static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu, + struct kvm_mmu *mmu, + gpa_t gpa, u32 access, + struct x86_exception *exception) +{ + if (mmu != &vcpu->arch.nested_mmu) + return gpa; + return translate_nested_gpa(vcpu, gpa, access, exception); +} #endif diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index d6579bae25..5628d0ba63 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -58,6 +58,7 @@ extern bool itlb_multihit_kvm_mitigation; int __read_mostly nx_huge_pages = -1; +static uint __read_mostly nx_huge_pages_recovery_period_ms; #ifdef CONFIG_PREEMPT_RT /* Recovery can cause latency spikes, disable it for PREEMPT_RT. */ static uint __read_mostly nx_huge_pages_recovery_ratio = 0; @@ -66,23 +67,26 @@ static uint __read_mostly nx_huge_pages_recovery_ratio = 60; #endif static int set_nx_huge_pages(const char *val, const struct kernel_param *kp); -static int set_nx_huge_pages_recovery_ratio(const char *val, const struct kernel_param *kp); +static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp); static const struct kernel_param_ops nx_huge_pages_ops = { .set = set_nx_huge_pages, .get = param_get_bool, }; -static const struct kernel_param_ops nx_huge_pages_recovery_ratio_ops = { - .set = set_nx_huge_pages_recovery_ratio, +static const struct kernel_param_ops nx_huge_pages_recovery_param_ops = { + .set = set_nx_huge_pages_recovery_param, .get = param_get_uint, }; module_param_cb(nx_huge_pages, &nx_huge_pages_ops, &nx_huge_pages, 0644); __MODULE_PARM_TYPE(nx_huge_pages, "bool"); -module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_ratio_ops, +module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_param_ops, &nx_huge_pages_recovery_ratio, 0644); __MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint"); +module_param_cb(nx_huge_pages_recovery_period_ms, &nx_huge_pages_recovery_param_ops, + &nx_huge_pages_recovery_period_ms, 0644); +__MODULE_PARM_TYPE(nx_huge_pages_recovery_period_ms, "uint"); static bool __read_mostly force_flush_and_sync_on_reuse; module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644); @@ -331,12 +335,6 @@ static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte) return likely(kvm_gen == spte_gen); } -static gpa_t translate_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, - struct x86_exception *exception) -{ - return gpa; -} - static int is_cpuid_PSE36(void) { return 1; @@ -1071,20 +1069,6 @@ static bool rmap_can_add(struct kvm_vcpu *vcpu) return kvm_mmu_memory_cache_nr_free_objects(mc); } -static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) -{ - struct kvm_memory_slot *slot; - struct kvm_mmu_page *sp; - struct kvm_rmap_head *rmap_head; - - sp = sptep_to_sp(spte); - kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn); - slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); - rmap_head = gfn_to_rmap(gfn, sp->role.level, slot); - return pte_list_add(vcpu, spte, rmap_head); -} - - static void rmap_remove(struct kvm *kvm, u64 *spte) { struct kvm_memslots *slots; @@ -1097,9 +1081,9 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt); /* - * Unlike rmap_add and rmap_recycle, rmap_remove does not run in the - * context of a vCPU so have to determine which memslots to use based - * on context information in sp->role. + * Unlike rmap_add, rmap_remove does not run in the context of a vCPU + * so we have to determine which memslots to use based on context + * information in sp->role. */ slots = kvm_memslots_for_spte_role(kvm, sp->role); @@ -1464,7 +1448,7 @@ static bool kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, { u64 *sptep; struct rmap_iterator iter; - int need_flush = 0; + bool need_flush = false; u64 new_spte; kvm_pfn_t new_pfn; @@ -1476,7 +1460,7 @@ static bool kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, rmap_printk("spte %p %llx gfn %llx (%d)\n", sptep, *sptep, gfn, level); - need_flush = 1; + need_flush = true; if (pte_write(pte)) { pte_list_remove(kvm, rmap_head, sptep); @@ -1492,7 +1476,7 @@ static bool kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, if (need_flush && kvm_available_flush_tlb_with_range()) { kvm_flush_remote_tlbs_with_address(kvm, gfn, 1); - return 0; + return false; } return need_flush; @@ -1633,25 +1617,29 @@ static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, for_each_rmap_spte(rmap_head, &iter, sptep) if (is_accessed_spte(*sptep)) - return 1; - return 0; + return true; + return false; } #define RMAP_RECYCLE_THRESHOLD 1000 -static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) +static void rmap_add(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, + u64 *spte, gfn_t gfn) { - struct kvm_memory_slot *slot; - struct kvm_rmap_head *rmap_head; struct kvm_mmu_page *sp; + struct kvm_rmap_head *rmap_head; + int rmap_count; sp = sptep_to_sp(spte); - slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); + kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn); rmap_head = gfn_to_rmap(gfn, sp->role.level, slot); + rmap_count = pte_list_add(vcpu, spte, rmap_head); - kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, __pte(0)); - kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn, - KVM_PAGES_PER_HPAGE(sp->role.level)); + if (rmap_count > RMAP_RECYCLE_THRESHOLD) { + kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, __pte(0)); + kvm_flush_remote_tlbs_with_address( + vcpu->kvm, sp->gfn, KVM_PAGES_PER_HPAGE(sp->role.level)); + } } bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) @@ -1795,7 +1783,7 @@ static void mark_unsync(u64 *spte) static int nonpaging_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { - return 0; + return -1; } #define KVM_PAGE_ARRAY_NR 16 @@ -1909,12 +1897,14 @@ static void kvm_mmu_commit_zap_page(struct kvm *kvm, static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, struct list_head *invalid_list) { - if (vcpu->arch.mmu->sync_page(vcpu, sp) == 0) { + int ret = vcpu->arch.mmu->sync_page(vcpu, sp); + + if (ret < 0) { kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); return false; } - return true; + return !!ret; } static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm, @@ -1931,17 +1921,6 @@ static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm, return true; } -static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu, - struct list_head *invalid_list, - bool remote_flush, bool local_flush) -{ - if (kvm_mmu_remote_flush_or_zap(vcpu->kvm, invalid_list, remote_flush)) - return; - - if (local_flush) - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); -} - #ifdef CONFIG_KVM_MMU_AUDIT #include "mmu_audit.c" #else @@ -1951,7 +1930,11 @@ static void mmu_audit_disable(void) { } static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp) { - return sp->role.invalid || + if (sp->role.invalid) + return true; + + /* TDP MMU pages due not use the MMU generation. */ + return !sp->tdp_mmu_page && unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen); } @@ -2044,7 +2027,7 @@ static int mmu_sync_children(struct kvm_vcpu *vcpu, protected |= rmap_write_protect(vcpu, sp->gfn); if (protected) { - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, true); flush = false; } @@ -2054,7 +2037,7 @@ static int mmu_sync_children(struct kvm_vcpu *vcpu, mmu_pages_clear_parents(&parents); } if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) { - kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); + kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush); if (!can_yield) { kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); return -EINTR; @@ -2065,7 +2048,7 @@ static int mmu_sync_children(struct kvm_vcpu *vcpu, } } - kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); + kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush); return 0; } @@ -2097,10 +2080,8 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, role = vcpu->arch.mmu->mmu_role.base; role.level = level; role.direct = direct; - if (role.direct) - role.gpte_is_8_bytes = true; role.access = access; - if (!direct_mmu && vcpu->arch.mmu->root_level <= PT32_ROOT_LEVEL) { + if (role.has_4_byte_gpte) { quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; role.quadrant = quadrant; @@ -2149,7 +2130,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, break; WARN_ON(!list_empty(&invalid_list)); - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); + kvm_flush_remote_tlbs(vcpu->kvm); } __clear_sp_write_flooding_count(sp); @@ -2229,7 +2210,7 @@ static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator) static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator, u64 spte) { - if (is_last_spte(spte, iterator->level)) { + if (!is_shadow_present_pte(spte) || is_last_spte(spte, iterator->level)) { iterator->level = 0; return; } @@ -2576,10 +2557,10 @@ static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) return r; } -static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) +static void kvm_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp) { trace_kvm_mmu_unsync_page(sp); - ++vcpu->kvm->stat.mmu_unsync; + ++kvm->stat.mmu_unsync; sp->unsync = 1; kvm_mmu_mark_parents_unsync(sp); @@ -2591,7 +2572,8 @@ static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) * were marked unsync (or if there is no shadow page), -EPERM if the SPTE must * be write-protected. */ -int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync) +int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, + gfn_t gfn, bool can_unsync, bool prefetch) { struct kvm_mmu_page *sp; bool locked = false; @@ -2601,7 +2583,7 @@ int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync) * track machinery is used to write-protect upper-level shadow pages, * i.e. this guards the role.level == 4K assertion below! */ - if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE)) + if (kvm_slot_page_track_is_active(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE)) return -EPERM; /* @@ -2610,13 +2592,16 @@ int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync) * that case, KVM must complete emulation of the guest TLB flush before * allowing shadow pages to become unsync (writable by the guest). */ - for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { + for_each_gfn_indirect_valid_sp(kvm, sp, gfn) { if (!can_unsync) return -EPERM; if (sp->unsync) continue; + if (prefetch) + return -EEXIST; + /* * TDP MMU page faults require an additional spinlock as they * run with mmu_lock held for read, not write, and the unsync @@ -2626,7 +2611,7 @@ int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync) */ if (!locked) { locked = true; - spin_lock(&vcpu->kvm->arch.mmu_unsync_pages_lock); + spin_lock(&kvm->arch.mmu_unsync_pages_lock); /* * Recheck after taking the spinlock, a different vCPU @@ -2641,10 +2626,10 @@ int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync) } WARN_ON(sp->role.level != PG_LEVEL_4K); - kvm_unsync_page(vcpu, sp); + kvm_unsync_page(kvm, sp); } if (locked) - spin_unlock(&vcpu->kvm->arch.mmu_unsync_pages_lock); + spin_unlock(&kvm->arch.mmu_unsync_pages_lock); /* * We need to ensure that the marking of unsync pages is visible @@ -2680,48 +2665,30 @@ int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync) * (sp->unsync = true) * * The write barrier below ensures that 1.1 happens before 1.2 and thus - * the situation in 2.4 does not arise. The implicit barrier in 2.2 - * pairs with this write barrier. + * the situation in 2.4 does not arise. It pairs with the read barrier + * in is_unsync_root(), placed between 2.1's load of SPTE.W and 2.3. */ smp_wmb(); return 0; } -static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, - unsigned int pte_access, int level, - gfn_t gfn, kvm_pfn_t pfn, bool speculative, - bool can_unsync, bool host_writable) -{ - u64 spte; - struct kvm_mmu_page *sp; - int ret; - - sp = sptep_to_sp(sptep); - - ret = make_spte(vcpu, pte_access, level, gfn, pfn, *sptep, speculative, - can_unsync, host_writable, sp_ad_disabled(sp), &spte); - - if (spte & PT_WRITABLE_MASK) - kvm_vcpu_mark_page_dirty(vcpu, gfn); - - if (*sptep == spte) - ret |= SET_SPTE_SPURIOUS; - else if (mmu_spte_update(sptep, spte)) - ret |= SET_SPTE_NEED_REMOTE_TLB_FLUSH; - return ret; -} - -static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, - unsigned int pte_access, bool write_fault, int level, - gfn_t gfn, kvm_pfn_t pfn, bool speculative, - bool host_writable) +static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, + u64 *sptep, unsigned int pte_access, gfn_t gfn, + kvm_pfn_t pfn, struct kvm_page_fault *fault) { + struct kvm_mmu_page *sp = sptep_to_sp(sptep); + int level = sp->role.level; int was_rmapped = 0; - int rmap_count; - int set_spte_ret; int ret = RET_PF_FIXED; bool flush = false; + bool wrprot; + u64 spte; + + /* Prefetching always gets a writable pfn. */ + bool host_writable = !fault || fault->map_writable; + bool prefetch = !fault || fault->prefetch; + bool write_fault = fault && fault->write; pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__, *sptep, write_fault, gfn); @@ -2752,52 +2719,36 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, was_rmapped = 1; } - set_spte_ret = set_spte(vcpu, sptep, pte_access, level, gfn, pfn, - speculative, true, host_writable); - if (set_spte_ret & SET_SPTE_WRITE_PROTECTED_PT) { - if (write_fault) - ret = RET_PF_EMULATE; - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); + wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch, + true, host_writable, &spte); + + if (*sptep == spte) { + ret = RET_PF_SPURIOUS; + } else { + trace_kvm_mmu_set_spte(level, gfn, sptep); + flush |= mmu_spte_update(sptep, spte); } - if (set_spte_ret & SET_SPTE_NEED_REMOTE_TLB_FLUSH || flush) + if (wrprot) { + if (write_fault) + ret = RET_PF_EMULATE; + } + + if (flush) kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, KVM_PAGES_PER_HPAGE(level)); - /* - * The fault is fully spurious if and only if the new SPTE and old SPTE - * are identical, and emulation is not required. - */ - if ((set_spte_ret & SET_SPTE_SPURIOUS) && ret == RET_PF_FIXED) { - WARN_ON_ONCE(!was_rmapped); - return RET_PF_SPURIOUS; - } - pgprintk("%s: setting spte %llx\n", __func__, *sptep); - trace_kvm_mmu_set_spte(level, gfn, sptep); if (!was_rmapped) { + WARN_ON_ONCE(ret == RET_PF_SPURIOUS); kvm_update_page_stats(vcpu->kvm, level, 1); - rmap_count = rmap_add(vcpu, sptep, gfn); - if (rmap_count > RMAP_RECYCLE_THRESHOLD) - rmap_recycle(vcpu, sptep, gfn); + rmap_add(vcpu, slot, sptep, gfn); } return ret; } -static kvm_pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, - bool no_dirty_log) -{ - struct kvm_memory_slot *slot; - - slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log); - if (!slot) - return KVM_PFN_ERR_FAULT; - - return gfn_to_pfn_memslot_atomic(slot, gfn); -} - static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *start, u64 *end) @@ -2818,8 +2769,8 @@ static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, return -1; for (i = 0; i < ret; i++, gfn++, start++) { - mmu_set_spte(vcpu, start, access, false, sp->role.level, gfn, - page_to_pfn(pages[i]), true, true); + mmu_set_spte(vcpu, slot, start, access, gfn, + page_to_pfn(pages[i]), NULL); put_page(pages[i]); } @@ -2842,11 +2793,13 @@ static void __direct_pte_prefetch(struct kvm_vcpu *vcpu, if (!start) continue; if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0) - break; + return; start = NULL; } else if (!start) start = spte; } + if (start) + direct_pte_prefetch_many(vcpu, sp, start, spte); } static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) @@ -2924,52 +2877,46 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm, return min(host_level, max_level); } -int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, - int max_level, kvm_pfn_t *pfnp, - bool huge_page_disallowed, int *req_level) +void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { - struct kvm_memory_slot *slot; - kvm_pfn_t pfn = *pfnp; + struct kvm_memory_slot *slot = fault->slot; kvm_pfn_t mask; - int level; - *req_level = PG_LEVEL_4K; + fault->huge_page_disallowed = fault->exec && fault->nx_huge_page_workaround_enabled; - if (unlikely(max_level == PG_LEVEL_4K)) - return PG_LEVEL_4K; + if (unlikely(fault->max_level == PG_LEVEL_4K)) + return; - if (is_error_noslot_pfn(pfn) || kvm_is_reserved_pfn(pfn)) - return PG_LEVEL_4K; + if (is_error_noslot_pfn(fault->pfn) || kvm_is_reserved_pfn(fault->pfn)) + return; - slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, true); - if (!slot) - return PG_LEVEL_4K; + if (kvm_slot_dirty_track_enabled(slot)) + return; /* * Enforce the iTLB multihit workaround after capturing the requested * level, which will be used to do precise, accurate accounting. */ - *req_level = level = kvm_mmu_max_mapping_level(vcpu->kvm, slot, gfn, pfn, max_level); - if (level == PG_LEVEL_4K || huge_page_disallowed) - return PG_LEVEL_4K; + fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, slot, + fault->gfn, fault->pfn, + fault->max_level); + if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed) + return; /* * mmu_notifier_retry() was successful and mmu_lock is held, so * the pmd can't be split from under us. */ - mask = KVM_PAGES_PER_HPAGE(level) - 1; - VM_BUG_ON((gfn & mask) != (pfn & mask)); - *pfnp = pfn & ~mask; - - return level; + fault->goal_level = fault->req_level; + mask = KVM_PAGES_PER_HPAGE(fault->goal_level) - 1; + VM_BUG_ON((fault->gfn & mask) != (fault->pfn & mask)); + fault->pfn &= ~mask; } -void disallowed_hugepage_adjust(u64 spte, gfn_t gfn, int cur_level, - kvm_pfn_t *pfnp, int *goal_levelp) +void disallowed_hugepage_adjust(struct kvm_page_fault *fault, u64 spte, int cur_level) { - int level = *goal_levelp; - - if (cur_level == level && level > PG_LEVEL_4K && + if (cur_level > PG_LEVEL_4K && + cur_level == fault->goal_level && is_shadow_present_pte(spte) && !is_large_pte(spte)) { /* @@ -2979,42 +2926,33 @@ void disallowed_hugepage_adjust(u64 spte, gfn_t gfn, int cur_level, * patching back for them into pfn the next 9 bits of * the address. */ - u64 page_mask = KVM_PAGES_PER_HPAGE(level) - - KVM_PAGES_PER_HPAGE(level - 1); - *pfnp |= gfn & page_mask; - (*goal_levelp)--; + u64 page_mask = KVM_PAGES_PER_HPAGE(cur_level) - + KVM_PAGES_PER_HPAGE(cur_level - 1); + fault->pfn |= fault->gfn & page_mask; + fault->goal_level--; } } -static int __direct_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, - int map_writable, int max_level, kvm_pfn_t pfn, - bool prefault, bool is_tdp) +static int __direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { - bool nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(); - bool write = error_code & PFERR_WRITE_MASK; - bool exec = error_code & PFERR_FETCH_MASK; - bool huge_page_disallowed = exec && nx_huge_page_workaround_enabled; struct kvm_shadow_walk_iterator it; struct kvm_mmu_page *sp; - int level, req_level, ret; - gfn_t gfn = gpa >> PAGE_SHIFT; - gfn_t base_gfn = gfn; + int ret; + gfn_t base_gfn = fault->gfn; - level = kvm_mmu_hugepage_adjust(vcpu, gfn, max_level, &pfn, - huge_page_disallowed, &req_level); + kvm_mmu_hugepage_adjust(vcpu, fault); - trace_kvm_mmu_spte_requested(gpa, level, pfn); - for_each_shadow_entry(vcpu, gpa, it) { + trace_kvm_mmu_spte_requested(fault); + for_each_shadow_entry(vcpu, fault->addr, it) { /* * We cannot overwrite existing page tables with an NX * large page, as the leaf could be executable. */ - if (nx_huge_page_workaround_enabled) - disallowed_hugepage_adjust(*it.sptep, gfn, it.level, - &pfn, &level); + if (fault->nx_huge_page_workaround_enabled) + disallowed_hugepage_adjust(fault, *it.sptep, it.level); - base_gfn = gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1); - if (it.level == level) + base_gfn = fault->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1); + if (it.level == fault->goal_level) break; drop_large_spte(vcpu, it.sptep); @@ -3025,14 +2963,16 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, it.level - 1, true, ACC_ALL); link_shadow_page(vcpu, it.sptep, sp); - if (is_tdp && huge_page_disallowed && - req_level >= it.level) + if (fault->is_tdp && fault->huge_page_disallowed && + fault->req_level >= it.level) account_huge_nx_page(vcpu->kvm, sp); } - ret = mmu_set_spte(vcpu, it.sptep, ACC_ALL, - write, level, base_gfn, pfn, prefault, - map_writable); + if (WARN_ON_ONCE(it.level != fault->goal_level)) + return -EFAULT; + + ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL, + base_gfn, fault->pfn, fault); if (ret == RET_PF_SPURIOUS) return ret; @@ -3064,18 +3004,19 @@ static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) return -EFAULT; } -static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, - kvm_pfn_t pfn, unsigned int access, - int *ret_val) +static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, + unsigned int access, int *ret_val) { /* The pfn is invalid, report the error! */ - if (unlikely(is_error_pfn(pfn))) { - *ret_val = kvm_handle_bad_page(vcpu, gfn, pfn); + if (unlikely(is_error_pfn(fault->pfn))) { + *ret_val = kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn); return true; } - if (unlikely(is_noslot_pfn(pfn))) { - vcpu_cache_mmio_info(vcpu, gva, gfn, + if (unlikely(!fault->slot)) { + gva_t gva = fault->is_tdp ? 0 : fault->addr; + + vcpu_cache_mmio_info(vcpu, gva, fault->gfn, access & shadow_mmio_access_mask); /* * If MMIO caching is disabled, emulate immediately without @@ -3091,18 +3032,17 @@ static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, return false; } -static bool page_fault_can_be_fast(u32 error_code) +static bool page_fault_can_be_fast(struct kvm_page_fault *fault) { /* * Do not fix the mmio spte with invalid generation number which * need to be updated by slow page fault path. */ - if (unlikely(error_code & PFERR_RSVD_MASK)) + if (fault->rsvd) return false; /* See if the page fault is due to an NX violation */ - if (unlikely(((error_code & (PFERR_FETCH_MASK | PFERR_PRESENT_MASK)) - == (PFERR_FETCH_MASK | PFERR_PRESENT_MASK)))) + if (unlikely(fault->exec && fault->present)) return false; /* @@ -3119,9 +3059,7 @@ static bool page_fault_can_be_fast(u32 error_code) * accesses to a present page. */ - return shadow_acc_track_mask != 0 || - ((error_code & (PFERR_WRITE_MASK | PFERR_PRESENT_MASK)) - == (PFERR_WRITE_MASK | PFERR_PRESENT_MASK)); + return shadow_acc_track_mask != 0 || (fault->write && fault->present); } /* @@ -3129,13 +3067,9 @@ static bool page_fault_can_be_fast(u32 error_code) * someone else modified the SPTE from its original value. */ static bool -fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, +fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, u64 *sptep, u64 old_spte, u64 new_spte) { - gfn_t gfn; - - WARN_ON(!sp->role.direct); - /* * Theoretically we could also set dirty bit (and flush TLB) here in * order to eliminate unnecessary PML logging. See comments in @@ -3151,24 +3085,18 @@ fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, if (cmpxchg64(sptep, old_spte, new_spte) != old_spte) return false; - if (is_writable_pte(new_spte) && !is_writable_pte(old_spte)) { - /* - * The gfn of direct spte is stable since it is - * calculated by sp->gfn. - */ - gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); - kvm_vcpu_mark_page_dirty(vcpu, gfn); - } + if (is_writable_pte(new_spte) && !is_writable_pte(old_spte)) + mark_page_dirty_in_slot(vcpu->kvm, fault->slot, fault->gfn); return true; } -static bool is_access_allowed(u32 fault_err_code, u64 spte) +static bool is_access_allowed(struct kvm_page_fault *fault, u64 spte) { - if (fault_err_code & PFERR_FETCH_MASK) + if (fault->exec) return is_executable_pte(spte); - if (fault_err_code & PFERR_WRITE_MASK) + if (fault->write) return is_writable_pte(spte); /* Fault was on Read access */ @@ -3193,9 +3121,6 @@ static u64 *fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gpa_t gpa, u64 *spte) for_each_shadow_entry_lockless(vcpu, gpa, iterator, old_spte) { sptep = iterator.sptep; *spte = old_spte; - - if (!is_shadow_present_pte(old_spte)) - break; } return sptep; @@ -3204,7 +3129,7 @@ static u64 *fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gpa_t gpa, u64 *spte) /* * Returns one of RET_PF_INVALID, RET_PF_FIXED or RET_PF_SPURIOUS. */ -static int fast_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code) +static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { struct kvm_mmu_page *sp; int ret = RET_PF_INVALID; @@ -3212,7 +3137,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code) u64 *sptep = NULL; uint retry_count = 0; - if (!page_fault_can_be_fast(error_code)) + if (!page_fault_can_be_fast(fault)) return ret; walk_shadow_page_lockless_begin(vcpu); @@ -3221,9 +3146,9 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code) u64 new_spte; if (is_tdp_mmu(vcpu->arch.mmu)) - sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, gpa, &spte); + sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, fault->addr, &spte); else - sptep = fast_pf_get_last_sptep(vcpu, gpa, &spte); + sptep = fast_pf_get_last_sptep(vcpu, fault->addr, &spte); if (!is_shadow_present_pte(spte)) break; @@ -3242,7 +3167,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code) * Need not check the access of upper level table entries since * they are always ACC_ALL. */ - if (is_access_allowed(error_code, spte)) { + if (is_access_allowed(fault, spte)) { ret = RET_PF_SPURIOUS; break; } @@ -3257,28 +3182,28 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code) * be removed in the fast path only if the SPTE was * write-protected for dirty-logging or access tracking. */ - if ((error_code & PFERR_WRITE_MASK) && + if (fault->write && spte_can_locklessly_be_made_writable(spte)) { new_spte |= PT_WRITABLE_MASK; /* - * Do not fix write-permission on the large spte. Since - * we only dirty the first page into the dirty-bitmap in + * Do not fix write-permission on the large spte when + * dirty logging is enabled. Since we only dirty the + * first page into the dirty-bitmap in * fast_pf_fix_direct_spte(), other pages are missed * if its slot has dirty logging enabled. * * Instead, we let the slow page fault path create a * normal spte to fix the access. - * - * See the comments in kvm_arch_commit_memory_region(). */ - if (sp->role.level > PG_LEVEL_4K) + if (sp->role.level > PG_LEVEL_4K && + kvm_slot_dirty_track_enabled(fault->slot)) break; } /* Verify that the fault can be handled in the fast path */ if (new_spte == spte || - !is_access_allowed(error_code, new_spte)) + !is_access_allowed(fault, new_spte)) break; /* @@ -3286,7 +3211,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code) * since the gfn is not stable for indirect shadow page. See * Documentation/virt/kvm/locking.rst to get more detail. */ - if (fast_pf_fix_direct_spte(vcpu, sp, sptep, spte, new_spte)) { + if (fast_pf_fix_direct_spte(vcpu, fault, sptep, spte, new_spte)) { ret = RET_PF_FIXED; break; } @@ -3299,7 +3224,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code) } while (true); - trace_fast_page_fault(vcpu, gpa, error_code, sptep, spte, ret); + trace_fast_page_fault(vcpu, fault, sptep, spte, ret); walk_shadow_page_lockless_end(vcpu); return ret; @@ -3472,6 +3397,67 @@ static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) return r; } +static int mmu_first_shadow_root_alloc(struct kvm *kvm) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *slot; + int r = 0, i, bkt; + + /* + * Check if this is the first shadow root being allocated before + * taking the lock. + */ + if (kvm_shadow_root_allocated(kvm)) + return 0; + + mutex_lock(&kvm->slots_arch_lock); + + /* Recheck, under the lock, whether this is the first shadow root. */ + if (kvm_shadow_root_allocated(kvm)) + goto out_unlock; + + /* + * Check if anything actually needs to be allocated, e.g. all metadata + * will be allocated upfront if TDP is disabled. + */ + if (kvm_memslots_have_rmaps(kvm) && + kvm_page_track_write_tracking_enabled(kvm)) + goto out_success; + + for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + slots = __kvm_memslots(kvm, i); + kvm_for_each_memslot(slot, bkt, slots) { + /* + * Both of these functions are no-ops if the target is + * already allocated, so unconditionally calling both + * is safe. Intentionally do NOT free allocations on + * failure to avoid having to track which allocations + * were made now versus when the memslot was created. + * The metadata is guaranteed to be freed when the slot + * is freed, and will be kept/used if userspace retries + * KVM_RUN instead of killing the VM. + */ + r = memslot_rmap_alloc(slot, slot->npages); + if (r) + goto out_unlock; + r = kvm_page_track_write_tracking_alloc(slot); + if (r) + goto out_unlock; + } + } + + /* + * Ensure that shadow_root_allocated becomes true strictly after + * all the related pointers are set. + */ +out_success: + smp_store_release(&kvm->arch.shadow_root_allocated, true); + +out_unlock: + mutex_unlock(&kvm->slots_arch_lock); + return r; +} + static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) { struct kvm_mmu *mmu = vcpu->arch.mmu; @@ -3502,7 +3488,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) } } - r = alloc_all_memslots_rmaps(vcpu->kvm); + r = mmu_first_shadow_root_alloc(vcpu->kvm); if (r) return r; @@ -3653,6 +3639,33 @@ static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu) #endif } +static bool is_unsync_root(hpa_t root) +{ + struct kvm_mmu_page *sp; + + if (!VALID_PAGE(root)) + return false; + + /* + * The read barrier orders the CPU's read of SPTE.W during the page table + * walk before the reads of sp->unsync/sp->unsync_children here. + * + * Even if another CPU was marking the SP as unsync-ed simultaneously, + * any guest page table changes are not guaranteed to be visible anyway + * until this VCPU issues a TLB flush strictly after those changes are + * made. We only need to ensure that the other CPU sets these flags + * before any actual changes to the page tables are made. The comments + * in mmu_try_to_unsync_pages() describe what could go wrong if this + * requirement isn't satisfied. + */ + smp_rmb(); + sp = to_shadow_page(root); + if (sp->unsync || sp->unsync_children) + return true; + + return false; +} + void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) { int i; @@ -3670,18 +3683,7 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) hpa_t root = vcpu->arch.mmu->root_hpa; sp = to_shadow_page(root); - /* - * Even if another CPU was marking the SP as unsync-ed - * simultaneously, any guest page table changes are not - * guaranteed to be visible anyway until this VCPU issues a TLB - * flush strictly after those changes are made. We only need to - * ensure that the other CPU sets these flags before any actual - * changes to the page tables are made. The comments in - * mmu_try_to_unsync_pages() describe what could go wrong if - * this requirement isn't satisfied. - */ - if (!smp_load_acquire(&sp->unsync) && - !smp_load_acquire(&sp->unsync_children)) + if (!is_unsync_root(root)) return; write_lock(&vcpu->kvm->mmu_lock); @@ -3711,21 +3713,26 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) write_unlock(&vcpu->kvm->mmu_lock); } -static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gpa_t vaddr, - u32 access, struct x86_exception *exception) +void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu) { - if (exception) - exception->error_code = 0; - return vaddr; + unsigned long roots_to_free = 0; + int i; + + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) + if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa)) + roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); + + /* sync prev_roots by simply freeing them */ + kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free); } -static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gpa_t vaddr, - u32 access, - struct x86_exception *exception) +static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, + gpa_t vaddr, u32 access, + struct x86_exception *exception) { if (exception) exception->error_code = 0; - return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception); + return kvm_translate_gpa(vcpu, mmu, vaddr, access, exception); } static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct) @@ -3763,9 +3770,6 @@ static int get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, int *root_level spte = mmu_spte_get_lockless(iterator.sptep); sptes[leaf] = spte; - - if (!is_shadow_present_pte(spte)) - break; } return leaf; @@ -3856,20 +3860,19 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct) } static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, - u32 error_code, gfn_t gfn) + struct kvm_page_fault *fault) { - if (unlikely(error_code & PFERR_RSVD_MASK)) + if (unlikely(fault->rsvd)) return false; - if (!(error_code & PFERR_PRESENT_MASK) || - !(error_code & PFERR_WRITE_MASK)) + if (!fault->present || !fault->write) return false; /* * guest is writing the page which is write tracked which can * not be fixed by page fault handler. */ - if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE)) + if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE)) return true; return false; @@ -3881,11 +3884,8 @@ static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr) u64 spte; walk_shadow_page_lockless_begin(vcpu); - for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) { + for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) clear_sp_write_flooding_count(iterator.sptep); - if (!is_shadow_present_pte(spte)) - break; - } walk_shadow_page_lockless_end(vcpu); } @@ -3914,11 +3914,9 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch); } -static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, - gpa_t cr2_or_gpa, kvm_pfn_t *pfn, hva_t *hva, - bool write, bool *writable, int *r) +static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, int *r) { - struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); + struct kvm_memory_slot *slot = fault->slot; bool async; /* @@ -3932,8 +3930,9 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, if (!kvm_is_visible_memslot(slot)) { /* Don't expose private memslots to L2. */ if (is_guest_mode(vcpu)) { - *pfn = KVM_PFN_NOSLOT; - *writable = false; + fault->slot = NULL; + fault->pfn = KVM_PFN_NOSLOT; + fault->map_writable = false; return false; } /* @@ -3950,23 +3949,25 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, } async = false; - *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, - write, writable, hva); + fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, &async, + fault->write, &fault->map_writable, + &fault->hva); if (!async) return false; /* *pfn has correct page already */ - if (!prefault && kvm_can_do_async_pf(vcpu)) { - trace_kvm_try_async_get_page(cr2_or_gpa, gfn); - if (kvm_find_async_pf_gfn(vcpu, gfn)) { - trace_kvm_async_pf_doublefault(cr2_or_gpa, gfn); + if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) { + trace_kvm_try_async_get_page(fault->addr, fault->gfn); + if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) { + trace_kvm_async_pf_doublefault(fault->addr, fault->gfn); kvm_make_request(KVM_REQ_APF_HALT, vcpu); goto out_retry; - } else if (kvm_arch_setup_async_pf(vcpu, cr2_or_gpa, gfn)) + } else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn)) goto out_retry; } - *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, - write, writable, hva); + fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, NULL, + fault->write, &fault->map_writable, + &fault->hva); return false; out_retry: @@ -3974,23 +3975,48 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, return true; } -static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, - bool prefault, int max_level, bool is_tdp) +/* + * Returns true if the page fault is stale and needs to be retried, i.e. if the + * root was invalidated by a memslot update or a relevant mmu_notifier fired. + */ +static bool is_page_fault_stale(struct kvm_vcpu *vcpu, + struct kvm_page_fault *fault, int mmu_seq) +{ + struct kvm_mmu_page *sp = to_shadow_page(vcpu->arch.mmu->root_hpa); + + /* Special roots, e.g. pae_root, are not backed by shadow pages. */ + if (sp && is_obsolete_sp(vcpu->kvm, sp)) + return true; + + /* + * Roots without an associated shadow page are considered invalid if + * there is a pending request to free obsolete roots. The request is + * only a hint that the current root _may_ be obsolete and needs to be + * reloaded, e.g. if the guest frees a PGD that KVM is tracking as a + * previous root, then __kvm_mmu_prepare_zap_page() signals all vCPUs + * to reload even if no vCPU is actively using the root. + */ + if (!sp && kvm_test_request(KVM_REQ_MMU_RELOAD, vcpu)) + return true; + + return fault->slot && + mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, fault->hva); +} + +static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { bool is_tdp_mmu_fault = is_tdp_mmu(vcpu->arch.mmu); - bool write = error_code & PFERR_WRITE_MASK; - bool map_writable; - gfn_t gfn = gpa >> PAGE_SHIFT; unsigned long mmu_seq; - kvm_pfn_t pfn; - hva_t hva; int r; - if (page_fault_handle_page_track(vcpu, error_code, gfn)) + fault->gfn = fault->addr >> PAGE_SHIFT; + fault->slot = kvm_vcpu_gfn_to_memslot(vcpu, fault->gfn); + + if (page_fault_handle_page_track(vcpu, fault)) return RET_PF_EMULATE; - r = fast_page_fault(vcpu, gpa, error_code); + r = fast_page_fault(vcpu, fault); if (r != RET_PF_INVALID) return r; @@ -4001,11 +4027,10 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - if (kvm_faultin_pfn(vcpu, prefault, gfn, gpa, &pfn, &hva, - write, &map_writable, &r)) + if (kvm_faultin_pfn(vcpu, fault, &r)) return r; - if (handle_abnormal_pfn(vcpu, is_tdp ? 0 : gpa, gfn, pfn, ACC_ALL, &r)) + if (handle_abnormal_pfn(vcpu, fault, ACC_ALL, &r)) return r; r = RET_PF_RETRY; @@ -4015,36 +4040,35 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, else write_lock(&vcpu->kvm->mmu_lock); - if (!is_noslot_pfn(pfn) && mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, hva)) + if (is_page_fault_stale(vcpu, fault, mmu_seq)) goto out_unlock; + r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; if (is_tdp_mmu_fault) - r = kvm_tdp_mmu_map(vcpu, gpa, error_code, map_writable, max_level, - pfn, prefault); + r = kvm_tdp_mmu_map(vcpu, fault); else - r = __direct_map(vcpu, gpa, error_code, map_writable, max_level, pfn, - prefault, is_tdp); + r = __direct_map(vcpu, fault); out_unlock: if (is_tdp_mmu_fault) read_unlock(&vcpu->kvm->mmu_lock); else write_unlock(&vcpu->kvm->mmu_lock); - kvm_release_pfn_clean(pfn); + kvm_release_pfn_clean(fault->pfn); return r; } -static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, - u32 error_code, bool prefault) +static int nonpaging_page_fault(struct kvm_vcpu *vcpu, + struct kvm_page_fault *fault) { - pgprintk("%s: gva %lx error %x\n", __func__, gpa, error_code); + pgprintk("%s: gva %lx error %x\n", __func__, fault->addr, fault->error_code); /* This path builds a PAE pagetable, we can map 2mb pages at maximum. */ - return direct_page_fault(vcpu, gpa & PAGE_MASK, error_code, prefault, - PG_LEVEL_2M, false); + fault->max_level = PG_LEVEL_2M; + return direct_page_fault(vcpu, fault); } int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, @@ -4080,23 +4104,19 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, } EXPORT_SYMBOL_GPL(kvm_handle_page_fault); -int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, - bool prefault) +int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { - int max_level; - - for (max_level = KVM_MAX_HUGEPAGE_LEVEL; - max_level > PG_LEVEL_4K; - max_level--) { - int page_num = KVM_PAGES_PER_HPAGE(max_level); - gfn_t base = (gpa >> PAGE_SHIFT) & ~(page_num - 1); + while (fault->max_level > PG_LEVEL_4K) { + int page_num = KVM_PAGES_PER_HPAGE(fault->max_level); + gfn_t base = (fault->addr >> PAGE_SHIFT) & ~(page_num - 1); if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num)) break; + + --fault->max_level; } - return direct_page_fault(vcpu, gpa, error_code, prefault, - max_level, true); + return direct_page_fault(vcpu, fault); } static void nonpaging_init_context(struct kvm_mmu *context) @@ -4217,7 +4237,7 @@ static unsigned long get_cr3(struct kvm_vcpu *vcpu) } static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, - unsigned int access, int *nr_present) + unsigned int access) { if (unlikely(is_mmio_spte(*sptep))) { if (gfn != get_mmio_spte_gfn(*sptep)) { @@ -4225,7 +4245,6 @@ static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, return true; } - (*nr_present)++; mark_mmio_spte(vcpu, sptep, gfn, access); return true; } @@ -4364,22 +4383,28 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check, - u64 pa_bits_rsvd, bool execonly) + u64 pa_bits_rsvd, bool execonly, int huge_page_level) { u64 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51); + u64 large_1g_rsvd = 0, large_2m_rsvd = 0; u64 bad_mt_xwr; + if (huge_page_level < PG_LEVEL_1G) + large_1g_rsvd = rsvd_bits(7, 7); + if (huge_page_level < PG_LEVEL_2M) + large_2m_rsvd = rsvd_bits(7, 7); + rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7); rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7); - rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6); - rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6); + rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd; + rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd; rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd; /* large page */ rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4]; rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3]; - rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29); - rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20); + rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd; + rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd; rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0]; bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */ @@ -4395,10 +4420,11 @@ __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check, } static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu, - struct kvm_mmu *context, bool execonly) + struct kvm_mmu *context, bool execonly, int huge_page_level) { __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check, - vcpu->arch.reserved_gpa_bits, execonly); + vcpu->arch.reserved_gpa_bits, execonly, + huge_page_level); } static inline u64 reserved_hpa_bits(void) @@ -4474,7 +4500,8 @@ reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, false, true); else __reset_rsvds_bits_mask_ept(shadow_zero_check, - reserved_hpa_bits(), false); + reserved_hpa_bits(), false, + max_huge_page_level); if (!shadow_me_mask) return; @@ -4494,7 +4521,8 @@ reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context, bool execonly) { __reset_rsvds_bits_mask_ept(&context->shadow_zero_check, - reserved_hpa_bits(), execonly); + reserved_hpa_bits(), execonly, + max_huge_page_level); } #define BYTE_MASK(access) \ @@ -4743,7 +4771,7 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, role.base.ad_disabled = (shadow_accessed_mask == 0); role.base.level = kvm_mmu_get_tdp_level(vcpu); role.base.direct = true; - role.base.gpte_is_8_bytes = true; + role.base.has_4_byte_gpte = false; return role; } @@ -4788,7 +4816,7 @@ kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, role.base.smep_andnot_wp = role.ext.cr4_smep && !____is_cr0_wp(regs); role.base.smap_andnot_wp = role.ext.cr4_smap && !____is_cr0_wp(regs); - role.base.gpte_is_8_bytes = ____is_cr0_pg(regs) && ____is_cr4_pae(regs); + role.base.has_4_byte_gpte = ____is_cr0_pg(regs) && !____is_cr4_pae(regs); return role; } @@ -4887,7 +4915,7 @@ kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, role.base.smm = vcpu->arch.root_mmu.mmu_role.base.smm; role.base.level = level; - role.base.gpte_is_8_bytes = true; + role.base.has_4_byte_gpte = false; role.base.direct = false; role.base.ad_disabled = !accessed_dirty; role.base.guest_mode = true; @@ -4902,7 +4930,8 @@ kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, } void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, - bool accessed_dirty, gpa_t new_eptp) + int huge_page_level, bool accessed_dirty, + gpa_t new_eptp) { struct kvm_mmu *context = &vcpu->arch.guest_mmu; u8 level = vmx_eptp_page_walk_level(new_eptp); @@ -4929,7 +4958,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, update_permission_bitmask(context, true); context->pkru_mask = 0; - reset_rsvds_bits_mask_ept(vcpu, context, execonly); + reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level); reset_ept_shadow_zero_bits_mask(vcpu, context, execonly); } EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu); @@ -4993,13 +5022,13 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) * the gva_to_gpa functions between mmu and nested_mmu are swapped. */ if (!is_paging(vcpu)) - g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested; + g_context->gva_to_gpa = nonpaging_gva_to_gpa; else if (is_long_mode(vcpu)) - g_context->gva_to_gpa = paging64_gva_to_gpa_nested; + g_context->gva_to_gpa = paging64_gva_to_gpa; else if (is_pae(vcpu)) - g_context->gva_to_gpa = paging64_gva_to_gpa_nested; + g_context->gva_to_gpa = paging64_gva_to_gpa; else - g_context->gva_to_gpa = paging32_gva_to_gpa_nested; + g_context->gva_to_gpa = paging32_gva_to_gpa; reset_guest_paging_metadata(vcpu, g_context); } @@ -5034,6 +5063,14 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu) /* * Invalidate all MMU roles to force them to reinitialize as CPUID * information is factored into reserved bit calculations. + * + * Correctly handling multiple vCPU models with respect to paging and + * physical address properties) in a single VM would require tracking + * all relevant CPUID information in kvm_mmu_page_role. That is very + * undesirable as it would increase the memory requirements for + * gfn_track (see struct kvm_mmu_page_role comments). For now that + * problem is swept under the rug; KVM's CPUID API is horrific and + * it's all but impossible to solve it without introducing a new API. */ vcpu->arch.root_mmu.mmu_role.ext.valid = 0; vcpu->arch.guest_mmu.mmu_role.ext.valid = 0; @@ -5041,24 +5078,10 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu) kvm_mmu_reset_context(vcpu); /* - * KVM does not correctly handle changing guest CPUID after KVM_RUN, as - * MAXPHYADDR, GBPAGES support, AMD reserved bit behavior, etc.. aren't - * tracked in kvm_mmu_page_role. As a result, KVM may miss guest page - * faults due to reusing SPs/SPTEs. Alert userspace, but otherwise - * sweep the problem under the rug. - * - * KVM's horrific CPUID ABI makes the problem all but impossible to - * solve, as correctly handling multiple vCPU models (with respect to - * paging and physical address properties) in a single VM would require - * tracking all relevant CPUID information in kvm_mmu_page_role. That - * is very undesirable as it would double the memory requirements for - * gfn_track (see struct kvm_mmu_page_role comments), and in practice - * no sane VMM mucks with the core vCPU model on the fly. + * Changing guest CPUID after KVM_RUN is forbidden, see the comment in + * kvm_arch_vcpu_ioctl(). */ - if (vcpu->arch.last_vmentry_cpu != -1) { - pr_warn_ratelimited("KVM: KVM_SET_CPUID{,2} after KVM_RUN may cause guest instability\n"); - pr_warn_ratelimited("KVM: KVM_SET_CPUID{,2} will fail after KVM_RUN starting with Linux 5.16\n"); - } + KVM_BUG_ON(vcpu->arch.last_vmentry_cpu != -1, vcpu->kvm); } void kvm_mmu_reset_context(struct kvm_vcpu *vcpu) @@ -5170,7 +5193,7 @@ static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa, gpa, bytes, sp->role.word); offset = offset_in_page(gpa); - pte_size = sp->role.gpte_is_8_bytes ? 8 : 4; + pte_size = sp->role.has_4_byte_gpte ? 4 : 8; /* * Sometimes, the OS only writes the last one bytes to update status @@ -5194,7 +5217,7 @@ static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte) page_offset = offset_in_page(gpa); level = sp->role.level; *nspte = 1; - if (!sp->role.gpte_is_8_bytes) { + if (sp->role.has_4_byte_gpte) { page_offset <<= 1; /* 32->64 */ /* * A 32-bit pde maps 4MB while the shadow pdes map @@ -5225,7 +5248,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, LIST_HEAD(invalid_list); u64 entry, gentry, *spte; int npte; - bool remote_flush, local_flush; + bool flush = false; /* * If we don't have indirect shadow pages, it means no page is @@ -5234,8 +5257,6 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages)) return; - remote_flush = local_flush = false; - pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes); /* @@ -5264,18 +5285,17 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, if (!spte) continue; - local_flush = true; while (npte--) { entry = *spte; mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL); if (gentry && sp->role.level != PG_LEVEL_4K) ++vcpu->kvm->stat.mmu_pde_zapped; if (need_remote_flush(entry, *spte)) - remote_flush = true; + flush = true; ++spte; } } - kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush); + kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush); kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE); write_unlock(&vcpu->kvm->mmu_lock); } @@ -5509,10 +5529,13 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) mmu->root_hpa = INVALID_PAGE; mmu->root_pgd = 0; - mmu->translate_gpa = translate_gpa; for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; + /* vcpu->arch.guest_mmu isn't used when !tdp_enabled. */ + if (!tdp_enabled && mmu == &vcpu->arch.guest_mmu) + return 0; + /* * When using PAE paging, the four PDPTEs are treated as 'root' pages, * while the PDP table is a per-vCPU construct that's allocated at MMU @@ -5522,7 +5545,7 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) * generally doesn't use PAE paging and can skip allocating the PDP * table. The main exception, handled here, is SVM's 32-bit NPT. The * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit - * KVM; that horror is handled on-demand by mmu_alloc_shadow_roots(). + * KVM; that horror is handled on-demand by mmu_alloc_special_roots(). */ if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) return 0; @@ -5567,8 +5590,6 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu) vcpu->arch.mmu = &vcpu->arch.root_mmu; vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; - vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa; - ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu); if (ret) return ret; @@ -5707,13 +5728,7 @@ void kvm_mmu_init_vm(struct kvm *kvm) spin_lock_init(&kvm->arch.mmu_unsync_pages_lock); - if (!kvm_mmu_init_tdp_mmu(kvm)) - /* - * No smp_load/store wrappers needed here as we are in - * VM init and there cannot be any memslots / other threads - * accessing this struct kvm yet. - */ - kvm->arch.memslots_have_rmaps = true; + kvm_mmu_init_tdp_mmu(kvm); node->track_write = kvm_mmu_pte_write; node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot; @@ -5729,43 +5744,55 @@ void kvm_mmu_uninit_vm(struct kvm *kvm) kvm_mmu_uninit_tdp_mmu(kvm); } +static bool __kvm_zap_rmaps(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) +{ + const struct kvm_memory_slot *memslot; + struct kvm_memslots *slots; + struct kvm_memslot_iter iter; + bool flush = false; + gfn_t start, end; + int i; + + if (!kvm_memslots_have_rmaps(kvm)) + return flush; + + for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + slots = __kvm_memslots(kvm, i); + + kvm_for_each_memslot_in_gfn_range(&iter, slots, gfn_start, gfn_end) { + memslot = iter.slot; + start = max(gfn_start, memslot->base_gfn); + end = min(gfn_end, memslot->base_gfn + memslot->npages); + if (WARN_ON_ONCE(start >= end)) + continue; + + flush = slot_handle_level_range(kvm, memslot, kvm_zap_rmapp, + + PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL, + start, end - 1, true, flush); + } + } + + return flush; +} + /* * Invalidate (zap) SPTEs that cover GFNs from gfn_start and up to gfn_end * (not including it) */ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) { - struct kvm_memslots *slots; - struct kvm_memory_slot *memslot; + bool flush; int i; - bool flush = false; + + if (WARN_ON_ONCE(gfn_end <= gfn_start)) + return; write_lock(&kvm->mmu_lock); kvm_inc_notifier_count(kvm, gfn_start, gfn_end); - if (kvm_memslots_have_rmaps(kvm)) { - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { - slots = __kvm_memslots(kvm, i); - kvm_for_each_memslot(memslot, slots) { - gfn_t start, end; - - start = max(gfn_start, memslot->base_gfn); - end = min(gfn_end, memslot->base_gfn + memslot->npages); - if (start >= end) - continue; - - flush = slot_handle_level_range(kvm, - (const struct kvm_memory_slot *) memslot, - kvm_zap_rmapp, PG_LEVEL_4K, - KVM_MAX_HUGEPAGE_LEVEL, start, - end - 1, true, flush); - } - } - if (flush) - kvm_flush_remote_tlbs_with_address(kvm, gfn_start, - gfn_end - gfn_start); - } + flush = __kvm_zap_rmaps(kvm, gfn_start, gfn_end); if (is_tdp_mmu_enabled(kvm)) { for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) @@ -5810,15 +5837,27 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, } /* - * We can flush all the TLBs out of the mmu lock without TLB - * corruption since we just change the spte from writable to - * readonly so that we only need to care the case of changing - * spte from present to present (changing the spte from present - * to nonpresent will flush all the TLBs immediately), in other - * words, the only case we care is mmu_spte_update() where we - * have checked Host-writable | MMU-writable instead of - * PT_WRITABLE_MASK, that means it does not depend on PT_WRITABLE_MASK - * anymore. + * Flush TLBs if any SPTEs had to be write-protected to ensure that + * guest writes are reflected in the dirty bitmap before the memslot + * update completes, i.e. before enabling dirty logging is visible to + * userspace. + * + * Perform the TLB flush outside the mmu_lock to reduce the amount of + * time the lock is held. However, this does mean that another CPU can + * now grab mmu_lock and encounter a write-protected SPTE while CPUs + * still have a writable mapping for the associated GFN in their TLB. + * + * This is safe but requires KVM to be careful when making decisions + * based on the write-protection status of an SPTE. Specifically, KVM + * also write-protects SPTEs to monitor changes to guest page tables + * during shadow paging, and must guarantee no CPUs can write to those + * page before the lock is dropped. As mentioned in the previous + * paragraph, a write-protected SPTE is no guarantee that CPU cannot + * perform writes. So to determine if a TLB flush is truly required, KVM + * will clear a separate software-only bit (MMU-writable) and skip the + * flush if-and-only-if this bit was already clear. + * + * See DEFAULT_SPTE_MMU_WRITEABLE for more details. */ if (flush) kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); @@ -6150,30 +6189,6 @@ int kvm_mmu_module_init(void) return ret; } -/* - * Calculate mmu pages needed for kvm. - */ -unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm) -{ - unsigned long nr_mmu_pages; - unsigned long nr_pages = 0; - struct kvm_memslots *slots; - struct kvm_memory_slot *memslot; - int i; - - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { - slots = __kvm_memslots(kvm, i); - - kvm_for_each_memslot(memslot, slots) - nr_pages += memslot->npages; - } - - nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000; - nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES); - - return nr_mmu_pages; -} - void kvm_mmu_destroy(struct kvm_vcpu *vcpu) { kvm_mmu_unload(vcpu); @@ -6190,18 +6205,47 @@ void kvm_mmu_module_exit(void) mmu_audit_disable(); } -static int set_nx_huge_pages_recovery_ratio(const char *val, const struct kernel_param *kp) +/* + * Calculate the effective recovery period, accounting for '0' meaning "let KVM + * select a halving time of 1 hour". Returns true if recovery is enabled. + */ +static bool calc_nx_huge_pages_recovery_period(uint *period) { - unsigned int old_val; + /* + * Use READ_ONCE to get the params, this may be called outside of the + * param setters, e.g. by the kthread to compute its next timeout. + */ + bool enabled = READ_ONCE(nx_huge_pages); + uint ratio = READ_ONCE(nx_huge_pages_recovery_ratio); + + if (!enabled || !ratio) + return false; + + *period = READ_ONCE(nx_huge_pages_recovery_period_ms); + if (!*period) { + /* Make sure the period is not less than one second. */ + ratio = min(ratio, 3600u); + *period = 60 * 60 * 1000 / ratio; + } + return true; +} + +static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp) +{ + bool was_recovery_enabled, is_recovery_enabled; + uint old_period, new_period; int err; - old_val = nx_huge_pages_recovery_ratio; + was_recovery_enabled = calc_nx_huge_pages_recovery_period(&old_period); + err = param_set_uint(val, kp); if (err) return err; - if (READ_ONCE(nx_huge_pages) && - !old_val && nx_huge_pages_recovery_ratio) { + is_recovery_enabled = calc_nx_huge_pages_recovery_period(&new_period); + + if (is_recovery_enabled && + (!was_recovery_enabled || old_period > new_period)) { struct kvm *kvm; mutex_lock(&kvm_lock); @@ -6264,9 +6308,13 @@ static void kvm_recover_nx_lpages(struct kvm *kvm) static long get_nx_lpage_recovery_timeout(u64 start_time) { - return READ_ONCE(nx_huge_pages) && READ_ONCE(nx_huge_pages_recovery_ratio) - ? start_time + 60 * HZ - get_jiffies_64() - : MAX_SCHEDULE_TIMEOUT; + bool enabled; + uint period; + + enabled = calc_nx_huge_pages_recovery_period(&period); + + return enabled ? start_time + msecs_to_jiffies(period) - get_jiffies_64() + : MAX_SCHEDULE_TIMEOUT; } static int kvm_nx_lpage_recovery_worker(struct kvm *kvm, uintptr_t data) diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index bf2bdbf333..da6166b5c3 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -104,7 +104,7 @@ static inline int kvm_mmu_page_as_id(struct kvm_mmu_page *sp) return kvm_mmu_role_as_id(sp->role); } -static inline bool kvm_vcpu_ad_need_write_protect(struct kvm_vcpu *vcpu) +static inline bool kvm_mmu_page_ad_need_write_protect(struct kvm_mmu_page *sp) { /* * When using the EPT page-modification log, the GPAs in the CPU dirty @@ -112,19 +112,13 @@ static inline bool kvm_vcpu_ad_need_write_protect(struct kvm_vcpu *vcpu) * on write protection to record dirty pages, which bypasses PML, since * writes now result in a vmexit. Note, the check on CPU dirty logging * being enabled is mandatory as the bits used to denote WP-only SPTEs - * are reserved for NPT w/ PAE (32-bit KVM). + * are reserved for PAE paging (32-bit KVM). */ - return vcpu->arch.mmu == &vcpu->arch.guest_mmu && - kvm_x86_ops.cpu_dirty_log_size; + return kvm_x86_ops.cpu_dirty_log_size && sp->role.guest_mode; } -extern int nx_huge_pages; -static inline bool is_nx_huge_page_enabled(void) -{ - return READ_ONCE(nx_huge_pages); -} - -int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync); +int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, + gfn_t gfn, bool can_unsync, bool prefetch); void kvm_mmu_gfn_disallow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn); void kvm_mmu_gfn_allow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn); @@ -155,19 +149,11 @@ enum { RET_PF_SPURIOUS, }; -/* Bits which may be returned by set_spte() */ -#define SET_SPTE_WRITE_PROTECTED_PT BIT(0) -#define SET_SPTE_NEED_REMOTE_TLB_FLUSH BIT(1) -#define SET_SPTE_SPURIOUS BIT(2) - int kvm_mmu_max_mapping_level(struct kvm *kvm, const struct kvm_memory_slot *slot, gfn_t gfn, kvm_pfn_t pfn, int max_level); -int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, - int max_level, kvm_pfn_t *pfnp, - bool huge_page_disallowed, int *req_level); -void disallowed_hugepage_adjust(u64 spte, gfn_t gfn, int cur_level, - kvm_pfn_t *pfnp, int *goal_levelp); +void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); +void disallowed_hugepage_adjust(struct kvm_page_fault *fault, u64 spte, int cur_level); void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc); diff --git a/arch/x86/kvm/mmu/mmutrace.h b/arch/x86/kvm/mmu/mmutrace.h index 2924a4081a..de5e8e4e1a 100644 --- a/arch/x86/kvm/mmu/mmutrace.h +++ b/arch/x86/kvm/mmu/mmutrace.h @@ -35,7 +35,7 @@ " %snxe %sad root %u %s%c", \ __entry->mmu_valid_gen, \ __entry->gfn, role.level, \ - role.gpte_is_8_bytes ? 8 : 4, \ + role.has_4_byte_gpte ? 4 : 8, \ role.quadrant, \ role.direct ? " direct" : "", \ access_str[role.access], \ @@ -252,9 +252,9 @@ TRACE_EVENT( TRACE_EVENT( fast_page_fault, - TP_PROTO(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u32 error_code, + TP_PROTO(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, u64 *sptep, u64 old_spte, int ret), - TP_ARGS(vcpu, cr2_or_gpa, error_code, sptep, old_spte, ret), + TP_ARGS(vcpu, fault, sptep, old_spte, ret), TP_STRUCT__entry( __field(int, vcpu_id) @@ -268,8 +268,8 @@ TRACE_EVENT( TP_fast_assign( __entry->vcpu_id = vcpu->vcpu_id; - __entry->cr2_or_gpa = cr2_or_gpa; - __entry->error_code = error_code; + __entry->cr2_or_gpa = fault->addr; + __entry->error_code = fault->error_code; __entry->sptep = sptep; __entry->old_spte = old_spte; __entry->new_spte = *sptep; @@ -367,8 +367,8 @@ TRACE_EVENT( TRACE_EVENT( kvm_mmu_spte_requested, - TP_PROTO(gpa_t addr, int level, kvm_pfn_t pfn), - TP_ARGS(addr, level, pfn), + TP_PROTO(struct kvm_page_fault *fault), + TP_ARGS(fault), TP_STRUCT__entry( __field(u64, gfn) @@ -377,9 +377,9 @@ TRACE_EVENT( ), TP_fast_assign( - __entry->gfn = addr >> PAGE_SHIFT; - __entry->pfn = pfn | (__entry->gfn & (KVM_PAGES_PER_HPAGE(level) - 1)); - __entry->level = level; + __entry->gfn = fault->gfn; + __entry->pfn = fault->pfn | (fault->gfn & (KVM_PAGES_PER_HPAGE(fault->goal_level) - 1)); + __entry->level = fault->goal_level; ), TP_printk("gfn %llx pfn %llx level %d", diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 21427e84a8..68eb1fb548 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -19,6 +19,12 @@ #include "mmu.h" #include "mmu_internal.h" +bool kvm_page_track_write_tracking_enabled(struct kvm *kvm) +{ + return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) || + !tdp_enabled || kvm_shadow_root_allocated(kvm); +} + void kvm_page_track_free_memslot(struct kvm_memory_slot *slot) { int i; @@ -29,12 +35,17 @@ void kvm_page_track_free_memslot(struct kvm_memory_slot *slot) } } -int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, +int kvm_page_track_create_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot, unsigned long npages) { - int i; + int i; for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { + if (i == KVM_PAGE_TRACK_WRITE && + !kvm_page_track_write_tracking_enabled(kvm)) + continue; + slot->arch.gfn_track[i] = kvcalloc(npages, sizeof(*slot->arch.gfn_track[i]), GFP_KERNEL_ACCOUNT); @@ -57,6 +68,21 @@ static inline bool page_track_mode_is_valid(enum kvm_page_track_mode mode) return true; } +int kvm_page_track_write_tracking_alloc(struct kvm_memory_slot *slot) +{ + unsigned short *gfn_track; + + if (slot->arch.gfn_track[KVM_PAGE_TRACK_WRITE]) + return 0; + + gfn_track = kvcalloc(slot->npages, sizeof(*gfn_track), GFP_KERNEL_ACCOUNT); + if (gfn_track == NULL) + return -ENOMEM; + + slot->arch.gfn_track[KVM_PAGE_TRACK_WRITE] = gfn_track; + return 0; +} + static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn, enum kvm_page_track_mode mode, short count) { @@ -92,6 +118,10 @@ void kvm_slot_page_track_add_page(struct kvm *kvm, if (WARN_ON(!page_track_mode_is_valid(mode))) return; + if (WARN_ON(mode == KVM_PAGE_TRACK_WRITE && + !kvm_page_track_write_tracking_enabled(kvm))) + return; + update_gfn_track(slot, gfn, mode, 1); /* @@ -126,6 +156,10 @@ void kvm_slot_page_track_remove_page(struct kvm *kvm, if (WARN_ON(!page_track_mode_is_valid(mode))) return; + if (WARN_ON(mode == KVM_PAGE_TRACK_WRITE && + !kvm_page_track_write_tracking_enabled(kvm))) + return; + update_gfn_track(slot, gfn, mode, -1); /* @@ -139,19 +173,22 @@ EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page); /* * check if the corresponding access on the specified guest page is tracked. */ -bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, - enum kvm_page_track_mode mode) +bool kvm_slot_page_track_is_active(struct kvm *kvm, + const struct kvm_memory_slot *slot, + gfn_t gfn, enum kvm_page_track_mode mode) { - struct kvm_memory_slot *slot; int index; if (WARN_ON(!page_track_mode_is_valid(mode))) return false; - slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); if (!slot) return false; + if (mode == KVM_PAGE_TRACK_WRITE && + !kvm_page_track_write_tracking_enabled(kvm)) + return false; + index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); return !!READ_ONCE(slot->arch.gfn_track[mode][index]); } diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 913d52a792..5b5bdac97c 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -403,9 +403,8 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, walker->table_gfn[walker->level - 1] = table_gfn; walker->pte_gpa[walker->level - 1] = pte_gpa; - real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(table_gfn), - nested_access, - &walker->fault); + real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(table_gfn), + nested_access, &walker->fault); /* * FIXME: This can happen if emulation (for of an INS/OUTS @@ -467,7 +466,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, if (PTTYPE == 32 && walker->level > PG_LEVEL_4K && is_cpuid_PSE36()) gfn += pse36_gfn_delta(pte); - real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), access, &walker->fault); + real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(gfn), access, &walker->fault); if (real_gpa == UNMAPPED_GVA) return 0; @@ -547,20 +546,11 @@ static int FNAME(walk_addr)(struct guest_walker *walker, access); } -#if PTTYPE != PTTYPE_EPT -static int FNAME(walk_addr_nested)(struct guest_walker *walker, - struct kvm_vcpu *vcpu, gva_t addr, - u32 access) -{ - return FNAME(walk_addr_generic)(walker, vcpu, &vcpu->arch.nested_mmu, - addr, access); -} -#endif - static bool FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *spte, pt_element_t gpte, bool no_dirty_log) { + struct kvm_memory_slot *slot; unsigned pte_access; gfn_t gfn; kvm_pfn_t pfn; @@ -573,30 +563,21 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, gfn = gpte_to_gfn(gpte); pte_access = sp->role.access & FNAME(gpte_access)(gpte); FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte); - pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn, + + slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log && (pte_access & ACC_WRITE_MASK)); + if (!slot) + return false; + + pfn = gfn_to_pfn_memslot_atomic(slot, gfn); if (is_error_pfn(pfn)) return false; - /* - * we call mmu_set_spte() with host_writable = true because - * pte_prefetch_gfn_to_pfn always gets a writable pfn. - */ - mmu_set_spte(vcpu, spte, pte_access, false, PG_LEVEL_4K, gfn, pfn, - true, true); - + mmu_set_spte(vcpu, slot, spte, pte_access, gfn, pfn, NULL); kvm_release_pfn_clean(pfn); return true; } -static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, - u64 *spte, const void *pte) -{ - pt_element_t gpte = *(const pt_element_t *)pte; - - FNAME(prefetch_gpte)(vcpu, sp, spte, gpte, false); -} - static bool FNAME(gpte_changed)(struct kvm_vcpu *vcpu, struct guest_walker *gw, int level) { @@ -663,21 +644,16 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, * If the guest tries to write a write-protected page, we need to * emulate this operation, return 1 to indicate this case. */ -static int FNAME(fetch)(struct kvm_vcpu *vcpu, gpa_t addr, - struct guest_walker *gw, u32 error_code, - int max_level, kvm_pfn_t pfn, bool map_writable, - bool prefault) +static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, + struct guest_walker *gw) { - bool nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(); - bool write_fault = error_code & PFERR_WRITE_MASK; - bool exec = error_code & PFERR_FETCH_MASK; - bool huge_page_disallowed = exec && nx_huge_page_workaround_enabled; struct kvm_mmu_page *sp = NULL; struct kvm_shadow_walk_iterator it; unsigned int direct_access, access; - int top_level, level, req_level, ret; - gfn_t base_gfn = gw->gfn; + int top_level, ret; + gfn_t base_gfn = fault->gfn; + WARN_ON_ONCE(gw->gfn != base_gfn); direct_access = gw->pte_access; top_level = vcpu->arch.mmu->root_level; @@ -695,7 +671,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, gpa_t addr, if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) goto out_gpte_changed; - for (shadow_walk_init(&it, vcpu, addr); + for (shadow_walk_init(&it, vcpu, fault->addr); shadow_walk_okay(&it) && it.level > gw->level; shadow_walk_next(&it)) { gfn_t table_gfn; @@ -707,7 +683,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, gpa_t addr, if (!is_shadow_present_pte(*it.sptep)) { table_gfn = gw->table_gfn[it.level - 2]; access = gw->pt_access[it.level - 2]; - sp = kvm_mmu_get_page(vcpu, table_gfn, addr, + sp = kvm_mmu_get_page(vcpu, table_gfn, fault->addr, it.level-1, false, access); /* * We must synchronize the pagetable before linking it @@ -741,10 +717,9 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, gpa_t addr, link_shadow_page(vcpu, it.sptep, sp); } - level = kvm_mmu_hugepage_adjust(vcpu, gw->gfn, max_level, &pfn, - huge_page_disallowed, &req_level); + kvm_mmu_hugepage_adjust(vcpu, fault); - trace_kvm_mmu_spte_requested(addr, gw->level, pfn); + trace_kvm_mmu_spte_requested(fault); for (; shadow_walk_okay(&it); shadow_walk_next(&it)) { clear_sp_write_flooding_count(it.sptep); @@ -753,12 +728,11 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, gpa_t addr, * We cannot overwrite existing page tables with an NX * large page, as the leaf could be executable. */ - if (nx_huge_page_workaround_enabled) - disallowed_hugepage_adjust(*it.sptep, gw->gfn, it.level, - &pfn, &level); + if (fault->nx_huge_page_workaround_enabled) + disallowed_hugepage_adjust(fault, *it.sptep, it.level); - base_gfn = gw->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1); - if (it.level == level) + base_gfn = fault->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1); + if (it.level == fault->goal_level) break; validate_direct_spte(vcpu, it.sptep, direct_access); @@ -766,16 +740,20 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, gpa_t addr, drop_large_spte(vcpu, it.sptep); if (!is_shadow_present_pte(*it.sptep)) { - sp = kvm_mmu_get_page(vcpu, base_gfn, addr, + sp = kvm_mmu_get_page(vcpu, base_gfn, fault->addr, it.level - 1, true, direct_access); link_shadow_page(vcpu, it.sptep, sp); - if (huge_page_disallowed && req_level >= it.level) + if (fault->huge_page_disallowed && + fault->req_level >= it.level) account_huge_nx_page(vcpu->kvm, sp); } } - ret = mmu_set_spte(vcpu, it.sptep, gw->pte_access, write_fault, - it.level, base_gfn, pfn, prefault, map_writable); + if (WARN_ON_ONCE(it.level != fault->goal_level)) + return -EFAULT; + + ret = mmu_set_spte(vcpu, fault->slot, it.sptep, gw->pte_access, + base_gfn, fault->pfn, fault); if (ret == RET_PF_SPURIOUS) return ret; @@ -841,45 +819,40 @@ FNAME(is_self_change_mapping)(struct kvm_vcpu *vcpu, * Returns: 1 if we need to emulate the instruction, 0 otherwise, or * a negative value on error. */ -static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, - bool prefault) +static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { - bool write_fault = error_code & PFERR_WRITE_MASK; - bool user_fault = error_code & PFERR_USER_MASK; struct guest_walker walker; int r; - kvm_pfn_t pfn; - hva_t hva; unsigned long mmu_seq; - bool map_writable, is_self_change_mapping; - int max_level; + bool is_self_change_mapping; - pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code); - - /* - * If PFEC.RSVD is set, this is a shadow page fault. - * The bit needs to be cleared before walking guest page tables. - */ - error_code &= ~PFERR_RSVD_MASK; + pgprintk("%s: addr %lx err %x\n", __func__, fault->addr, fault->error_code); + WARN_ON_ONCE(fault->is_tdp); /* * Look up the guest pte for the faulting address. + * If PFEC.RSVD is set, this is a shadow page fault. + * The bit needs to be cleared before walking guest page tables. */ - r = FNAME(walk_addr)(&walker, vcpu, addr, error_code); + r = FNAME(walk_addr)(&walker, vcpu, fault->addr, + fault->error_code & ~PFERR_RSVD_MASK); /* * The page is not mapped by the guest. Let the guest handle it. */ if (!r) { pgprintk("%s: guest page fault\n", __func__); - if (!prefault) + if (!fault->prefetch) kvm_inject_emulated_page_fault(vcpu, &walker.fault); return RET_PF_RETRY; } - if (page_fault_handle_page_track(vcpu, error_code, walker.gfn)) { - shadow_page_table_clear_flood(vcpu, addr); + fault->gfn = walker.gfn; + fault->slot = kvm_vcpu_gfn_to_memslot(vcpu, fault->gfn); + + if (page_fault_handle_page_track(vcpu, fault)) { + shadow_page_table_clear_flood(vcpu, fault->addr); return RET_PF_EMULATE; } @@ -890,29 +863,28 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, vcpu->arch.write_fault_to_shadow_pgtable = false; is_self_change_mapping = FNAME(is_self_change_mapping)(vcpu, - &walker, user_fault, &vcpu->arch.write_fault_to_shadow_pgtable); + &walker, fault->user, &vcpu->arch.write_fault_to_shadow_pgtable); if (is_self_change_mapping) - max_level = PG_LEVEL_4K; + fault->max_level = PG_LEVEL_4K; else - max_level = walker.level; + fault->max_level = walker.level; mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - if (kvm_faultin_pfn(vcpu, prefault, walker.gfn, addr, &pfn, &hva, - write_fault, &map_writable, &r)) + if (kvm_faultin_pfn(vcpu, fault, &r)) return r; - if (handle_abnormal_pfn(vcpu, addr, walker.gfn, pfn, walker.pte_access, &r)) + if (handle_abnormal_pfn(vcpu, fault, walker.pte_access, &r)) return r; /* * Do not change pte_access if the pfn is a mmio page, otherwise * we will cache the incorrect access into mmio spte. */ - if (write_fault && !(walker.pte_access & ACC_WRITE_MASK) && - !is_cr0_wp(vcpu->arch.mmu) && !user_fault && !is_noslot_pfn(pfn)) { + if (fault->write && !(walker.pte_access & ACC_WRITE_MASK) && + !is_cr0_wp(vcpu->arch.mmu) && !fault->user && fault->slot) { walker.pte_access |= ACC_WRITE_MASK; walker.pte_access &= ~ACC_USER_MASK; @@ -928,20 +900,20 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (!is_noslot_pfn(pfn) && mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, hva)) + + if (is_page_fault_stale(vcpu, fault, mmu_seq)) goto out_unlock; kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT); r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; - r = FNAME(fetch)(vcpu, addr, &walker, error_code, max_level, pfn, - map_writable, prefault); + r = FNAME(fetch)(vcpu, fault, &walker); kvm_mmu_audit(vcpu, AUDIT_POST_PAGE_FAULT); out_unlock: write_unlock(&vcpu->kvm->mmu_lock); - kvm_release_pfn_clean(pfn); + kvm_release_pfn_clean(fault->pfn); return r; } @@ -1007,24 +979,30 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa) sizeof(pt_element_t))) break; - FNAME(update_pte)(vcpu, sp, sptep, &gpte); + FNAME(prefetch_gpte)(vcpu, sp, sptep, gpte, false); } - if (!is_shadow_present_pte(*sptep) || !sp->unsync_children) + if (!sp->unsync_children) break; } write_unlock(&vcpu->kvm->mmu_lock); } /* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */ -static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gpa_t addr, u32 access, +static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, + gpa_t addr, u32 access, struct x86_exception *exception) { struct guest_walker walker; gpa_t gpa = UNMAPPED_GVA; int r; - r = FNAME(walk_addr)(&walker, vcpu, addr, access); +#ifndef CONFIG_X86_64 + /* A 64-bit GVA should be impossible on 32-bit KVM. */ + WARN_ON_ONCE((addr >> 32) && mmu == vcpu->arch.walk_mmu); +#endif + + r = FNAME(walk_addr_generic)(&walker, vcpu, mmu, addr, access); if (r) { gpa = gfn_to_gpa(walker.gfn); @@ -1035,45 +1013,23 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gpa_t addr, u32 access, return gpa; } -#if PTTYPE != PTTYPE_EPT -/* Note, gva_to_gpa_nested() is only used to translate L2 GVAs. */ -static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gpa_t vaddr, - u32 access, - struct x86_exception *exception) -{ - struct guest_walker walker; - gpa_t gpa = UNMAPPED_GVA; - int r; - -#ifndef CONFIG_X86_64 - /* A 64-bit GVA should be impossible on 32-bit KVM. */ - WARN_ON_ONCE(vaddr >> 32); -#endif - - r = FNAME(walk_addr_nested)(&walker, vcpu, vaddr, access); - - if (r) { - gpa = gfn_to_gpa(walker.gfn); - gpa |= vaddr & ~PAGE_MASK; - } else if (exception) - *exception = walker.fault; - - return gpa; -} -#endif - /* * Using the cached information from sp->gfns is safe because: * - The spte has a reference to the struct page, so the pfn for a given gfn * can't change unless all sptes pointing to it are nuked first. + * + * Returns + * < 0: the sp should be zapped + * 0: the sp is synced and no tlb flushing is required + * > 0: the sp is synced and tlb flushing is required */ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { union kvm_mmu_page_role mmu_role = vcpu->arch.mmu->mmu_role.base; - int i, nr_present = 0; + int i; bool host_writable; gpa_t first_pte_gpa; - int set_spte_ret = 0; + bool flush = false; /* * Ignore various flags when verifying that it's safe to sync a shadow @@ -1098,11 +1054,13 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) */ if (WARN_ON_ONCE(sp->role.direct || (sp->role.word ^ mmu_role.word) & ~sync_role_ign.word)) - return 0; + return -1; first_pte_gpa = FNAME(get_level1_sp_gpa)(sp); for (i = 0; i < PT64_ENT_PER_PAGE; i++) { + u64 *sptep, spte; + struct kvm_memory_slot *slot; unsigned pte_access; pt_element_t gpte; gpa_t pte_gpa; @@ -1115,10 +1073,10 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) if (kvm_vcpu_read_guest_atomic(vcpu, pte_gpa, &gpte, sizeof(pt_element_t))) - return 0; + return -1; if (FNAME(prefetch_invalid_gpte)(vcpu, sp, &sp->spt[i], gpte)) { - set_spte_ret |= SET_SPTE_NEED_REMOTE_TLB_FLUSH; + flush = true; continue; } @@ -1127,30 +1085,27 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) pte_access &= FNAME(gpte_access)(gpte); FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte); - if (sync_mmio_spte(vcpu, &sp->spt[i], gfn, pte_access, - &nr_present)) + if (sync_mmio_spte(vcpu, &sp->spt[i], gfn, pte_access)) continue; if (gfn != sp->gfns[i]) { drop_spte(vcpu->kvm, &sp->spt[i]); - set_spte_ret |= SET_SPTE_NEED_REMOTE_TLB_FLUSH; + flush = true; continue; } - nr_present++; + sptep = &sp->spt[i]; + spte = *sptep; + host_writable = spte & shadow_host_writable_mask; + slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); + make_spte(vcpu, sp, slot, pte_access, gfn, + spte_to_pfn(spte), spte, true, false, + host_writable, &spte); - host_writable = sp->spt[i] & shadow_host_writable_mask; - - set_spte_ret |= set_spte(vcpu, &sp->spt[i], - pte_access, PG_LEVEL_4K, - gfn, spte_to_pfn(sp->spt[i]), - true, false, host_writable); + flush |= mmu_spte_update(sptep, spte); } - if (set_spte_ret & SET_SPTE_NEED_REMOTE_TLB_FLUSH) - kvm_flush_remote_tlbs(vcpu->kvm); - - return nr_present; + return flush; } #undef pt_element_t diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c index 3e97cdb13e..73cfe62fda 100644 --- a/arch/x86/kvm/mmu/spte.c +++ b/arch/x86/kvm/mmu/spte.c @@ -16,6 +16,7 @@ #include "spte.h" #include +#include #include static bool __read_mostly enable_mmio_caching = true; @@ -89,17 +90,19 @@ static bool kvm_is_mmio_pfn(kvm_pfn_t pfn) E820_TYPE_RAM); } -int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, - gfn_t gfn, kvm_pfn_t pfn, u64 old_spte, bool speculative, - bool can_unsync, bool host_writable, bool ad_disabled, - u64 *new_spte) +bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, + const struct kvm_memory_slot *slot, + unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn, + u64 old_spte, bool prefetch, bool can_unsync, + bool host_writable, u64 *new_spte) { + int level = sp->role.level; u64 spte = SPTE_MMU_PRESENT_MASK; - int ret = 0; + bool wrprot = false; - if (ad_disabled) + if (sp->role.ad_disabled) spte |= SPTE_TDP_AD_DISABLED_MASK; - else if (kvm_vcpu_ad_need_write_protect(vcpu)) + else if (kvm_mmu_page_ad_need_write_protect(sp)) spte |= SPTE_TDP_AD_WRPROT_ONLY_MASK; /* @@ -109,7 +112,7 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, * read access. See FNAME(gpte_access) in paging_tmpl.h. */ spte |= shadow_present_mask; - if (!speculative) + if (!prefetch) spte |= spte_shadow_accessed_mask(spte); if (level > PG_LEVEL_4K && (pte_access & ACC_EXEC_MASK) && @@ -150,7 +153,7 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, * is responsibility of kvm_mmu_get_page / kvm_mmu_sync_roots. * Same reasoning can be applied to dirty page accounting. */ - if (!can_unsync && is_writable_pte(old_spte)) + if (is_writable_pte(old_spte)) goto out; /* @@ -159,10 +162,10 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, * e.g. it's write-tracked (upper-level SPs) or has one or more * shadow pages and unsync'ing pages is not allowed. */ - if (mmu_try_to_unsync_pages(vcpu, gfn, can_unsync)) { + if (mmu_try_to_unsync_pages(vcpu->kvm, slot, gfn, can_unsync, prefetch)) { pgprintk("%s: found shadow page for %llx, marking ro\n", __func__, gfn); - ret |= SET_SPTE_WRITE_PROTECTED_PT; + wrprot = true; pte_access &= ~ACC_WRITE_MASK; spte &= ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask); } @@ -171,16 +174,22 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, if (pte_access & ACC_WRITE_MASK) spte |= spte_shadow_dirty_mask(spte); - if (speculative) +out: + if (prefetch) spte = mark_spte_for_access_track(spte); -out: WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->shadow_zero_check, spte, level), "spte = 0x%llx, level = %d, rsvd bits = 0x%llx", spte, level, get_rsvd_bits(&vcpu->arch.mmu->shadow_zero_check, spte, level)); + if ((spte & PT_WRITABLE_MASK) && kvm_slot_dirty_track_enabled(slot)) { + /* Enforced by kvm_mmu_hugepage_adjust. */ + WARN_ON(level > PG_LEVEL_4K); + mark_page_dirty_in_slot(vcpu->kvm, slot, gfn); + } + *new_spte = spte; - return ret; + return wrprot; } u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled) @@ -207,6 +216,7 @@ u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn) new_spte &= ~PT_WRITABLE_MASK; new_spte &= ~shadow_host_writable_mask; + new_spte &= ~shadow_mmu_writable_mask; new_spte = mark_spte_for_access_track(new_spte); diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h index 31d6456d8a..be6a007a4a 100644 --- a/arch/x86/kvm/mmu/spte.h +++ b/arch/x86/kvm/mmu/spte.h @@ -60,10 +60,6 @@ static_assert(SPTE_TDP_AD_ENABLED_MASK == 0); (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1)) #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) -/* Bits 9 and 10 are ignored by all non-EPT PTEs. */ -#define DEFAULT_SPTE_HOST_WRITEABLE BIT_ULL(9) -#define DEFAULT_SPTE_MMU_WRITEABLE BIT_ULL(10) - /* * The mask/shift to use for saving the original R/X bits when marking the PTE * as not-present for access tracking purposes. We do not save the W bit as the @@ -78,6 +74,35 @@ static_assert(SPTE_TDP_AD_ENABLED_MASK == 0); SHADOW_ACC_TRACK_SAVED_BITS_SHIFT) static_assert(!(SPTE_TDP_AD_MASK & SHADOW_ACC_TRACK_SAVED_MASK)); +/* + * *_SPTE_HOST_WRITEABLE (aka Host-writable) indicates whether the host permits + * writes to the guest page mapped by the SPTE. This bit is cleared on SPTEs + * that map guest pages in read-only memslots and read-only VMAs. + * + * Invariants: + * - If Host-writable is clear, PT_WRITABLE_MASK must be clear. + * + * + * *_SPTE_MMU_WRITEABLE (aka MMU-writable) indicates whether the shadow MMU + * allows writes to the guest page mapped by the SPTE. This bit is cleared when + * the guest page mapped by the SPTE contains a page table that is being + * monitored for shadow paging. In this case the SPTE can only be made writable + * by unsyncing the shadow page under the mmu_lock. + * + * Invariants: + * - If MMU-writable is clear, PT_WRITABLE_MASK must be clear. + * - If MMU-writable is set, Host-writable must be set. + * + * If MMU-writable is set, PT_WRITABLE_MASK is normally set but can be cleared + * to track writes for dirty logging. For such SPTEs, KVM will locklessly set + * PT_WRITABLE_MASK upon the next write from the guest and record the write in + * the dirty log (see fast_page_fault()). + */ + +/* Bits 9 and 10 are ignored by all non-EPT PTEs. */ +#define DEFAULT_SPTE_HOST_WRITEABLE BIT_ULL(9) +#define DEFAULT_SPTE_MMU_WRITEABLE BIT_ULL(10) + /* * Low ignored bits are at a premium for EPT, use high ignored bits, taking care * to not overlap the A/D type mask or the saved access bits of access-tracked @@ -316,8 +341,13 @@ static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check, static inline bool spte_can_locklessly_be_made_writable(u64 spte) { - return (spte & shadow_host_writable_mask) && - (spte & shadow_mmu_writable_mask); + if (spte & shadow_mmu_writable_mask) { + WARN_ON_ONCE(!(spte & shadow_host_writable_mask)); + return true; + } + + WARN_ON_ONCE(spte & PT_WRITABLE_MASK); + return false; } static inline u64 get_mmio_spte_generation(u64 spte) @@ -329,15 +359,11 @@ static inline u64 get_mmio_spte_generation(u64 spte) return gen; } -/* Bits which may be returned by set_spte() */ -#define SET_SPTE_WRITE_PROTECTED_PT BIT(0) -#define SET_SPTE_NEED_REMOTE_TLB_FLUSH BIT(1) -#define SET_SPTE_SPURIOUS BIT(2) - -int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, - gfn_t gfn, kvm_pfn_t pfn, u64 old_spte, bool speculative, - bool can_unsync, bool host_writable, bool ad_disabled, - u64 *new_spte); +bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, + const struct kvm_memory_slot *slot, + unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn, + u64 old_spte, bool prefetch, bool can_unsync, + bool host_writable, u64 *new_spte); u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled); u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access); u64 mark_spte_for_access_track(u64 spte); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index d479b2b121..bc9e3553fb 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -165,8 +165,9 @@ static union kvm_mmu_page_role page_role_for_level(struct kvm_vcpu *vcpu, role = vcpu->arch.mmu->mmu_role.base; role.level = level; role.direct = true; - role.gpte_is_8_bytes = true; + role.has_4_byte_gpte = false; role.access = ACC_ALL; + role.ad_disabled = !shadow_accessed_mask; return role; } @@ -487,8 +488,8 @@ static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, } /* - * tdp_mmu_set_spte_atomic_no_dirty_log - Set a TDP MMU SPTE atomically - * and handle the associated bookkeeping, but do not mark the page dirty + * tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically + * and handle the associated bookkeeping. Do not mark the page dirty * in KVM's dirty bitmaps. * * @kvm: kvm instance @@ -497,9 +498,9 @@ static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, * Returns: true if the SPTE was set, false if it was not. If false is returned, * this function will have no side-effects. */ -static inline bool tdp_mmu_set_spte_atomic_no_dirty_log(struct kvm *kvm, - struct tdp_iter *iter, - u64 new_spte) +static inline bool tdp_mmu_set_spte_atomic(struct kvm *kvm, + struct tdp_iter *iter, + u64 new_spte) { WARN_ON_ONCE(iter->yielded); @@ -527,43 +528,6 @@ static inline bool tdp_mmu_set_spte_atomic_no_dirty_log(struct kvm *kvm, return true; } -/* - * tdp_mmu_map_set_spte_atomic - Set a leaf TDP MMU SPTE atomically to resolve a - * TDP page fault. - * - * @vcpu: The vcpu instance that took the TDP page fault. - * @iter: a tdp_iter instance currently on the SPTE that should be set - * @new_spte: The value the SPTE should be set to - * - * Returns: true if the SPTE was set, false if it was not. If false is returned, - * this function will have no side-effects. - */ -static inline bool tdp_mmu_map_set_spte_atomic(struct kvm_vcpu *vcpu, - struct tdp_iter *iter, - u64 new_spte) -{ - struct kvm *kvm = vcpu->kvm; - - if (!tdp_mmu_set_spte_atomic_no_dirty_log(kvm, iter, new_spte)) - return false; - - /* - * Use kvm_vcpu_gfn_to_memslot() instead of going through - * handle_changed_spte_dirty_log() to leverage vcpu->last_used_slot. - */ - if (is_writable_pte(new_spte)) { - struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, iter->gfn); - - if (slot && kvm_slot_dirty_track_enabled(slot)) { - /* Enforced by kvm_mmu_hugepage_adjust. */ - WARN_ON_ONCE(iter->level > PG_LEVEL_4K); - mark_page_dirty_in_slot(kvm, slot, iter->gfn); - } - } - - return true; -} - static inline bool tdp_mmu_zap_spte_atomic(struct kvm *kvm, struct tdp_iter *iter) { @@ -573,7 +537,7 @@ static inline bool tdp_mmu_zap_spte_atomic(struct kvm *kvm, * immediately installing a present entry in its place * before the TLBs are flushed. */ - if (!tdp_mmu_set_spte_atomic_no_dirty_log(kvm, iter, REMOVED_SPTE)) + if (!tdp_mmu_set_spte_atomic(kvm, iter, REMOVED_SPTE)) return false; kvm_flush_remote_tlbs_with_address(kvm, iter->gfn, @@ -930,26 +894,26 @@ void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm) * Installs a last-level SPTE to handle a TDP page fault. * (NPT/EPT violation/misconfiguration) */ -static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, - int map_writable, - struct tdp_iter *iter, - kvm_pfn_t pfn, bool prefault) +static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, + struct kvm_page_fault *fault, + struct tdp_iter *iter) { + struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep)); u64 new_spte; int ret = RET_PF_FIXED; - int make_spte_ret = 0; + bool wrprot = false; - if (unlikely(is_noslot_pfn(pfn))) + WARN_ON(sp->role.level != fault->goal_level); + if (unlikely(!fault->slot)) new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL); else - make_spte_ret = make_spte(vcpu, ACC_ALL, iter->level, iter->gfn, - pfn, iter->old_spte, prefault, true, - map_writable, !shadow_accessed_mask, - &new_spte); + wrprot = make_spte(vcpu, sp, fault->slot, ACC_ALL, iter->gfn, + fault->pfn, iter->old_spte, fault->prefetch, true, + fault->map_writable, &new_spte); if (new_spte == iter->old_spte) ret = RET_PF_SPURIOUS; - else if (!tdp_mmu_map_set_spte_atomic(vcpu, iter, new_spte)) + else if (!tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte)) return RET_PF_RETRY; /* @@ -957,10 +921,9 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, * protected, emulation is needed. If the emulation was skipped, * the vCPU would have the same fault again. */ - if (make_spte_ret & SET_SPTE_WRITE_PROTECTED_PT) { - if (write) + if (wrprot) { + if (fault->write) ret = RET_PF_EMULATE; - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } /* If a MMIO SPTE is installed, the MMIO will need to be emulated. */ @@ -987,37 +950,26 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, * Handle a TDP page fault (NPT/EPT violation/misconfiguration) by installing * page tables and SPTEs to translate the faulting guest physical address. */ -int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, - int map_writable, int max_level, kvm_pfn_t pfn, - bool prefault) +int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { - bool nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(); - bool write = error_code & PFERR_WRITE_MASK; - bool exec = error_code & PFERR_FETCH_MASK; - bool huge_page_disallowed = exec && nx_huge_page_workaround_enabled; struct kvm_mmu *mmu = vcpu->arch.mmu; struct tdp_iter iter; struct kvm_mmu_page *sp; u64 *child_pt; u64 new_spte; int ret; - gfn_t gfn = gpa >> PAGE_SHIFT; - int level; - int req_level; - level = kvm_mmu_hugepage_adjust(vcpu, gfn, max_level, &pfn, - huge_page_disallowed, &req_level); + kvm_mmu_hugepage_adjust(vcpu, fault); - trace_kvm_mmu_spte_requested(gpa, level, pfn); + trace_kvm_mmu_spte_requested(fault); rcu_read_lock(); - tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { - if (nx_huge_page_workaround_enabled) - disallowed_hugepage_adjust(iter.old_spte, gfn, - iter.level, &pfn, &level); + tdp_mmu_for_each_pte(iter, mmu, fault->gfn, fault->gfn + 1) { + if (fault->nx_huge_page_workaround_enabled) + disallowed_hugepage_adjust(fault, iter.old_spte, iter.level); - if (iter.level == level) + if (iter.level == fault->goal_level) break; /* @@ -1053,10 +1005,10 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, new_spte = make_nonleaf_spte(child_pt, !shadow_accessed_mask); - if (tdp_mmu_set_spte_atomic_no_dirty_log(vcpu->kvm, &iter, new_spte)) { + if (tdp_mmu_set_spte_atomic(vcpu->kvm, &iter, new_spte)) { tdp_mmu_link_page(vcpu->kvm, sp, - huge_page_disallowed && - req_level >= iter.level); + fault->huge_page_disallowed && + fault->req_level >= iter.level); trace_kvm_mmu_get_page(sp, true); } else { @@ -1066,13 +1018,12 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, } } - if (iter.level != level) { + if (iter.level != fault->goal_level) { rcu_read_unlock(); return RET_PF_RETRY; } - ret = tdp_mmu_map_handle_target_level(vcpu, write, map_writable, &iter, - pfn, prefault); + ret = tdp_mmu_map_handle_target_level(vcpu, fault, &iter); rcu_read_unlock(); return ret; @@ -1083,7 +1034,7 @@ bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range, { struct kvm_mmu_page *root; - for_each_tdp_mmu_root(kvm, root, range->slot->as_id) + for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false) flush = zap_gfn_range(kvm, root, range->start, range->end, range->may_block, flush, false); @@ -1242,8 +1193,7 @@ static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, new_spte = iter.old_spte & ~PT_WRITABLE_MASK; - if (!tdp_mmu_set_spte_atomic_no_dirty_log(kvm, &iter, - new_spte)) { + if (!tdp_mmu_set_spte_atomic(kvm, &iter, new_spte)) { /* * The iter must explicitly re-read the SPTE because * the atomic cmpxchg failed. @@ -1311,8 +1261,7 @@ static bool clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, continue; } - if (!tdp_mmu_set_spte_atomic_no_dirty_log(kvm, &iter, - new_spte)) { + if (!tdp_mmu_set_spte_atomic(kvm, &iter, new_spte)) { /* * The iter must explicitly re-read the SPTE because * the atomic cmpxchg failed. diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h index ba3681cd38..3899004a5d 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.h +++ b/arch/x86/kvm/mmu/tdp_mmu.h @@ -48,9 +48,7 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm); void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm); void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm); -int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, - int map_writable, int max_level, kvm_pfn_t pfn, - bool prefault); +int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range, bool flush); @@ -91,7 +89,6 @@ u64 *kvm_tdp_mmu_fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, u64 addr, #ifdef CONFIG_X86_64 bool kvm_mmu_init_tdp_mmu(struct kvm *kvm); void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm); -static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return kvm->arch.tdp_mmu_enabled; } static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return sp->tdp_mmu_page; } static inline bool is_tdp_mmu(struct kvm_mmu *mmu) @@ -113,7 +110,6 @@ static inline bool is_tdp_mmu(struct kvm_mmu *mmu) #else static inline bool kvm_mmu_init_tdp_mmu(struct kvm *kvm) { return false; } static inline void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) {} -static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return false; } static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return false; } static inline bool is_tdp_mmu(struct kvm_mmu *mmu) { return false; } #endif diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index f256f01056..b1a0299378 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include "x86.h" #include "cpuid.h" @@ -55,43 +57,41 @@ static void kvm_pmi_trigger_fn(struct irq_work *irq_work) kvm_pmu_deliver_pmi(vcpu); } +static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi) +{ + struct kvm_pmu *pmu = pmc_to_pmu(pmc); + + /* Ignore counters that have been reprogrammed already. */ + if (test_and_set_bit(pmc->idx, pmu->reprogram_pmi)) + return; + + __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); + kvm_make_request(KVM_REQ_PMU, pmc->vcpu); + + if (!pmc->intr) + return; + + /* + * Inject PMI. If vcpu was in a guest mode during NMI PMI + * can be ejected on a guest mode re-entry. Otherwise we can't + * be sure that vcpu wasn't executing hlt instruction at the + * time of vmexit and is not going to re-enter guest mode until + * woken up. So we should wake it, but this is impossible from + * NMI context. Do it from irq work instead. + */ + if (in_pmi && !kvm_handling_nmi_from_guest(pmc->vcpu)) + irq_work_queue(&pmc_to_pmu(pmc)->irq_work); + else + kvm_make_request(KVM_REQ_PMI, pmc->vcpu); +} + static void kvm_perf_overflow(struct perf_event *perf_event, struct perf_sample_data *data, struct pt_regs *regs) { struct kvm_pmc *pmc = perf_event->overflow_handler_context; - struct kvm_pmu *pmu = pmc_to_pmu(pmc); - if (!test_and_set_bit(pmc->idx, pmu->reprogram_pmi)) { - __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); - kvm_make_request(KVM_REQ_PMU, pmc->vcpu); - } -} - -static void kvm_perf_overflow_intr(struct perf_event *perf_event, - struct perf_sample_data *data, - struct pt_regs *regs) -{ - struct kvm_pmc *pmc = perf_event->overflow_handler_context; - struct kvm_pmu *pmu = pmc_to_pmu(pmc); - - if (!test_and_set_bit(pmc->idx, pmu->reprogram_pmi)) { - __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); - kvm_make_request(KVM_REQ_PMU, pmc->vcpu); - - /* - * Inject PMI. If vcpu was in a guest mode during NMI PMI - * can be ejected on a guest mode re-entry. Otherwise we can't - * be sure that vcpu wasn't executing hlt instruction at the - * time of vmexit and is not going to re-enter guest mode until - * woken up. So we should wake it, but this is impossible from - * NMI context. Do it from irq work instead. - */ - if (!kvm_is_in_guest()) - irq_work_queue(&pmc_to_pmu(pmc)->irq_work); - else - kvm_make_request(KVM_REQ_PMI, pmc->vcpu); - } + __kvm_perf_overflow(pmc, true); } static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, @@ -111,6 +111,9 @@ static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, .config = config, }; + if (type == PERF_TYPE_HARDWARE && config >= PERF_COUNT_HW_MAX) + return; + attr.sample_period = get_sample_period(pmc, pmc->counter); if (in_tx) @@ -126,7 +129,6 @@ static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, } event = perf_event_create_kernel_counter(&attr, -1, current, - intr ? kvm_perf_overflow_intr : kvm_perf_overflow, pmc); if (IS_ERR(event)) { pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n", @@ -138,6 +140,7 @@ static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, pmc_to_pmu(pmc)->event_count++; clear_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi); pmc->is_paused = false; + pmc->intr = intr; } static void pmc_pause_counter(struct kvm_pmc *pmc) @@ -171,13 +174,17 @@ static bool pmc_resume_counter(struct kvm_pmc *pmc) return true; } +static int cmp_u64(const void *a, const void *b) +{ + return *(__u64 *)a - *(__u64 *)b; +} + void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) { u64 config; u32 type = PERF_TYPE_RAW; struct kvm *kvm = pmc->vcpu->kvm; struct kvm_pmu_event_filter *filter; - int i; bool allow_event = true; if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) @@ -192,16 +199,13 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu); if (filter) { - for (i = 0; i < filter->nevents; i++) - if (filter->events[i] == - (eventsel & AMD64_RAW_EVENT_MASK_NB)) - break; - if (filter->action == KVM_PMU_EVENT_ALLOW && - i == filter->nevents) - allow_event = false; - if (filter->action == KVM_PMU_EVENT_DENY && - i < filter->nevents) - allow_event = false; + __u64 key = eventsel & AMD64_RAW_EVENT_MASK_NB; + + if (bsearch(&key, filter->events, filter->nevents, + sizeof(__u64), cmp_u64)) + allow_event = filter->action == KVM_PMU_EVENT_ALLOW; + else + allow_event = filter->action == KVM_PMU_EVENT_DENY; } if (!allow_event) return; @@ -263,7 +267,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) pmc->current_config = (u64)ctrl; pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE, - kvm_x86_ops.pmu_ops->find_fixed_event(idx), + kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc), !(en_field & 0x2), /* exclude user */ !(en_field & 0x1), /* exclude kernel */ pmi, false, false); @@ -314,7 +318,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) } /* check if idx is a valid index to access PMU */ -int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) +bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { return kvm_x86_ops.pmu_ops->is_valid_rdpmc_ecx(vcpu, idx); } @@ -485,6 +489,66 @@ void kvm_pmu_destroy(struct kvm_vcpu *vcpu) kvm_pmu_reset(vcpu); } +static void kvm_pmu_incr_counter(struct kvm_pmc *pmc) +{ + struct kvm_pmu *pmu = pmc_to_pmu(pmc); + u64 prev_count; + + prev_count = pmc->counter; + pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc); + + reprogram_counter(pmu, pmc->idx); + if (pmc->counter < prev_count) + __kvm_perf_overflow(pmc, false); +} + +static inline bool eventsel_match_perf_hw_id(struct kvm_pmc *pmc, + unsigned int perf_hw_id) +{ + u64 old_eventsel = pmc->eventsel; + unsigned int config; + + pmc->eventsel &= (ARCH_PERFMON_EVENTSEL_EVENT | ARCH_PERFMON_EVENTSEL_UMASK); + config = kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc); + pmc->eventsel = old_eventsel; + return config == perf_hw_id; +} + +static inline bool cpl_is_matched(struct kvm_pmc *pmc) +{ + bool select_os, select_user; + u64 config = pmc->current_config; + + if (pmc_is_gp(pmc)) { + select_os = config & ARCH_PERFMON_EVENTSEL_OS; + select_user = config & ARCH_PERFMON_EVENTSEL_USR; + } else { + select_os = config & 0x1; + select_user = config & 0x2; + } + + return (static_call(kvm_x86_get_cpl)(pmc->vcpu) == 0) ? select_os : select_user; +} + +void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 perf_hw_id) +{ + struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + struct kvm_pmc *pmc; + int i; + + for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) { + pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, i); + + if (!pmc || !pmc_is_enabled(pmc) || !pmc_speculative_in_use(pmc)) + continue; + + /* Ignore checks for edge detect, pin control, invert and CMASK bits */ + if (eventsel_match_perf_hw_id(pmc, perf_hw_id) && cpl_is_matched(pmc)) + kvm_pmu_incr_counter(pmc); + } +} +EXPORT_SYMBOL_GPL(kvm_pmu_trigger_event); + int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp) { struct kvm_pmu_event_filter tmp, *filter; @@ -516,6 +580,11 @@ int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp) /* Ensure nevents can't be changed between the user copies. */ *filter = tmp; + /* + * Sort the in-kernel list so that we can search it with bsearch. + */ + sort(&filter->events, filter->nevents, sizeof(__u64), cmp_u64, NULL); + mutex_lock(&kvm->lock); filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter, mutex_is_locked(&kvm->lock)); diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index a06d95165a..7a7b8d5b77 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -25,13 +25,12 @@ struct kvm_event_hw_type_mapping { struct kvm_pmu_ops { unsigned int (*pmc_perf_hw_id)(struct kvm_pmc *pmc); - unsigned (*find_fixed_event)(int idx); bool (*pmc_is_enabled)(struct kvm_pmc *pmc); struct kvm_pmc *(*pmc_idx_to_pmc)(struct kvm_pmu *pmu, int pmc_idx); struct kvm_pmc *(*rdpmc_ecx_to_pmc)(struct kvm_vcpu *vcpu, unsigned int idx, u64 *mask); struct kvm_pmc *(*msr_idx_to_pmc)(struct kvm_vcpu *vcpu, u32 msr); - int (*is_valid_rdpmc_ecx)(struct kvm_vcpu *vcpu, unsigned int idx); + bool (*is_valid_rdpmc_ecx)(struct kvm_vcpu *vcpu, unsigned int idx); bool (*is_valid_msr)(struct kvm_vcpu *vcpu, u32 msr); int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr_info); int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr_info); @@ -148,7 +147,7 @@ void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx); void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu); void kvm_pmu_handle_event(struct kvm_vcpu *vcpu); int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned pmc, u64 *data); -int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx); +bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx); bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr); int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info); int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info); @@ -158,6 +157,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu); void kvm_pmu_cleanup(struct kvm_vcpu *vcpu); void kvm_pmu_destroy(struct kvm_vcpu *vcpu); int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp); +void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 perf_hw_id); bool is_vmware_backdoor_pmc(u32 pmc_idx); diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index cdbb48e127..fb3e207913 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -27,20 +27,6 @@ #include "irq.h" #include "svm.h" -#define SVM_AVIC_DOORBELL 0xc001011b - -#define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) - -/* - * 0xff is broadcast, so the max index allowed for physical APIC ID - * table is 0xfe. APIC IDs above 0xff are reserved. - */ -#define AVIC_MAX_PHYSICAL_ID_COUNT 255 - -#define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 -#define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 -#define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF - /* AVIC GATAG is encoded using VM and VCPU IDs */ #define AVIC_VCPU_ID_BITS 8 #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) @@ -73,12 +59,6 @@ struct amd_svm_iommu_ir { void *data; /* Storing pointer to struct amd_ir_data */ }; -enum avic_ipi_failure_cause { - AVIC_IPI_FAILURE_INVALID_INT_TYPE, - AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, - AVIC_IPI_FAILURE_INVALID_TARGET, - AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, -}; /* Note: * This function is called from IOMMU driver to notify @@ -289,20 +269,44 @@ static int avic_init_backing_page(struct kvm_vcpu *vcpu) return 0; } +void avic_ring_doorbell(struct kvm_vcpu *vcpu) +{ + /* + * Note, the vCPU could get migrated to a different pCPU at any point, + * which could result in signalling the wrong/previous pCPU. But if + * that happens the vCPU is guaranteed to do a VMRUN (after being + * migrated) and thus will process pending interrupts, i.e. a doorbell + * is not needed (and the spurious one is harmless). + */ + int cpu = READ_ONCE(vcpu->cpu); + + if (cpu != get_cpu()) + wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu)); + put_cpu(); +} + static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, u32 icrl, u32 icrh) { struct kvm_vcpu *vcpu; - int i; + unsigned long i; + /* + * Wake any target vCPUs that are blocking, i.e. waiting for a wake + * event. There's no need to signal doorbells, as hardware has handled + * vCPUs that were in guest at the time of the IPI, and vCPUs that have + * since entered the guest will have processed pending IRQs at VMRUN. + */ kvm_for_each_vcpu(i, vcpu, kvm) { - bool m = kvm_apic_match_dest(vcpu, source, - icrl & APIC_SHORT_MASK, - GET_APIC_DEST_FIELD(icrh), - icrl & APIC_DEST_MASK); - - if (m && !avic_vcpu_is_running(vcpu)) - kvm_vcpu_wake_up(vcpu); + if (kvm_apic_match_dest(vcpu, source, icrl & APIC_SHORT_MASK, + GET_APIC_DEST_FIELD(icrh), + icrl & APIC_DEST_MASK)) { + vcpu->arch.apic->irr_pending = true; + svm_complete_interrupt_delivery(vcpu, + icrl & APIC_MODE_MASK, + icrl & APIC_INT_LEVELTRIG, + icrl & APIC_VECTOR_MASK); + } } } @@ -664,26 +668,6 @@ void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) return; } -int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec) -{ - if (!vcpu->arch.apicv_active) - return -1; - - kvm_lapic_set_irr(vec, vcpu->arch.apic); - smp_mb__after_atomic(); - - if (avic_vcpu_is_running(vcpu)) { - int cpuid = vcpu->cpu; - - if (cpuid != get_cpu()) - wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid)); - put_cpu(); - } else - kvm_vcpu_wake_up(vcpu); - - return 0; -} - bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) { return false; @@ -898,11 +882,13 @@ int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, bool svm_check_apicv_inhibit_reasons(ulong bit) { ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | + BIT(APICV_INHIBIT_REASON_ABSENT) | BIT(APICV_INHIBIT_REASON_HYPERV) | BIT(APICV_INHIBIT_REASON_NESTED) | BIT(APICV_INHIBIT_REASON_IRQWIN) | BIT(APICV_INHIBIT_REASON_PIT_REINJ) | - BIT(APICV_INHIBIT_REASON_X2APIC); + BIT(APICV_INHIBIT_REASON_X2APIC) | + BIT(APICV_INHIBIT_REASON_BLOCKIRQ); return supported & BIT(bit); } @@ -945,6 +931,8 @@ void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) int h_physical_id = kvm_cpu_get_apicid(cpu); struct vcpu_svm *svm = to_svm(vcpu); + lockdep_assert_preemption_disabled(); + /* * Since the host physical APIC id is 8 bits, * we can support host APIC ID upto 255. @@ -952,19 +940,25 @@ void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) return; + /* + * No need to update anything if the vCPU is blocking, i.e. if the vCPU + * is being scheduled in after being preempted. The CPU entries in the + * Physical APIC table and IRTE are consumed iff IsRun{ning} is '1'. + * If the vCPU was migrated, its new CPU value will be stuffed when the + * vCPU unblocks. + */ + if (kvm_vcpu_is_blocking(vcpu)) + return; + entry = READ_ONCE(*(svm->avic_physical_id_cache)); WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); - - entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; - if (svm->avic_is_running) - entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; + entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; WRITE_ONCE(*(svm->avic_physical_id_cache), entry); - avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, - svm->avic_is_running); + avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, true); } void avic_vcpu_put(struct kvm_vcpu *vcpu) @@ -972,42 +966,56 @@ void avic_vcpu_put(struct kvm_vcpu *vcpu) u64 entry; struct vcpu_svm *svm = to_svm(vcpu); + lockdep_assert_preemption_disabled(); + entry = READ_ONCE(*(svm->avic_physical_id_cache)); - if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) - avic_update_iommu_vcpu_affinity(vcpu, -1, 0); + + /* Nothing to do if IsRunning == '0' due to vCPU blocking. */ + if (!(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)) + return; + + avic_update_iommu_vcpu_affinity(vcpu, -1, 0); entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; WRITE_ONCE(*(svm->avic_physical_id_cache), entry); } -/* - * This function is called during VCPU halt/unhalt. - */ -static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run) +void avic_vcpu_blocking(struct kvm_vcpu *vcpu) { - struct vcpu_svm *svm = to_svm(vcpu); - int cpu = get_cpu(); + if (!kvm_vcpu_apicv_active(vcpu)) + return; + preempt_disable(); + + /* + * Unload the AVIC when the vCPU is about to block, _before_ + * the vCPU actually blocks. + * + * Any IRQs that arrive before IsRunning=0 will not cause an + * incomplete IPI vmexit on the source, therefore vIRR will also + * be checked by kvm_vcpu_check_block() before blocking. The + * memory barrier implicit in set_current_state orders writing + * IsRunning=0 before reading the vIRR. The processor needs a + * matching memory barrier on interrupt delivery between writing + * IRR and reading IsRunning; the lack of this barrier might be + * the cause of errata #1235). + */ + avic_vcpu_put(vcpu); + + preempt_enable(); +} + +void avic_vcpu_unblocking(struct kvm_vcpu *vcpu) +{ + int cpu; + + if (!kvm_vcpu_apicv_active(vcpu)) + return; + + cpu = get_cpu(); WARN_ON(cpu != vcpu->cpu); - svm->avic_is_running = is_run; - if (kvm_vcpu_apicv_active(vcpu)) { - if (is_run) - avic_vcpu_load(vcpu, cpu); - else - avic_vcpu_put(vcpu); - } + avic_vcpu_load(vcpu, cpu); + put_cpu(); } - -void svm_vcpu_blocking(struct kvm_vcpu *vcpu) -{ - avic_set_running(vcpu, false); -} - -void svm_vcpu_unblocking(struct kvm_vcpu *vcpu) -{ - if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) - kvm_vcpu_update_apicv(vcpu); - avic_set_running(vcpu, true); -} diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 556e7a3f35..39d280e7e8 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -58,8 +58,9 @@ static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_excep struct vcpu_svm *svm = to_svm(vcpu); WARN_ON(!is_guest_mode(vcpu)); - if (vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && - !svm->nested.nested_run_pending) { + if (vmcb12_is_intercept(&svm->nested.ctl, + INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && + !svm->nested.nested_run_pending) { svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; svm->vmcb->control.exit_code_hi = 0; svm->vmcb->control.exit_info_1 = fault->error_code; @@ -121,7 +122,8 @@ static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) void recalc_intercepts(struct vcpu_svm *svm) { - struct vmcb_control_area *c, *h, *g; + struct vmcb_control_area *c, *h; + struct vmcb_ctrl_area_cached *g; unsigned int i; vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); @@ -163,37 +165,6 @@ void recalc_intercepts(struct vcpu_svm *svm) vmcb_set_intercept(c, INTERCEPT_VMSAVE); } -static void copy_vmcb_control_area(struct vmcb_control_area *dst, - struct vmcb_control_area *from) -{ - unsigned int i; - - for (i = 0; i < MAX_INTERCEPT; i++) - dst->intercepts[i] = from->intercepts[i]; - - dst->iopm_base_pa = from->iopm_base_pa; - dst->msrpm_base_pa = from->msrpm_base_pa; - dst->tsc_offset = from->tsc_offset; - /* asid not copied, it is handled manually for svm->vmcb. */ - dst->tlb_ctl = from->tlb_ctl; - dst->int_ctl = from->int_ctl; - dst->int_vector = from->int_vector; - dst->int_state = from->int_state; - dst->exit_code = from->exit_code; - dst->exit_code_hi = from->exit_code_hi; - dst->exit_info_1 = from->exit_info_1; - dst->exit_info_2 = from->exit_info_2; - dst->exit_int_info = from->exit_int_info; - dst->exit_int_info_err = from->exit_int_info_err; - dst->nested_ctl = from->nested_ctl; - dst->event_inj = from->event_inj; - dst->event_inj_err = from->event_inj_err; - dst->nested_cr3 = from->nested_cr3; - dst->virt_ext = from->virt_ext; - dst->pause_filter_count = from->pause_filter_count; - dst->pause_filter_thresh = from->pause_filter_thresh; -} - static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) { /* @@ -203,7 +174,7 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) */ int i; - if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) + if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) return true; for (i = 0; i < MSRPM_OFFSETS; i++) { @@ -238,10 +209,22 @@ static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); } -static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu, - struct vmcb_control_area *control) +static bool nested_svm_check_tlb_ctl(struct kvm_vcpu *vcpu, u8 tlb_ctl) { - if (CC(!vmcb_is_intercept(control, INTERCEPT_VMRUN))) + /* Nested FLUSHBYASID is not supported yet. */ + switch(tlb_ctl) { + case TLB_CONTROL_DO_NOTHING: + case TLB_CONTROL_FLUSH_ALL_ASID: + return true; + default: + return false; + } +} + +static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu, + struct vmcb_ctrl_area_cached *control) +{ + if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN))) return false; if (CC(control->asid == 0)) @@ -257,12 +240,26 @@ static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu, IOPM_SIZE))) return false; + if (CC(!nested_svm_check_tlb_ctl(vcpu, control->tlb_ctl))) + return false; + return true; } -static bool nested_vmcb_check_cr3_cr4(struct kvm_vcpu *vcpu, - struct vmcb_save_area *save) +/* Common checks that apply to both L1 and L2 state. */ +static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu, + struct vmcb_save_area_cached *save) { + if (CC(!(save->efer & EFER_SVME))) + return false; + + if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || + CC(save->cr0 & ~0xffffffffULL)) + return false; + + if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) + return false; + /* * These checks are also performed by KVM_SET_SREGS, * except that EFER.LMA is not checked by SVM against @@ -278,48 +275,90 @@ static bool nested_vmcb_check_cr3_cr4(struct kvm_vcpu *vcpu, if (CC(!kvm_is_valid_cr4(vcpu, save->cr4))) return false; - return true; -} - -/* Common checks that apply to both L1 and L2 state. */ -static bool nested_vmcb_valid_sregs(struct kvm_vcpu *vcpu, - struct vmcb_save_area *save) -{ - /* - * FIXME: these should be done after copying the fields, - * to avoid TOC/TOU races. For these save area checks - * the possible damage is limited since kvm_set_cr0 and - * kvm_set_cr4 handle failure; EFER_SVME is an exception - * so it is force-set later in nested_prepare_vmcb_save. - */ - if (CC(!(save->efer & EFER_SVME))) - return false; - - if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || - CC(save->cr0 & ~0xffffffffULL)) - return false; - - if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) - return false; - - if (!nested_vmcb_check_cr3_cr4(vcpu, save)) - return false; - if (CC(!kvm_valid_efer(vcpu, save->efer))) return false; return true; } -void nested_load_control_from_vmcb12(struct vcpu_svm *svm, - struct vmcb_control_area *control) +static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu) { - copy_vmcb_control_area(&svm->nested.ctl, control); + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb_save_area_cached *save = &svm->nested.save; - /* Copy it here because nested_svm_check_controls will check it. */ - svm->nested.ctl.asid = control->asid; - svm->nested.ctl.msrpm_base_pa &= ~0x0fffULL; - svm->nested.ctl.iopm_base_pa &= ~0x0fffULL; + return __nested_vmcb_check_save(vcpu, save); +} + +static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb_ctrl_area_cached *ctl = &svm->nested.ctl; + + return __nested_vmcb_check_controls(vcpu, ctl); +} + +static +void __nested_copy_vmcb_control_to_cache(struct vmcb_ctrl_area_cached *to, + struct vmcb_control_area *from) +{ + unsigned int i; + + for (i = 0; i < MAX_INTERCEPT; i++) + to->intercepts[i] = from->intercepts[i]; + + to->iopm_base_pa = from->iopm_base_pa; + to->msrpm_base_pa = from->msrpm_base_pa; + to->tsc_offset = from->tsc_offset; + to->tlb_ctl = from->tlb_ctl; + to->int_ctl = from->int_ctl; + to->int_vector = from->int_vector; + to->int_state = from->int_state; + to->exit_code = from->exit_code; + to->exit_code_hi = from->exit_code_hi; + to->exit_info_1 = from->exit_info_1; + to->exit_info_2 = from->exit_info_2; + to->exit_int_info = from->exit_int_info; + to->exit_int_info_err = from->exit_int_info_err; + to->nested_ctl = from->nested_ctl; + to->event_inj = from->event_inj; + to->event_inj_err = from->event_inj_err; + to->nested_cr3 = from->nested_cr3; + to->virt_ext = from->virt_ext; + to->pause_filter_count = from->pause_filter_count; + to->pause_filter_thresh = from->pause_filter_thresh; + + /* Copy asid here because nested_vmcb_check_controls will check it. */ + to->asid = from->asid; + to->msrpm_base_pa &= ~0x0fffULL; + to->iopm_base_pa &= ~0x0fffULL; +} + +void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, + struct vmcb_control_area *control) +{ + __nested_copy_vmcb_control_to_cache(&svm->nested.ctl, control); +} + +static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to, + struct vmcb_save_area *from) +{ + /* + * Copy only fields that are validated, as we need them + * to avoid TOC/TOU races. + */ + to->efer = from->efer; + to->cr0 = from->cr0; + to->cr3 = from->cr3; + to->cr4 = from->cr4; + + to->dr6 = from->dr6; + to->dr7 = from->dr7; +} + +void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, + struct vmcb_save_area *save) +{ + __nested_copy_vmcb_save_to_cache(&svm->nested.save, save); } /* @@ -422,14 +461,13 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, return -EINVAL; if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && - CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) + CC(!load_pdptrs(vcpu, cr3))) return -EINVAL; if (!nested_npt) kvm_mmu_new_pgd(vcpu, cr3); vcpu->arch.cr3 = cr3; - kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ kvm_init_mmu(vcpu); @@ -475,15 +513,10 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED); - /* - * Force-set EFER_SVME even though it is checked earlier on the - * VMCB12, because the guest can flip the bit between the check - * and now. Clearing EFER_SVME would call svm_free_nested. - */ - svm_set_efer(&svm->vcpu, vmcb12->save.efer | EFER_SVME); + svm_set_efer(&svm->vcpu, svm->nested.save.efer); - svm_set_cr0(&svm->vcpu, vmcb12->save.cr0); - svm_set_cr4(&svm->vcpu, vmcb12->save.cr4); + svm_set_cr0(&svm->vcpu, svm->nested.save.cr0); + svm_set_cr4(&svm->vcpu, svm->nested.save.cr4); svm->vcpu.arch.cr2 = vmcb12->save.cr2; @@ -498,8 +531,8 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 /* These bits will be set properly on the first execution when new_vmc12 is true */ if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) { - svm->vmcb->save.dr7 = vmcb12->save.dr7 | DR7_FIXED_1; - svm->vcpu.arch.dr6 = vmcb12->save.dr6 | DR6_ACTIVE_LOW; + svm->vmcb->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; + svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; vmcb_mark_dirty(svm->vmcb, VMCB_DR); } } @@ -538,8 +571,17 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) if (nested_npt_enabled(svm)) nested_svm_init_mmu_context(vcpu); - svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset = - vcpu->arch.l1_tsc_offset + svm->nested.ctl.tsc_offset; + vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( + vcpu->arch.l1_tsc_offset, + svm->nested.ctl.tsc_offset, + svm->tsc_ratio_msr); + + svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset; + + if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { + WARN_ON(!svm->tsc_scaling_enabled); + nested_svm_update_tsc_ratio_msr(vcpu); + } svm->vmcb->control.int_ctl = (svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) | @@ -550,9 +592,6 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) svm->vmcb->control.event_inj = svm->nested.ctl.event_inj; svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; - svm->vmcb->control.pause_filter_count = svm->nested.ctl.pause_filter_count; - svm->vmcb->control.pause_filter_thresh = svm->nested.ctl.pause_filter_thresh; - nested_svm_transition_tlb_flush(vcpu); /* Enter Guest-Mode */ @@ -607,7 +646,7 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, nested_vmcb02_prepare_control(svm); nested_vmcb02_prepare_save(svm, vmcb12); - ret = nested_svm_load_cr3(&svm->vcpu, vmcb12->save.cr3, + ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3, nested_npt_enabled(svm), from_vmrun); if (ret) return ret; @@ -657,10 +696,11 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) if (WARN_ON_ONCE(!svm->nested.initialized)) return -EINVAL; - nested_load_control_from_vmcb12(svm, &vmcb12->control); + nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); + nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); - if (!nested_vmcb_valid_sregs(vcpu, &vmcb12->save) || - !nested_vmcb_check_controls(vcpu, &svm->nested.ctl)) { + if (!nested_vmcb_check_save(vcpu) || + !nested_vmcb_check_controls(vcpu)) { vmcb12->control.exit_code = SVM_EXIT_ERR; vmcb12->control.exit_code_hi = 0; vmcb12->control.exit_info_1 = 0; @@ -810,11 +850,6 @@ int nested_svm_vmexit(struct vcpu_svm *svm) vmcb12->control.event_inj = svm->nested.ctl.event_inj; vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; - vmcb12->control.pause_filter_count = - svm->vmcb->control.pause_filter_count; - vmcb12->control.pause_filter_thresh = - svm->vmcb->control.pause_filter_thresh; - nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); svm_switch_vmcb(svm, &svm->vmcb01); @@ -832,6 +867,12 @@ int nested_svm_vmexit(struct vcpu_svm *svm) vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); } + if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { + WARN_ON(!svm->tsc_scaling_enabled); + vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; + svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); + } + svm->nested.ctl.nested_cr3 = 0; /* @@ -966,7 +1007,7 @@ static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) u32 offset, msr, value; int write, mask; - if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) + if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) return NESTED_EXIT_HOST; msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; @@ -993,7 +1034,7 @@ static int nested_svm_intercept_ioio(struct vcpu_svm *svm) u8 start_bit; u64 gpa; - if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) + if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) return NESTED_EXIT_HOST; port = svm->vmcb->control.exit_info_1 >> 16; @@ -1024,12 +1065,12 @@ static int nested_svm_intercept(struct vcpu_svm *svm) vmexit = nested_svm_intercept_ioio(svm); break; case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { - if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) + if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) vmexit = NESTED_EXIT_DONE; break; } case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { - if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) + if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) vmexit = NESTED_EXIT_DONE; break; } @@ -1047,7 +1088,7 @@ static int nested_svm_intercept(struct vcpu_svm *svm) break; } default: { - if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) + if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) vmexit = NESTED_EXIT_DONE; } } @@ -1125,7 +1166,7 @@ static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) static inline bool nested_exit_on_init(struct vcpu_svm *svm) { - return vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); + return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); } static int svm_check_nested_events(struct kvm_vcpu *vcpu) @@ -1219,11 +1260,57 @@ int nested_svm_exit_special(struct vcpu_svm *svm) return NESTED_EXIT_CONTINUE; } +void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + vcpu->arch.tsc_scaling_ratio = + kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio, + svm->tsc_ratio_msr); + svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); +} + +/* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */ +static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst, + struct vmcb_ctrl_area_cached *from) +{ + unsigned int i; + + memset(dst, 0, sizeof(struct vmcb_control_area)); + + for (i = 0; i < MAX_INTERCEPT; i++) + dst->intercepts[i] = from->intercepts[i]; + + dst->iopm_base_pa = from->iopm_base_pa; + dst->msrpm_base_pa = from->msrpm_base_pa; + dst->tsc_offset = from->tsc_offset; + dst->asid = from->asid; + dst->tlb_ctl = from->tlb_ctl; + dst->int_ctl = from->int_ctl; + dst->int_vector = from->int_vector; + dst->int_state = from->int_state; + dst->exit_code = from->exit_code; + dst->exit_code_hi = from->exit_code_hi; + dst->exit_info_1 = from->exit_info_1; + dst->exit_info_2 = from->exit_info_2; + dst->exit_int_info = from->exit_int_info; + dst->exit_int_info_err = from->exit_int_info_err; + dst->nested_ctl = from->nested_ctl; + dst->event_inj = from->event_inj; + dst->event_inj_err = from->event_inj_err; + dst->nested_cr3 = from->nested_cr3; + dst->virt_ext = from->virt_ext; + dst->pause_filter_count = from->pause_filter_count; + dst->pause_filter_thresh = from->pause_filter_thresh; +} + static int svm_get_nested_state(struct kvm_vcpu *vcpu, struct kvm_nested_state __user *user_kvm_nested_state, u32 user_data_size) { struct vcpu_svm *svm; + struct vmcb_control_area *ctl; + unsigned long r; struct kvm_nested_state kvm_state = { .flags = 0, .format = KVM_STATE_NESTED_FORMAT_SVM, @@ -1265,9 +1352,18 @@ static int svm_get_nested_state(struct kvm_vcpu *vcpu, */ if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) return -EFAULT; - if (copy_to_user(&user_vmcb->control, &svm->nested.ctl, - sizeof(user_vmcb->control))) + + ctl = kzalloc(sizeof(*ctl), GFP_KERNEL); + if (!ctl) + return -ENOMEM; + + nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl); + r = copy_to_user(&user_vmcb->control, ctl, + sizeof(user_vmcb->control)); + kfree(ctl); + if (r) return -EFAULT; + if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save, sizeof(user_vmcb->save))) return -EFAULT; @@ -1284,6 +1380,8 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, &user_kvm_nested_state->data.svm[0]; struct vmcb_control_area *ctl; struct vmcb_save_area *save; + struct vmcb_save_area_cached save_cached; + struct vmcb_ctrl_area_cached ctl_cached; unsigned long cr0; int ret; @@ -1336,7 +1434,8 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, goto out_free; ret = -EINVAL; - if (!nested_vmcb_check_controls(vcpu, ctl)) + __nested_copy_vmcb_control_to_cache(&ctl_cached, ctl); + if (!__nested_vmcb_check_controls(vcpu, &ctl_cached)) goto out_free; /* @@ -1351,10 +1450,11 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, * Validate host state saved from before VMRUN (see * nested_svm_check_permissions). */ + __nested_copy_vmcb_save_to_cache(&save_cached, save); if (!(save->cr0 & X86_CR0_PG) || !(save->cr0 & X86_CR0_PE) || (save->rflags & X86_EFLAGS_VM) || - !nested_vmcb_valid_sregs(vcpu, save)) + !__nested_vmcb_check_save(vcpu, &save_cached)) goto out_free; @@ -1378,7 +1478,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); - nested_load_control_from_vmcb12(svm, ctl); + nested_copy_vmcb_control_to_cache(svm, ctl); svm_switch_vmcb(svm, &svm->nested.vmcb02); nested_vmcb02_prepare_control(svm); @@ -1419,7 +1519,7 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) * the guest CR3 might be restored prior to setting the nested * state which can lead to a load of wrong PDPTRs. */ - if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3))) + if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) return false; if (!nested_svm_vmrun_msrpm(svm)) { diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 06f8034f62..5aa45f13b1 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -16,6 +16,7 @@ #include "cpuid.h" #include "lapic.h" #include "pmu.h" +#include "svm.h" enum pmu_type { PMU_TYPE_COUNTER = 0, @@ -100,6 +101,9 @@ static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr, { struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu); + if (!enable_pmu) + return NULL; + switch (msr) { case MSR_F15H_PERF_CTL0: case MSR_F15H_PERF_CTL1: @@ -140,6 +144,10 @@ static unsigned int amd_pmc_perf_hw_id(struct kvm_pmc *pmc) u8 unit_mask = (pmc->eventsel & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; int i; + /* return PERF_COUNT_HW_MAX as AMD doesn't have fixed events */ + if (WARN_ON(pmc_is_fixed(pmc))) + return PERF_COUNT_HW_MAX; + for (i = 0; i < ARRAY_SIZE(amd_event_mapping); i++) if (amd_event_mapping[i].eventsel == event_select && amd_event_mapping[i].unit_mask == unit_mask) @@ -151,12 +159,6 @@ static unsigned int amd_pmc_perf_hw_id(struct kvm_pmc *pmc) return amd_event_mapping[i].event_type; } -/* return PERF_COUNT_HW_MAX as AMD doesn't have fixed events */ -static unsigned amd_find_fixed_event(int idx) -{ - return PERF_COUNT_HW_MAX; -} - /* check if a PMC is enabled by comparing it against global_ctrl bits. Because * AMD CPU doesn't have global_ctrl MSR, all PMCs are enabled (return TRUE). */ @@ -181,14 +183,13 @@ static struct kvm_pmc *amd_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx) return get_gp_pmc_amd(pmu, base + pmc_idx, PMU_TYPE_COUNTER); } -/* returns 0 if idx's corresponding MSR exists; otherwise returns 1. */ -static int amd_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) +static bool amd_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); idx &= ~(3u << 30); - return (idx >= pmu->nr_arch_gp_counters); + return idx < pmu->nr_arch_gp_counters; } /* idx is the ECX register of RDPMC instruction */ @@ -321,7 +322,6 @@ static void amd_pmu_reset(struct kvm_vcpu *vcpu) struct kvm_pmu_ops amd_pmu_ops = { .pmc_perf_hw_id = amd_pmc_perf_hw_id, - .find_fixed_event = amd_find_fixed_event, .pmc_is_enabled = amd_pmc_is_enabled, .pmc_idx_to_pmc = amd_pmc_idx_to_pmc, .rdpmc_ecx_to_pmc = amd_rdpmc_ecx_to_pmc, diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 134c4ea5e6..17b53457d8 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -17,10 +17,10 @@ #include #include #include -#include #include #include +#include #include "x86.h" #include "svm.h" @@ -120,16 +120,26 @@ static bool __sev_recycle_asids(int min_asid, int max_asid) return true; } +static int sev_misc_cg_try_charge(struct kvm_sev_info *sev) +{ + enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; + return misc_cg_try_charge(type, sev->misc_cg, 1); +} + +static void sev_misc_cg_uncharge(struct kvm_sev_info *sev) +{ + enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; + misc_cg_uncharge(type, sev->misc_cg, 1); +} + static int sev_asid_new(struct kvm_sev_info *sev) { int asid, min_asid, max_asid, ret; bool retry = true; - enum misc_res_type type; - type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; WARN_ON(sev->misc_cg); sev->misc_cg = get_current_misc_cg(); - ret = misc_cg_try_charge(type, sev->misc_cg, 1); + ret = sev_misc_cg_try_charge(sev); if (ret) { put_misc_cg(sev->misc_cg); sev->misc_cg = NULL; @@ -162,7 +172,7 @@ static int sev_asid_new(struct kvm_sev_info *sev) return asid; e_uncharge: - misc_cg_uncharge(type, sev->misc_cg, 1); + sev_misc_cg_uncharge(sev); put_misc_cg(sev->misc_cg); sev->misc_cg = NULL; return ret; @@ -179,7 +189,6 @@ static void sev_asid_free(struct kvm_sev_info *sev) { struct svm_cpu_data *sd; int cpu; - enum misc_res_type type; mutex_lock(&sev_bitmap_lock); @@ -192,8 +201,7 @@ static void sev_asid_free(struct kvm_sev_info *sev) mutex_unlock(&sev_bitmap_lock); - type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; - misc_cg_uncharge(type, sev->misc_cg, 1); + sev_misc_cg_uncharge(sev); put_misc_cg(sev->misc_cg); sev->misc_cg = NULL; } @@ -229,7 +237,6 @@ static void sev_unbind_asid(struct kvm *kvm, unsigned int handle) static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) { struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - bool es_active = argp->id == KVM_SEV_ES_INIT; int asid, ret; if (kvm->created_vcpus) @@ -239,7 +246,8 @@ static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) if (unlikely(sev->active)) return ret; - sev->es_active = es_active; + sev->active = true; + sev->es_active = argp->id == KVM_SEV_ES_INIT; asid = sev_asid_new(sev); if (asid < 0) goto e_no_asid; @@ -249,8 +257,6 @@ static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) if (ret) goto e_free; - sev->active = true; - sev->asid = asid; INIT_LIST_HEAD(&sev->regions_list); return 0; @@ -260,6 +266,7 @@ static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) sev->asid = 0; e_no_asid: sev->es_active = false; + sev->active = false; return ret; } @@ -590,7 +597,7 @@ static int sev_es_sync_vmsa(struct vcpu_svm *svm) * traditional VMSA as it has been built so far (in prep * for LAUNCH_UPDATE_VMSA) to be the initial SEV-ES state. */ - memcpy(svm->vmsa, save, sizeof(*save)); + memcpy(svm->sev_es.vmsa, save, sizeof(*save)); return 0; } @@ -612,11 +619,11 @@ static int __sev_launch_update_vmsa(struct kvm *kvm, struct kvm_vcpu *vcpu, * the VMSA memory content (i.e it will write the same memory region * with the guest's key), so invalidate it first. */ - clflush_cache_range(svm->vmsa, PAGE_SIZE); + clflush_cache_range(svm->sev_es.vmsa, PAGE_SIZE); vmsa.reserved = 0; vmsa.handle = to_kvm_svm(kvm)->sev_info.handle; - vmsa.address = __sme_pa(svm->vmsa); + vmsa.address = __sme_pa(svm->sev_es.vmsa); vmsa.len = PAGE_SIZE; ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_VMSA, &vmsa, error); if (ret) @@ -629,7 +636,8 @@ static int __sev_launch_update_vmsa(struct kvm *kvm, struct kvm_vcpu *vcpu, static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) { struct kvm_vcpu *vcpu; - int i, ret; + unsigned long i; + int ret; if (!sev_es_guest(kvm)) return -ENOTTY; @@ -1522,7 +1530,7 @@ static int sev_receive_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) return sev_issue_cmd(kvm, SEV_CMD_RECEIVE_FINISH, &data, &argp->error); } -static bool cmd_allowed_from_miror(u32 cmd_id) +static bool is_cmd_allowed_from_mirror(u32 cmd_id) { /* * Allow mirrors VM to call KVM_SEV_LAUNCH_UPDATE_VMSA to enable SEV-ES @@ -1536,6 +1544,223 @@ static bool cmd_allowed_from_miror(u32 cmd_id) return false; } +static int sev_lock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) +{ + struct kvm_sev_info *dst_sev = &to_kvm_svm(dst_kvm)->sev_info; + struct kvm_sev_info *src_sev = &to_kvm_svm(src_kvm)->sev_info; + int r = -EBUSY; + + if (dst_kvm == src_kvm) + return -EINVAL; + + /* + * Bail if these VMs are already involved in a migration to avoid + * deadlock between two VMs trying to migrate to/from each other. + */ + if (atomic_cmpxchg_acquire(&dst_sev->migration_in_progress, 0, 1)) + return -EBUSY; + + if (atomic_cmpxchg_acquire(&src_sev->migration_in_progress, 0, 1)) + goto release_dst; + + r = -EINTR; + if (mutex_lock_killable(&dst_kvm->lock)) + goto release_src; + if (mutex_lock_killable_nested(&src_kvm->lock, SINGLE_DEPTH_NESTING)) + goto unlock_dst; + return 0; + +unlock_dst: + mutex_unlock(&dst_kvm->lock); +release_src: + atomic_set_release(&src_sev->migration_in_progress, 0); +release_dst: + atomic_set_release(&dst_sev->migration_in_progress, 0); + return r; +} + +static void sev_unlock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) +{ + struct kvm_sev_info *dst_sev = &to_kvm_svm(dst_kvm)->sev_info; + struct kvm_sev_info *src_sev = &to_kvm_svm(src_kvm)->sev_info; + + mutex_unlock(&dst_kvm->lock); + mutex_unlock(&src_kvm->lock); + atomic_set_release(&dst_sev->migration_in_progress, 0); + atomic_set_release(&src_sev->migration_in_progress, 0); +} + + +static int sev_lock_vcpus_for_migration(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu; + unsigned long i, j; + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (mutex_lock_killable(&vcpu->mutex)) + goto out_unlock; + } + + return 0; + +out_unlock: + kvm_for_each_vcpu(j, vcpu, kvm) { + if (i == j) + break; + + mutex_unlock(&vcpu->mutex); + } + return -EINTR; +} + +static void sev_unlock_vcpus_for_migration(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu; + unsigned long i; + + kvm_for_each_vcpu(i, vcpu, kvm) { + mutex_unlock(&vcpu->mutex); + } +} + +static void sev_migrate_from(struct kvm_sev_info *dst, + struct kvm_sev_info *src) +{ + dst->active = true; + dst->asid = src->asid; + dst->handle = src->handle; + dst->pages_locked = src->pages_locked; + dst->enc_context_owner = src->enc_context_owner; + + src->asid = 0; + src->active = false; + src->handle = 0; + src->pages_locked = 0; + src->enc_context_owner = NULL; + + list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); +} + +static int sev_es_migrate_from(struct kvm *dst, struct kvm *src) +{ + unsigned long i; + struct kvm_vcpu *dst_vcpu, *src_vcpu; + struct vcpu_svm *dst_svm, *src_svm; + + if (atomic_read(&src->online_vcpus) != atomic_read(&dst->online_vcpus)) + return -EINVAL; + + kvm_for_each_vcpu(i, src_vcpu, src) { + if (!src_vcpu->arch.guest_state_protected) + return -EINVAL; + } + + kvm_for_each_vcpu(i, src_vcpu, src) { + src_svm = to_svm(src_vcpu); + dst_vcpu = kvm_get_vcpu(dst, i); + dst_svm = to_svm(dst_vcpu); + + /* + * Transfer VMSA and GHCB state to the destination. Nullify and + * clear source fields as appropriate, the state now belongs to + * the destination. + */ + memcpy(&dst_svm->sev_es, &src_svm->sev_es, sizeof(src_svm->sev_es)); + dst_svm->vmcb->control.ghcb_gpa = src_svm->vmcb->control.ghcb_gpa; + dst_svm->vmcb->control.vmsa_pa = src_svm->vmcb->control.vmsa_pa; + dst_vcpu->arch.guest_state_protected = true; + + memset(&src_svm->sev_es, 0, sizeof(src_svm->sev_es)); + src_svm->vmcb->control.ghcb_gpa = INVALID_PAGE; + src_svm->vmcb->control.vmsa_pa = INVALID_PAGE; + src_vcpu->arch.guest_state_protected = false; + } + to_kvm_svm(src)->sev_info.es_active = false; + to_kvm_svm(dst)->sev_info.es_active = true; + + return 0; +} + +int svm_vm_migrate_from(struct kvm *kvm, unsigned int source_fd) +{ + struct kvm_sev_info *dst_sev = &to_kvm_svm(kvm)->sev_info; + struct kvm_sev_info *src_sev, *cg_cleanup_sev; + struct file *source_kvm_file; + struct kvm *source_kvm; + bool charged = false; + int ret; + + source_kvm_file = fget(source_fd); + if (!file_is_kvm(source_kvm_file)) { + ret = -EBADF; + goto out_fput; + } + + source_kvm = source_kvm_file->private_data; + ret = sev_lock_two_vms(kvm, source_kvm); + if (ret) + goto out_fput; + + if (sev_guest(kvm) || !sev_guest(source_kvm)) { + ret = -EINVAL; + goto out_unlock; + } + + src_sev = &to_kvm_svm(source_kvm)->sev_info; + + /* + * VMs mirroring src's encryption context rely on it to keep the + * ASID allocated, but below we are clearing src_sev->asid. + */ + if (src_sev->num_mirrored_vms) { + ret = -EBUSY; + goto out_unlock; + } + + dst_sev->misc_cg = get_current_misc_cg(); + cg_cleanup_sev = dst_sev; + if (dst_sev->misc_cg != src_sev->misc_cg) { + ret = sev_misc_cg_try_charge(dst_sev); + if (ret) + goto out_dst_cgroup; + charged = true; + } + + ret = sev_lock_vcpus_for_migration(kvm); + if (ret) + goto out_dst_cgroup; + ret = sev_lock_vcpus_for_migration(source_kvm); + if (ret) + goto out_dst_vcpu; + + if (sev_es_guest(source_kvm)) { + ret = sev_es_migrate_from(kvm, source_kvm); + if (ret) + goto out_source_vcpu; + } + sev_migrate_from(dst_sev, src_sev); + kvm_vm_dead(source_kvm); + cg_cleanup_sev = src_sev; + ret = 0; + +out_source_vcpu: + sev_unlock_vcpus_for_migration(source_kvm); +out_dst_vcpu: + sev_unlock_vcpus_for_migration(kvm); +out_dst_cgroup: + /* Operates on the source on success, on the destination on failure. */ + if (charged) + sev_misc_cg_uncharge(cg_cleanup_sev); + put_misc_cg(cg_cleanup_sev->misc_cg); + cg_cleanup_sev->misc_cg = NULL; +out_unlock: + sev_unlock_two_vms(kvm, source_kvm); +out_fput: + if (source_kvm_file) + fput(source_kvm_file); + return ret; +} + int svm_mem_enc_op(struct kvm *kvm, void __user *argp) { struct kvm_sev_cmd sev_cmd; @@ -1554,7 +1779,7 @@ int svm_mem_enc_op(struct kvm *kvm, void __user *argp) /* Only the enc_context_owner handles some memory enc operations. */ if (is_mirroring_enc_context(kvm) && - !cmd_allowed_from_miror(sev_cmd.id)) { + !is_cmd_allowed_from_mirror(sev_cmd.id)) { r = -EINVAL; goto out; } @@ -1751,77 +1976,60 @@ int svm_vm_copy_asid_from(struct kvm *kvm, unsigned int source_fd) { struct file *source_kvm_file; struct kvm *source_kvm; - struct kvm_sev_info source_sev, *mirror_sev; + struct kvm_sev_info *source_sev, *mirror_sev; int ret; source_kvm_file = fget(source_fd); if (!file_is_kvm(source_kvm_file)) { ret = -EBADF; - goto e_source_put; + goto e_source_fput; } source_kvm = source_kvm_file->private_data; - mutex_lock(&source_kvm->lock); + ret = sev_lock_two_vms(kvm, source_kvm); + if (ret) + goto e_source_fput; - if (!sev_guest(source_kvm)) { + /* + * Mirrors of mirrors should work, but let's not get silly. Also + * disallow out-of-band SEV/SEV-ES init if the target is already an + * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being + * created after SEV/SEV-ES initialization, e.g. to init intercepts. + */ + if (sev_guest(kvm) || !sev_guest(source_kvm) || + is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) { ret = -EINVAL; - goto e_source_unlock; + goto e_unlock; } - /* Mirrors of mirrors should work, but let's not get silly */ - if (is_mirroring_enc_context(source_kvm) || source_kvm == kvm) { - ret = -EINVAL; - goto e_source_unlock; - } - - memcpy(&source_sev, &to_kvm_svm(source_kvm)->sev_info, - sizeof(source_sev)); - /* * The mirror kvm holds an enc_context_owner ref so its asid can't * disappear until we're done with it */ + source_sev = &to_kvm_svm(source_kvm)->sev_info; kvm_get_kvm(source_kvm); - - fput(source_kvm_file); - mutex_unlock(&source_kvm->lock); - mutex_lock(&kvm->lock); - - /* - * Disallow out-of-band SEV/SEV-ES init if the target is already an - * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being - * created after SEV/SEV-ES initialization, e.g. to init intercepts. - */ - if (sev_guest(kvm) || kvm->created_vcpus) { - ret = -EINVAL; - goto e_mirror_unlock; - } + source_sev->num_mirrored_vms++; /* Set enc_context_owner and copy its encryption context over */ mirror_sev = &to_kvm_svm(kvm)->sev_info; mirror_sev->enc_context_owner = source_kvm; mirror_sev->active = true; - mirror_sev->asid = source_sev.asid; - mirror_sev->fd = source_sev.fd; - mirror_sev->es_active = source_sev.es_active; - mirror_sev->handle = source_sev.handle; + mirror_sev->asid = source_sev->asid; + mirror_sev->fd = source_sev->fd; + mirror_sev->es_active = source_sev->es_active; + mirror_sev->handle = source_sev->handle; INIT_LIST_HEAD(&mirror_sev->regions_list); + ret = 0; + /* * Do not copy ap_jump_table. Since the mirror does not share the same * KVM contexts as the original, and they may have different * memory-views. */ - mutex_unlock(&kvm->lock); - return 0; - -e_mirror_unlock: - mutex_unlock(&kvm->lock); - kvm_put_kvm(source_kvm); - return ret; -e_source_unlock: - mutex_unlock(&source_kvm->lock); -e_source_put: +e_unlock: + sev_unlock_two_vms(kvm, source_kvm); +e_source_fput: if (source_kvm_file) fput(source_kvm_file); return ret; @@ -1833,17 +2041,24 @@ void sev_vm_destroy(struct kvm *kvm) struct list_head *head = &sev->regions_list; struct list_head *pos, *q; + WARN_ON(sev->num_mirrored_vms); + if (!sev_guest(kvm)) return; /* If this is a mirror_kvm release the enc_context_owner and skip sev cleanup */ if (is_mirroring_enc_context(kvm)) { - kvm_put_kvm(sev->enc_context_owner); + struct kvm *owner_kvm = sev->enc_context_owner; + struct kvm_sev_info *owner_sev = &to_kvm_svm(owner_kvm)->sev_info; + + mutex_lock(&owner_kvm->lock); + if (!WARN_ON(!owner_sev->num_mirrored_vms)) + owner_sev->num_mirrored_vms--; + mutex_unlock(&owner_kvm->lock); + kvm_put_kvm(owner_kvm); return; } - mutex_lock(&kvm->lock); - /* * Ensure that all guest tagged cache entries are flushed before * releasing the pages back to the system for use. CLFLUSH will @@ -1863,8 +2078,6 @@ void sev_vm_destroy(struct kvm *kvm) } } - mutex_unlock(&kvm->lock); - sev_unbind_asid(kvm, sev->handle); sev_asid_free(sev); } @@ -1887,8 +2100,13 @@ void __init sev_hardware_setup(void) if (!sev_enabled || !npt_enabled) goto out; - /* Does the CPU support SEV? */ - if (!boot_cpu_has(X86_FEATURE_SEV)) + /* + * SEV must obviously be supported in hardware. Sanity check that the + * CPU supports decode assists, which is mandatory for SEV guests to + * support instruction emulation. + */ + if (!boot_cpu_has(X86_FEATURE_SEV) || + WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_DECODEASSISTS))) goto out; /* Retrieve SEV CPUID information */ @@ -2044,16 +2262,16 @@ void sev_free_vcpu(struct kvm_vcpu *vcpu) svm = to_svm(vcpu); if (vcpu->arch.guest_state_protected) - sev_flush_guest_memory(svm, svm->vmsa, PAGE_SIZE); - __free_page(virt_to_page(svm->vmsa)); + sev_flush_guest_memory(svm, svm->sev_es.vmsa, PAGE_SIZE); + __free_page(virt_to_page(svm->sev_es.vmsa)); - if (svm->ghcb_sa_free) - kfree(svm->ghcb_sa); + if (svm->sev_es.ghcb_sa_free) + kvfree(svm->sev_es.ghcb_sa); } static void dump_ghcb(struct vcpu_svm *svm) { - struct ghcb *ghcb = svm->ghcb; + struct ghcb *ghcb = svm->sev_es.ghcb; unsigned int nbits; /* Re-use the dump_invalid_vmcb module parameter */ @@ -2079,7 +2297,7 @@ static void dump_ghcb(struct vcpu_svm *svm) static void sev_es_sync_to_ghcb(struct vcpu_svm *svm) { struct kvm_vcpu *vcpu = &svm->vcpu; - struct ghcb *ghcb = svm->ghcb; + struct ghcb *ghcb = svm->sev_es.ghcb; /* * The GHCB protocol so far allows for the following data @@ -2099,7 +2317,7 @@ static void sev_es_sync_from_ghcb(struct vcpu_svm *svm) { struct vmcb_control_area *control = &svm->vmcb->control; struct kvm_vcpu *vcpu = &svm->vcpu; - struct ghcb *ghcb = svm->ghcb; + struct ghcb *ghcb = svm->sev_es.ghcb; u64 exit_code; /* @@ -2140,24 +2358,29 @@ static void sev_es_sync_from_ghcb(struct vcpu_svm *svm) memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); } -static int sev_es_validate_vmgexit(struct vcpu_svm *svm) +static bool sev_es_validate_vmgexit(struct vcpu_svm *svm) { struct kvm_vcpu *vcpu; struct ghcb *ghcb; - u64 exit_code = 0; + u64 exit_code; + u64 reason; - ghcb = svm->ghcb; - - /* Only GHCB Usage code 0 is supported */ - if (ghcb->ghcb_usage) - goto vmgexit_err; + ghcb = svm->sev_es.ghcb; /* - * Retrieve the exit code now even though is may not be marked valid + * Retrieve the exit code now even though it may not be marked valid * as it could help with debugging. */ exit_code = ghcb_get_sw_exit_code(ghcb); + /* Only GHCB Usage code 0 is supported */ + if (ghcb->ghcb_usage) { + reason = GHCB_ERR_INVALID_USAGE; + goto vmgexit_err; + } + + reason = GHCB_ERR_MISSING_INPUT; + if (!ghcb_sw_exit_code_is_valid(ghcb) || !ghcb_sw_exit_info_1_is_valid(ghcb) || !ghcb_sw_exit_info_2_is_valid(ghcb)) @@ -2236,61 +2459,66 @@ static int sev_es_validate_vmgexit(struct vcpu_svm *svm) case SVM_VMGEXIT_UNSUPPORTED_EVENT: break; default: + reason = GHCB_ERR_INVALID_EVENT; goto vmgexit_err; } - return 0; + return true; vmgexit_err: vcpu = &svm->vcpu; - if (ghcb->ghcb_usage) { + if (reason == GHCB_ERR_INVALID_USAGE) { vcpu_unimpl(vcpu, "vmgexit: ghcb usage %#x is not valid\n", ghcb->ghcb_usage); + } else if (reason == GHCB_ERR_INVALID_EVENT) { + vcpu_unimpl(vcpu, "vmgexit: exit code %#llx is not valid\n", + exit_code); } else { - vcpu_unimpl(vcpu, "vmgexit: exit reason %#llx is not valid\n", + vcpu_unimpl(vcpu, "vmgexit: exit code %#llx input is not valid\n", exit_code); dump_ghcb(svm); } - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; - vcpu->run->internal.ndata = 2; - vcpu->run->internal.data[0] = exit_code; - vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; + /* Clear the valid entries fields */ + memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); - return -EINVAL; + ghcb_set_sw_exit_info_1(ghcb, 2); + ghcb_set_sw_exit_info_2(ghcb, reason); + + return false; } void sev_es_unmap_ghcb(struct vcpu_svm *svm) { - if (!svm->ghcb) + if (!svm->sev_es.ghcb) return; - if (svm->ghcb_sa_free) { + if (svm->sev_es.ghcb_sa_free) { /* * The scratch area lives outside the GHCB, so there is a * buffer that, depending on the operation performed, may * need to be synced, then freed. */ - if (svm->ghcb_sa_sync) { + if (svm->sev_es.ghcb_sa_sync) { kvm_write_guest(svm->vcpu.kvm, - ghcb_get_sw_scratch(svm->ghcb), - svm->ghcb_sa, svm->ghcb_sa_len); - svm->ghcb_sa_sync = false; + ghcb_get_sw_scratch(svm->sev_es.ghcb), + svm->sev_es.ghcb_sa, + svm->sev_es.ghcb_sa_len); + svm->sev_es.ghcb_sa_sync = false; } - kfree(svm->ghcb_sa); - svm->ghcb_sa = NULL; - svm->ghcb_sa_free = false; + kvfree(svm->sev_es.ghcb_sa); + svm->sev_es.ghcb_sa = NULL; + svm->sev_es.ghcb_sa_free = false; } - trace_kvm_vmgexit_exit(svm->vcpu.vcpu_id, svm->ghcb); + trace_kvm_vmgexit_exit(svm->vcpu.vcpu_id, svm->sev_es.ghcb); sev_es_sync_to_ghcb(svm); - kvm_vcpu_unmap(&svm->vcpu, &svm->ghcb_map, true); - svm->ghcb = NULL; + kvm_vcpu_unmap(&svm->vcpu, &svm->sev_es.ghcb_map, true); + svm->sev_es.ghcb = NULL; } void pre_sev_run(struct vcpu_svm *svm, int cpu) @@ -2317,10 +2545,10 @@ void pre_sev_run(struct vcpu_svm *svm, int cpu) } #define GHCB_SCRATCH_AREA_LIMIT (16ULL * PAGE_SIZE) -static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len) +static bool setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len) { struct vmcb_control_area *control = &svm->vmcb->control; - struct ghcb *ghcb = svm->ghcb; + struct ghcb *ghcb = svm->sev_es.ghcb; u64 ghcb_scratch_beg, ghcb_scratch_end; u64 scratch_gpa_beg, scratch_gpa_end; void *scratch_va; @@ -2328,14 +2556,14 @@ static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len) scratch_gpa_beg = ghcb_get_sw_scratch(ghcb); if (!scratch_gpa_beg) { pr_err("vmgexit: scratch gpa not provided\n"); - return -EINVAL; + goto e_scratch; } scratch_gpa_end = scratch_gpa_beg + len; if (scratch_gpa_end < scratch_gpa_beg) { pr_err("vmgexit: scratch length (%#llx) not valid for scratch address (%#llx)\n", len, scratch_gpa_beg); - return -EINVAL; + goto e_scratch; } if ((scratch_gpa_beg & PAGE_MASK) == control->ghcb_gpa) { @@ -2353,10 +2581,10 @@ static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len) scratch_gpa_end > ghcb_scratch_end) { pr_err("vmgexit: scratch area is outside of GHCB shared buffer area (%#llx - %#llx)\n", scratch_gpa_beg, scratch_gpa_end); - return -EINVAL; + goto e_scratch; } - scratch_va = (void *)svm->ghcb; + scratch_va = (void *)svm->sev_es.ghcb; scratch_va += (scratch_gpa_beg - control->ghcb_gpa); } else { /* @@ -2366,18 +2594,18 @@ static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len) if (len > GHCB_SCRATCH_AREA_LIMIT) { pr_err("vmgexit: scratch area exceeds KVM limits (%#llx requested, %#llx limit)\n", len, GHCB_SCRATCH_AREA_LIMIT); - return -EINVAL; + goto e_scratch; } - scratch_va = kzalloc(len, GFP_KERNEL_ACCOUNT); + scratch_va = kvzalloc(len, GFP_KERNEL_ACCOUNT); if (!scratch_va) - return -ENOMEM; + goto e_scratch; if (kvm_read_guest(svm->vcpu.kvm, scratch_gpa_beg, scratch_va, len)) { /* Unable to copy scratch area from guest */ pr_err("vmgexit: kvm_read_guest for scratch area failed\n"); - kfree(scratch_va); - return -EFAULT; + kvfree(scratch_va); + goto e_scratch; } /* @@ -2386,14 +2614,20 @@ static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len) * the vCPU next time (i.e. a read was requested so the data * must be written back to the guest memory). */ - svm->ghcb_sa_sync = sync; - svm->ghcb_sa_free = true; + svm->sev_es.ghcb_sa_sync = sync; + svm->sev_es.ghcb_sa_free = true; } - svm->ghcb_sa = scratch_va; - svm->ghcb_sa_len = len; + svm->sev_es.ghcb_sa = scratch_va; + svm->sev_es.ghcb_sa_len = len; - return 0; + return true; + +e_scratch: + ghcb_set_sw_exit_info_1(ghcb, 2); + ghcb_set_sw_exit_info_2(ghcb, GHCB_ERR_INVALID_SCRATCH_AREA); + + return false; } static void set_ghcb_msr_bits(struct vcpu_svm *svm, u64 value, u64 mask, @@ -2444,7 +2678,7 @@ static int sev_handle_vmgexit_msr_protocol(struct vcpu_svm *svm) ret = svm_invoke_exit_handler(vcpu, SVM_EXIT_CPUID); if (!ret) { - ret = -EINVAL; + /* Error, keep GHCB MSR value as-is */ break; } @@ -2480,10 +2714,13 @@ static int sev_handle_vmgexit_msr_protocol(struct vcpu_svm *svm) GHCB_MSR_TERM_REASON_POS); pr_info("SEV-ES guest requested termination: %#llx:%#llx\n", reason_set, reason_code); - fallthrough; + + ret = -EINVAL; + break; } default: - ret = -EINVAL; + /* Error, keep GHCB MSR value as-is */ + break; } trace_kvm_vmgexit_msr_protocol_exit(svm->vcpu.vcpu_id, @@ -2507,51 +2744,53 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu) if (!ghcb_gpa) { vcpu_unimpl(vcpu, "vmgexit: GHCB gpa is not set\n"); - return -EINVAL; + + /* Without a GHCB, just return right back to the guest */ + return 1; } - if (kvm_vcpu_map(vcpu, ghcb_gpa >> PAGE_SHIFT, &svm->ghcb_map)) { + if (kvm_vcpu_map(vcpu, ghcb_gpa >> PAGE_SHIFT, &svm->sev_es.ghcb_map)) { /* Unable to map GHCB from guest */ vcpu_unimpl(vcpu, "vmgexit: error mapping GHCB [%#llx] from guest\n", ghcb_gpa); - return -EINVAL; + + /* Without a GHCB, just return right back to the guest */ + return 1; } - svm->ghcb = svm->ghcb_map.hva; - ghcb = svm->ghcb_map.hva; + svm->sev_es.ghcb = svm->sev_es.ghcb_map.hva; + ghcb = svm->sev_es.ghcb_map.hva; trace_kvm_vmgexit_enter(vcpu->vcpu_id, ghcb); exit_code = ghcb_get_sw_exit_code(ghcb); - ret = sev_es_validate_vmgexit(svm); - if (ret) - return ret; + if (!sev_es_validate_vmgexit(svm)) + return 1; sev_es_sync_from_ghcb(svm); ghcb_set_sw_exit_info_1(ghcb, 0); ghcb_set_sw_exit_info_2(ghcb, 0); + ret = 1; switch (exit_code) { case SVM_VMGEXIT_MMIO_READ: - ret = setup_vmgexit_scratch(svm, true, control->exit_info_2); - if (ret) + if (!setup_vmgexit_scratch(svm, true, control->exit_info_2)) break; ret = kvm_sev_es_mmio_read(vcpu, control->exit_info_1, control->exit_info_2, - svm->ghcb_sa); + svm->sev_es.ghcb_sa); break; case SVM_VMGEXIT_MMIO_WRITE: - ret = setup_vmgexit_scratch(svm, false, control->exit_info_2); - if (ret) + if (!setup_vmgexit_scratch(svm, false, control->exit_info_2)) break; ret = kvm_sev_es_mmio_write(vcpu, control->exit_info_1, control->exit_info_2, - svm->ghcb_sa); + svm->sev_es.ghcb_sa); break; case SVM_VMGEXIT_NMI_COMPLETE: ret = svm_invoke_exit_handler(vcpu, SVM_EXIT_IRET); @@ -2574,14 +2813,10 @@ int sev_handle_vmgexit(struct kvm_vcpu *vcpu) default: pr_err("svm: vmgexit: unsupported AP jump table request - exit_info_1=%#llx\n", control->exit_info_1); - ghcb_set_sw_exit_info_1(ghcb, 1); - ghcb_set_sw_exit_info_2(ghcb, - X86_TRAP_UD | - SVM_EVTINJ_TYPE_EXEPT | - SVM_EVTINJ_VALID); + ghcb_set_sw_exit_info_1(ghcb, 2); + ghcb_set_sw_exit_info_2(ghcb, GHCB_ERR_INVALID_INPUT); } - ret = 1; break; } case SVM_VMGEXIT_UNSUPPORTED_EVENT: @@ -2601,7 +2836,6 @@ int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in) { int count; int bytes; - int r; if (svm->vmcb->control.exit_info_2 > INT_MAX) return -EINVAL; @@ -2610,11 +2844,11 @@ int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in) if (unlikely(check_mul_overflow(count, size, &bytes))) return -EINVAL; - r = setup_vmgexit_scratch(svm, in, bytes); - if (r) - return r; + if (!setup_vmgexit_scratch(svm, in, bytes)) + return 1; - return kvm_sev_es_string_io(&svm->vcpu, size, port, svm->ghcb_sa, count, in); + return kvm_sev_es_string_io(&svm->vcpu, size, port, svm->sev_es.ghcb_sa, + count, in); } void sev_es_init_vmcb(struct vcpu_svm *svm) @@ -2629,7 +2863,7 @@ void sev_es_init_vmcb(struct vcpu_svm *svm) * VMCB page. Do not include the encryption mask on the VMSA physical * address since hardware will access it using the guest key. */ - svm->vmcb->control.vmsa_pa = __pa(svm->vmsa); + svm->vmcb->control.vmsa_pa = __pa(svm->sev_es.vmsa); /* Can't intercept CR register access, HV can't modify CR registers */ svm_clr_intercept(svm, INTERCEPT_CR0_READ); @@ -2662,11 +2896,11 @@ void sev_es_init_vmcb(struct vcpu_svm *svm) set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); } -void sev_es_create_vcpu(struct vcpu_svm *svm) +void sev_es_vcpu_reset(struct vcpu_svm *svm) { /* - * Set the GHCB MSR value as per the GHCB specification when creating - * a vCPU for an SEV-ES guest. + * Set the GHCB MSR value as per the GHCB specification when emulating + * vCPU RESET for an SEV-ES guest. */ set_ghcb_msr(svm, GHCB_MSR_SEV_INFO(GHCB_VERSION_MAX, GHCB_VERSION_MIN, @@ -2701,8 +2935,8 @@ void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) struct vcpu_svm *svm = to_svm(vcpu); /* First SIPI: Use the values as initially set by the VMM */ - if (!svm->received_first_sipi) { - svm->received_first_sipi = true; + if (!svm->sev_es.received_first_sipi) { + svm->sev_es.received_first_sipi = true; return; } @@ -2711,8 +2945,8 @@ void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) * the guest will set the CS and RIP. Set SW_EXIT_INFO_2 to a * non-zero value. */ - if (!svm->ghcb) + if (!svm->sev_es.ghcb) return; - ghcb_set_sw_exit_info_2(svm->ghcb, 1); + ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, 1); } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 26f2da1590..fd3a00c892 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,7 @@ #include #include #include +#include #include #include "trace.h" @@ -186,6 +188,13 @@ module_param(vls, int, 0444); static int vgif = true; module_param(vgif, int, 0444); +/* enable/disable LBR virtualization */ +static int lbrv = true; +module_param(lbrv, int, 0444); + +static int tsc_scaling = true; +module_param(tsc_scaling, int, 0444); + /* * enable / disable AVIC. Because the defaults differ for APICv * support between VMX and SVM we cannot use module_param_named. @@ -256,7 +265,7 @@ u32 svm_msrpm_offset(u32 msr) #define MAX_INST_SIZE 15 -static int get_max_npt_level(void) +static int get_npt_level(void) { #ifdef CONFIG_X86_64 return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; @@ -459,7 +468,7 @@ static int has_svm(void) return 0; } - if (sev_active()) { + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { pr_info("KVM is unsupported when running as an SEV guest\n"); return 0; } @@ -470,7 +479,7 @@ static int has_svm(void) static void svm_hardware_disable(void) { /* Make sure we clean up behind us */ - if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) + if (tsc_scaling) wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); cpu_svm_disable(); @@ -513,6 +522,10 @@ static int svm_hardware_enable(void) wrmsrl(MSR_VM_HSAVE_PA, __sme_page_pa(sd->save_area)); if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { + /* + * Set the default value, even if we don't use TSC scaling + * to avoid having stale value in the msr + */ wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT); } @@ -576,12 +589,10 @@ static int svm_cpu_init(int cpu) if (!sd) return ret; sd->cpu = cpu; - sd->save_area = alloc_page(GFP_KERNEL); + sd->save_area = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!sd->save_area) goto free_cpu_data; - clear_page(page_address(sd->save_area)); - ret = sev_cpu_init(sd); if (ret) goto free_save_area; @@ -862,47 +873,6 @@ static void shrink_ple_window(struct kvm_vcpu *vcpu) } } -/* - * The default MMIO mask is a single bit (excluding the present bit), - * which could conflict with the memory encryption bit. Check for - * memory encryption support and override the default MMIO mask if - * memory encryption is enabled. - */ -static __init void svm_adjust_mmio_mask(void) -{ - unsigned int enc_bit, mask_bit; - u64 msr, mask; - - /* If there is no memory encryption support, use existing mask */ - if (cpuid_eax(0x80000000) < 0x8000001f) - return; - - /* If memory encryption is not enabled, use existing mask */ - rdmsrl(MSR_AMD64_SYSCFG, msr); - if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) - return; - - enc_bit = cpuid_ebx(0x8000001f) & 0x3f; - mask_bit = boot_cpu_data.x86_phys_bits; - - /* Increment the mask bit if it is the same as the encryption bit */ - if (enc_bit == mask_bit) - mask_bit++; - - /* - * If the mask bit location is below 52, then some bits above the - * physical addressing limit will always be reserved, so use the - * rsvd_bits() function to generate the mask. This mask, along with - * the present bit, will be used to generate a page fault with - * PFER.RSV = 1. - * - * If the mask bit location is 52 (or above), then clear the mask. - */ - mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0; - - kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK); -} - static void svm_hardware_teardown(void) { int cpu; @@ -917,176 +887,6 @@ static void svm_hardware_teardown(void) iopm_base = 0; } -static __init void svm_set_cpu_caps(void) -{ - kvm_set_cpu_caps(); - - supported_xss = 0; - - /* CPUID 0x80000001 and 0x8000000A (SVM features) */ - if (nested) { - kvm_cpu_cap_set(X86_FEATURE_SVM); - - if (nrips) - kvm_cpu_cap_set(X86_FEATURE_NRIPS); - - if (npt_enabled) - kvm_cpu_cap_set(X86_FEATURE_NPT); - - /* Nested VM can receive #VMEXIT instead of triggering #GP */ - kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); - } - - /* CPUID 0x80000008 */ - if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || - boot_cpu_has(X86_FEATURE_AMD_SSBD)) - kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); - - /* CPUID 0x8000001F (SME/SEV features) */ - sev_set_cpu_caps(); -} - -static __init int svm_hardware_setup(void) -{ - int cpu; - struct page *iopm_pages; - void *iopm_va; - int r; - unsigned int order = get_order(IOPM_SIZE); - - /* - * NX is required for shadow paging and for NPT if the NX huge pages - * mitigation is enabled. - */ - if (!boot_cpu_has(X86_FEATURE_NX)) { - pr_err_ratelimited("NX (Execute Disable) not supported\n"); - return -EOPNOTSUPP; - } - kvm_enable_efer_bits(EFER_NX); - - iopm_pages = alloc_pages(GFP_KERNEL, order); - - if (!iopm_pages) - return -ENOMEM; - - iopm_va = page_address(iopm_pages); - memset(iopm_va, 0xff, PAGE_SIZE * (1 << order)); - iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; - - init_msrpm_offsets(); - - supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); - - if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) - kvm_enable_efer_bits(EFER_FFXSR); - - if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { - kvm_has_tsc_control = true; - kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX; - kvm_tsc_scaling_ratio_frac_bits = 32; - } - - tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX); - - /* Check for pause filtering support */ - if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { - pause_filter_count = 0; - pause_filter_thresh = 0; - } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) { - pause_filter_thresh = 0; - } - - if (nested) { - printk(KERN_INFO "kvm: Nested Virtualization enabled\n"); - kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE); - } - - /* - * KVM's MMU doesn't support using 2-level paging for itself, and thus - * NPT isn't supported if the host is using 2-level paging since host - * CR4 is unchanged on VMRUN. - */ - if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE)) - npt_enabled = false; - - if (!boot_cpu_has(X86_FEATURE_NPT)) - npt_enabled = false; - - /* Force VM NPT level equal to the host's max NPT level */ - kvm_configure_mmu(npt_enabled, get_max_npt_level(), - get_max_npt_level(), PG_LEVEL_1G); - pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); - - /* Note, SEV setup consumes npt_enabled. */ - sev_hardware_setup(); - - svm_hv_hardware_setup(); - - svm_adjust_mmio_mask(); - - for_each_possible_cpu(cpu) { - r = svm_cpu_init(cpu); - if (r) - goto err; - } - - if (nrips) { - if (!boot_cpu_has(X86_FEATURE_NRIPS)) - nrips = false; - } - - enable_apicv = avic = avic && npt_enabled && boot_cpu_has(X86_FEATURE_AVIC); - - if (enable_apicv) { - pr_info("AVIC enabled\n"); - - amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); - } - - if (vls) { - if (!npt_enabled || - !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) || - !IS_ENABLED(CONFIG_X86_64)) { - vls = false; - } else { - pr_info("Virtual VMLOAD VMSAVE supported\n"); - } - } - - if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK)) - svm_gp_erratum_intercept = false; - - if (vgif) { - if (!boot_cpu_has(X86_FEATURE_VGIF)) - vgif = false; - else - pr_info("Virtual GIF supported\n"); - } - - svm_set_cpu_caps(); - - /* - * It seems that on AMD processors PTE's accessed bit is - * being set by the CPU hardware before the NPF vmexit. - * This is not expected behaviour and our tests fail because - * of it. - * A workaround here is to disable support for - * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled. - * In this case userspace can know if there is support using - * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle - * it - * If future AMD CPU models change the behaviour described above, - * this variable can be changed accordingly - */ - allow_smaller_maxphyaddr = !npt_enabled; - - return 0; - -err: - svm_hardware_teardown(); - return r; -} - static void init_seg(struct vmcb_seg *seg) { seg->selector = 0; @@ -1113,7 +913,9 @@ static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu) static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) { - return kvm_default_tsc_scaling_ratio; + struct vcpu_svm *svm = to_svm(vcpu); + + return svm->tsc_ratio_msr; } static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) @@ -1125,7 +927,7 @@ static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); } -static void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier) +void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier) { wrmsrl(MSR_AMD64_TSC_RATIO, multiplier); } @@ -1154,6 +956,38 @@ static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu, } } +static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (guest_cpuid_is_intel(vcpu)) { + /* + * We must intercept SYSENTER_EIP and SYSENTER_ESP + * accesses because the processor only stores 32 bits. + * For the same reason we cannot use virtual VMLOAD/VMSAVE. + */ + svm_set_intercept(svm, INTERCEPT_VMLOAD); + svm_set_intercept(svm, INTERCEPT_VMSAVE); + svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; + + set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0); + set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0); + } else { + /* + * If hardware supports Virtual VMLOAD VMSAVE then enable it + * in VMCB and clear intercepts to avoid #VMEXIT. + */ + if (vls) { + svm_clr_intercept(svm, INTERCEPT_VMLOAD); + svm_clr_intercept(svm, INTERCEPT_VMSAVE); + svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; + } + /* No need to intercept these MSRs */ + set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1); + set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1); + } +} + static void init_vmcb(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -1301,11 +1135,25 @@ static void init_vmcb(struct kvm_vcpu *vcpu) } svm_hv_init_vmcb(svm->vmcb); + init_vmcb_after_set_cpuid(vcpu); vmcb_mark_all_dirty(svm->vmcb); enable_gif(svm); +} +static void __svm_vcpu_reset(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm_vcpu_init_msrpm(vcpu, svm->msrpm); + + svm_init_osvw(vcpu); + vcpu->arch.microcode_version = 0x01000065; + svm->tsc_ratio_msr = kvm_default_tsc_scaling_ratio; + + if (sev_es_guest(vcpu->kvm)) + sev_es_vcpu_reset(svm); } static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) @@ -1316,6 +1164,9 @@ static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) svm->virt_spec_ctrl = 0; init_vmcb(vcpu); + + if (!init_event) + __svm_vcpu_reset(vcpu); } void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb) @@ -1351,22 +1202,16 @@ static int svm_create_vcpu(struct kvm_vcpu *vcpu) /* * SEV-ES guests maintain an encrypted version of their FPU * state which is restored and saved on VMRUN and VMEXIT. - * Free the fpu structure to prevent KVM from attempting to - * access the FPU state. + * Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't + * do xsave/xrstor on it. */ - kvm_free_guest_fpu(vcpu); + fpstate_set_confidential(&vcpu->arch.guest_fpu); } err = avic_init_vcpu(svm); if (err) goto error_free_vmsa_page; - /* We initialize this flag to true to make sure that the is_running - * bit would be set the first time the vcpu is loaded. - */ - if (irqchip_in_kernel(vcpu->kvm) && kvm_apicv_activated(vcpu->kvm)) - svm->avic_is_running = true; - svm->msrpm = svm_vcpu_alloc_msrpm(); if (!svm->msrpm) { err = -ENOMEM; @@ -1375,24 +1220,13 @@ static int svm_create_vcpu(struct kvm_vcpu *vcpu) svm->vmcb01.ptr = page_address(vmcb01_page); svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT); + svm_switch_vmcb(svm, &svm->vmcb01); if (vmsa_page) - svm->vmsa = page_address(vmsa_page); + svm->sev_es.vmsa = page_address(vmsa_page); svm->guest_state_loaded = false; - svm_switch_vmcb(svm, &svm->vmcb01); - init_vmcb(vcpu); - - svm_vcpu_init_msrpm(vcpu, svm->msrpm); - - svm_init_osvw(vcpu); - vcpu->arch.microcode_version = 0x01000065; - - if (sev_es_guest(vcpu->kvm)) - /* Perform SEV-ES specific VMCB creation updates */ - sev_es_create_vcpu(svm); - return 0; error_free_vmsa_page: @@ -1452,7 +1286,7 @@ static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) vmsave(__sme_page_pa(sd->save_area)); } - if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { + if (tsc_scaling) { u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio; if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) { __this_cpu_write(current_tsc_ratio, tsc_ratio); @@ -1533,10 +1367,16 @@ static bool svm_get_if_flag(struct kvm_vcpu *vcpu) static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) { + kvm_register_mark_available(vcpu, reg); + switch (reg) { case VCPU_EXREG_PDPTR: - BUG_ON(!npt_enabled); - load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); + /* + * When !npt_enabled, mmu->pdptrs[] is already available since + * it is always updated per SDM when moving to CRs. + */ + if (npt_enabled) + load_pdptrs(vcpu, kvm_read_cr3(vcpu)); break; default: KVM_BUG_ON(1, vcpu->kvm); @@ -1723,6 +1563,24 @@ static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) vmcb_mark_dirty(svm->vmcb, VMCB_DT); } +static void svm_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + /* + * For guests that don't set guest_state_protected, the cr3 update is + * handled via kvm_mmu_load() while entering the guest. For guests + * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to + * VMCB save area now, since the save area will become the initial + * contents of the VMSA, and future VMCB save area updates won't be + * seen. + */ + if (sev_es_guest(vcpu->kvm)) { + svm->vmcb->save.cr3 = cr3; + vmcb_mark_dirty(svm->vmcb, VMCB_CR); + } +} + void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { struct vcpu_svm *svm = to_svm(vcpu); @@ -2463,7 +2321,7 @@ static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu, bool ret = false; if (!is_guest_mode(vcpu) || - (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))) + (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))) return false; cr0 &= ~SVM_CR0_SELECTIVE_MASK; @@ -2680,6 +2538,11 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) struct vcpu_svm *svm = to_svm(vcpu); switch (msr_info->index) { + case MSR_AMD64_TSC_RATIO: + if (!msr_info->host_initiated && !svm->tsc_scaling_enabled) + return 1; + msr_info->data = svm->tsc_ratio_msr; + break; case MSR_STAR: msr_info->data = svm->vmcb01.ptr->save.star; break; @@ -2785,11 +2648,11 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) { struct vcpu_svm *svm = to_svm(vcpu); - if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->ghcb)) + if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb)) return kvm_complete_insn_gp(vcpu, err); - ghcb_set_sw_exit_info_1(svm->ghcb, 1); - ghcb_set_sw_exit_info_2(svm->ghcb, + ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1); + ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, X86_TRAP_GP | SVM_EVTINJ_TYPE_EXEPT | SVM_EVTINJ_VALID); @@ -2829,6 +2692,34 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) u32 ecx = msr->index; u64 data = msr->data; switch (ecx) { + case MSR_AMD64_TSC_RATIO: + + if (!svm->tsc_scaling_enabled) { + + if (!msr->host_initiated) + return 1; + /* + * In case TSC scaling is not enabled, always + * leave this MSR at the default value. + * + * Due to bug in qemu 6.2.0, it would try to set + * this msr to 0 if tsc scaling is not enabled. + * Ignore this value as well. + */ + if (data != 0 && data != svm->tsc_ratio_msr) + return 1; + break; + } + + if (data & TSC_RATIO_RSVD) + return 1; + + svm->tsc_ratio_msr = data; + + if (svm->tsc_scaling_enabled && is_guest_mode(vcpu)) + nested_svm_update_tsc_ratio_msr(vcpu); + + break; case MSR_IA32_CR_PAT: if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data)) return 1; @@ -2941,7 +2832,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) svm->tsc_aux = data; break; case MSR_IA32_DEBUGCTLMSR: - if (!boot_cpu_has(X86_FEATURE_LBRV)) { + if (!lbrv) { vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n", __func__, data); break; @@ -3058,11 +2949,6 @@ static int invpcid_interception(struct kvm_vcpu *vcpu) type = svm->vmcb->control.exit_info_2; gva = svm->vmcb->control.exit_info_1; - if (type > 3) { - kvm_inject_gp(vcpu, 0); - return 1; - } - return kvm_handle_invpcid(vcpu, type, gva); } @@ -3301,11 +3187,13 @@ int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code) return svm_exit_handlers[exit_code](vcpu); } -static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2, +static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, + u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; + *reason = control->exit_code; *info1 = control->exit_info_1; *info2 = control->exit_info_2; *intr_info = control->exit_int_info; @@ -3322,7 +3210,7 @@ static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) struct kvm_run *kvm_run = vcpu->run; u32 exit_code = svm->vmcb->control.exit_code; - trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM); + trace_kvm_exit(vcpu, KVM_ISA_SVM); /* SEV-ES guests must use the CR write traps to track CR registers. */ if (!sev_es_guest(vcpu->kvm)) { @@ -3335,7 +3223,7 @@ static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) if (is_guest_mode(vcpu)) { int vmexit; - trace_kvm_nested_vmexit(exit_code, vcpu, KVM_ISA_SVM); + trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM); vmexit = nested_svm_exit_special(svm); @@ -3426,6 +3314,55 @@ static void svm_set_irq(struct kvm_vcpu *vcpu) SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR; } +void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode, + int trig_mode, int vector) +{ + /* + * vcpu->arch.apicv_active must be read after vcpu->mode. + * Pairs with smp_store_release in vcpu_enter_guest. + */ + bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE); + + if (!READ_ONCE(vcpu->arch.apicv_active)) { + /* Process the interrupt via inject_pending_event */ + kvm_make_request(KVM_REQ_EVENT, vcpu); + kvm_vcpu_kick(vcpu); + return; + } + + trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); + if (in_guest_mode) { + /* + * Signal the doorbell to tell hardware to inject the IRQ. If + * the vCPU exits the guest before the doorbell chimes, hardware + * will automatically process AVIC interrupts at the next VMRUN. + */ + avic_ring_doorbell(vcpu); + } else { + /* + * Wake the vCPU if it was blocking. KVM will then detect the + * pending IRQ when checking if the vCPU has a wake event. + */ + kvm_vcpu_wake_up(vcpu); + } +} + +static void svm_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, + int trig_mode, int vector) +{ + kvm_lapic_set_irr(vector, apic); + + /* + * Pairs with the smp_mb_*() after setting vcpu->guest_mode in + * vcpu_enter_guest() to ensure the write to the vIRR is ordered before + * the read of guest_mode. This guarantees that either VMRUN will see + * and process the new vIRR entry, or that svm_complete_interrupt_delivery + * will signal the doorbell if the CPU has already entered the guest. + */ + smp_mb__after_atomic(); + svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector); +} + static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { struct vcpu_svm *svm = to_svm(vcpu); @@ -3473,11 +3410,13 @@ static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) if (svm->nested.nested_run_pending) return -EBUSY; + if (svm_nmi_blocked(vcpu)) + return 0; + /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) return -EBUSY; - - return !svm_nmi_blocked(vcpu); + return 1; } static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) @@ -3529,9 +3468,13 @@ bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) { struct vcpu_svm *svm = to_svm(vcpu); + if (svm->nested.nested_run_pending) return -EBUSY; + if (svm_interrupt_blocked(vcpu)) + return 0; + /* * An IRQ must not be injected into L2 if it's supposed to VM-Exit, * e.g. if the IRQ arrived asynchronously after checking nested events. @@ -3539,7 +3482,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) return -EBUSY; - return !svm_interrupt_blocked(vcpu); + return 1; } static void svm_enable_irq_window(struct kvm_vcpu *vcpu) @@ -3731,6 +3674,11 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu) svm_complete_interrupts(vcpu); } +static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu) +{ + return 1; +} + static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) { if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && @@ -3745,7 +3693,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); unsigned long vmcb_pa = svm->current_vmcb->pa; - kvm_guest_enter_irqoff(); + guest_state_enter_irqoff(); if (sev_es_guest(vcpu->kvm)) { __svm_sev_es_vcpu_run(vmcb_pa); @@ -3765,7 +3713,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) vmload(__sme_page_pa(sd->save_area)); } - kvm_guest_exit_irqoff(); + guest_state_exit_irqoff(); } static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) @@ -3796,8 +3744,6 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) pre_svm_run(vcpu); - WARN_ON_ONCE(kvm_apicv_activated(vcpu->kvm) != kvm_vcpu_apicv_active(vcpu)); - sync_lapic_to_cr8(vcpu); if (unlikely(svm->asid != svm->vmcb->control.asid)) { @@ -3864,9 +3810,10 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; } + vcpu->arch.regs_dirty = 0; if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) - kvm_before_interrupt(vcpu); + kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); kvm_load_host_xsave_state(vcpu); stgi(); @@ -3898,8 +3845,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); - if (npt_enabled) - kvm_register_clear_available(vcpu, VCPU_EXREG_PDPTR); + vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET; /* * We need to handle MC intercepts here before the vcpu has a chance to @@ -3929,9 +3875,6 @@ static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, hv_track_root_tdp(vcpu, root_hpa); - /* Loading L2's CR3 is handled by enter_svm_guest_mode. */ - if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) - return; cr3 = vcpu->arch.cr3; } else if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu); @@ -4017,6 +3960,8 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) && guest_cpuid_has(vcpu, X86_FEATURE_NRIPS); + svm->tsc_scaling_enabled = tsc_scaling && guest_cpuid_has(vcpu, X86_FEATURE_TSCRATEMSR); + svm_recalc_instruction_intercepts(vcpu, svm); /* For sev guests, the memory encryption bit is not reserved in CR3. */ @@ -4043,33 +3988,7 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) kvm_request_apicv_update(vcpu->kvm, false, APICV_INHIBIT_REASON_NESTED); } - - if (guest_cpuid_is_intel(vcpu)) { - /* - * We must intercept SYSENTER_EIP and SYSENTER_ESP - * accesses because the processor only stores 32 bits. - * For the same reason we cannot use virtual VMLOAD/VMSAVE. - */ - svm_set_intercept(svm, INTERCEPT_VMLOAD); - svm_set_intercept(svm, INTERCEPT_VMSAVE); - svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; - - set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0); - set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0); - } else { - /* - * If hardware supports Virtual VMLOAD VMSAVE then enable it - * in VMCB and clear intercepts to avoid #VMEXIT. - */ - if (vls) { - svm_clr_intercept(svm, INTERCEPT_VMLOAD); - svm_clr_intercept(svm, INTERCEPT_VMSAVE); - svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; - } - /* No need to intercept these MSRs */ - set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1); - set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1); - } + init_vmcb_after_set_cpuid(vcpu); } static bool svm_has_wbinvd_exit(void) @@ -4174,7 +4093,7 @@ static int svm_check_intercept(struct kvm_vcpu *vcpu, info->intercept == x86_intercept_clts) break; - if (!(vmcb_is_intercept(&svm->nested.ctl, + if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))) break; @@ -4294,11 +4213,14 @@ static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) if (svm->nested.nested_run_pending) return -EBUSY; + if (svm_smi_blocked(vcpu)) + return 0; + /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) return -EBUSY; - return !svm_smi_blocked(vcpu); + return 1; } static int svm_enter_smm(struct kvm_vcpu *vcpu, char *smstate) @@ -4395,7 +4317,8 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) vmcb_mark_all_dirty(svm->vmcb01.ptr); vmcb12 = map.hva; - nested_load_control_from_vmcb12(svm, &vmcb12->control); + nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); + nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); ret = enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, false); if (ret) @@ -4423,15 +4346,22 @@ static void svm_enable_smi_window(struct kvm_vcpu *vcpu) } } -static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int insn_len) +static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, + void *insn, int insn_len) { bool smep, smap, is_user; unsigned long cr4; + u64 error_code; /* Emulation is always possible when KVM has access to all guest state. */ if (!sev_guest(vcpu->kvm)) return true; + /* #UD and #GP should never be intercepted for SEV guests. */ + WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD | + EMULTYPE_TRAP_UD_FORCED | + EMULTYPE_VMWARE_GP)); + /* * Emulation is impossible for SEV-ES guests as KVM doesn't have access * to guest register state. @@ -4439,48 +4369,78 @@ static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int i if (sev_es_guest(vcpu->kvm)) return false; + /* + * Emulation is possible if the instruction is already decoded, e.g. + * when completing I/O after returning from userspace. + */ + if (emul_type & EMULTYPE_NO_DECODE) + return true; + + /* + * Emulation is possible for SEV guests if and only if a prefilled + * buffer containing the bytes of the intercepted instruction is + * available. SEV guest memory is encrypted with a guest specific key + * and cannot be decrypted by KVM, i.e. KVM would read cyphertext and + * decode garbage. + * + * Inject #UD if KVM reached this point without an instruction buffer. + * In practice, this path should never be hit by a well-behaved guest, + * e.g. KVM doesn't intercept #UD or #GP for SEV guests, but this path + * is still theoretically reachable, e.g. via unaccelerated fault-like + * AVIC access, and needs to be handled by KVM to avoid putting the + * guest into an infinite loop. Injecting #UD is somewhat arbitrary, + * but its the least awful option given lack of insight into the guest. + */ + if (unlikely(!insn)) { + kvm_queue_exception(vcpu, UD_VECTOR); + return false; + } + + /* + * Emulate for SEV guests if the insn buffer is not empty. The buffer + * will be empty if the DecodeAssist microcode cannot fetch bytes for + * the faulting instruction because the code fetch itself faulted, e.g. + * the guest attempted to fetch from emulated MMIO or a guest page + * table used to translate CS:RIP resides in emulated MMIO. + */ + if (likely(insn_len)) + return true; + /* * Detect and workaround Errata 1096 Fam_17h_00_0Fh. * * Errata: - * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is - * possible that CPU microcode implementing DecodeAssist will fail - * to read bytes of instruction which caused #NPF. In this case, - * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly - * return 0 instead of the correct guest instruction bytes. + * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is + * possible that CPU microcode implementing DecodeAssist will fail to + * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly + * be '0'. This happens because microcode reads CS:RIP using a _data_ + * loap uop with CPL=0 privileges. If the load hits a SMAP #PF, ucode + * gives up and does not fill the instruction bytes buffer. * - * This happens because CPU microcode reading instruction bytes - * uses a special opcode which attempts to read data using CPL=0 - * privileges. The microcode reads CS:RIP and if it hits a SMAP - * fault, it gives up and returns no instruction bytes. + * As above, KVM reaches this point iff the VM is an SEV guest, the CPU + * supports DecodeAssist, a #NPF was raised, KVM's page fault handler + * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the + * GuestIntrBytes field of the VMCB. * - * Detection: - * We reach here in case CPU supports DecodeAssist, raised #NPF and - * returned 0 in GuestIntrBytes field of the VMCB. - * First, errata can only be triggered in case vCPU CR4.SMAP=1. - * Second, if vCPU CR4.SMEP=1, errata could only be triggered - * in case vCPU CPL==3 (Because otherwise guest would have triggered - * a SMEP fault instead of #NPF). - * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL. - * As most guests enable SMAP if they have also enabled SMEP, use above - * logic in order to attempt minimize false-positive of detecting errata - * while still preserving all cases semantic correctness. + * This does _not_ mean that the erratum has been encountered, as the + * DecodeAssist will also fail if the load for CS:RIP hits a legitimate + * #PF, e.g. if the guest attempt to execute from emulated MMIO and + * encountered a reserved/not-present #PF. * - * Workaround: - * To determine what instruction the guest was executing, the hypervisor - * will have to decode the instruction at the instruction pointer. + * To hit the erratum, the following conditions must be true: + * 1. CR4.SMAP=1 (obviously). + * 2. CR4.SMEP=0 || CPL=3. If SMEP=1 and CPL<3, the erratum cannot + * have been hit as the guest would have encountered a SMEP + * violation #PF, not a #NPF. + * 3. The #NPF is not due to a code fetch, in which case failure to + * retrieve the instruction bytes is legitimate (see abvoe). * - * In non SEV guest, hypervisor will be able to read the guest - * memory to decode the instruction pointer when insn_len is zero - * so we return true to indicate that decoding is possible. - * - * But in the SEV guest, the guest memory is encrypted with the - * guest specific key and hypervisor will not be able to decode the - * instruction pointer so we will not able to workaround it. Lets - * print the error and request to kill the guest. + * In addition, don't apply the erratum workaround if the #NPF occurred + * while translating guest page tables (see below). */ - if (likely(!insn || insn_len)) - return true; + error_code = to_svm(vcpu)->vmcb->control.exit_info_1; + if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK)) + goto resume_guest; cr4 = kvm_read_cr4(vcpu); smep = cr4 & X86_CR4_SMEP; @@ -4505,6 +4465,21 @@ static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int i kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); } +resume_guest: + /* + * If the erratum was not hit, simply resume the guest and let it fault + * again. While awful, e.g. the vCPU may get stuck in an infinite loop + * if the fault is at CPL=0, it's the lesser of all evils. Exiting to + * userspace will kill the guest, and letting the emulator read garbage + * will yield random behavior and potentially corrupt the guest. + * + * Simply resuming the guest is technically not a violation of the SEV + * architecture. AMD's APM states that all code fetches and page table + * accesses for SEV guest are encrypted, regardless of the C-Bit. The + * APM also states that encrypted accesses to MMIO are "ignored", but + * doesn't explicitly define "ignored", i.e. doing nothing and letting + * the guest spin is technically "ignoring" the access. + */ return false; } @@ -4552,6 +4527,8 @@ static int svm_vm_init(struct kvm *kvm) } static struct kvm_x86_ops svm_x86_ops __initdata = { + .name = "kvm_amd", + .hardware_unsetup = svm_hardware_teardown, .hardware_enable = svm_hardware_enable, .hardware_disable = svm_hardware_disable, @@ -4569,8 +4546,8 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .prepare_guest_switch = svm_prepare_guest_switch, .vcpu_load = svm_vcpu_load, .vcpu_put = svm_vcpu_put, - .vcpu_blocking = svm_vcpu_blocking, - .vcpu_unblocking = svm_vcpu_unblocking, + .vcpu_blocking = avic_vcpu_blocking, + .vcpu_unblocking = avic_vcpu_unblocking, .update_exception_bitmap = svm_update_exception_bitmap, .get_msr_feature = svm_get_msr_feature, @@ -4582,6 +4559,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .get_cpl = svm_get_cpl, .get_cs_db_l_bits = kvm_get_cs_db_l_bits, .set_cr0 = svm_set_cr0, + .post_set_cr3 = svm_post_set_cr3, .is_valid_cr4 = svm_is_valid_cr4, .set_cr4 = svm_set_cr4, .set_efer = svm_set_efer, @@ -4601,6 +4579,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .tlb_flush_gva = svm_flush_tlb_gva, .tlb_flush_guest = svm_flush_tlb, + .vcpu_pre_run = svm_vcpu_pre_run, .run = svm_vcpu_run, .handle_exit = handle_exit, .skip_emulated_instruction = skip_emulated_instruction, @@ -4654,7 +4633,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .pmu_ops = &amd_pmu_ops, .nested_ops = &svm_nested_ops, - .deliver_posted_interrupt = svm_deliver_avic_intr, + .deliver_interrupt = svm_deliver_interrupt, .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt, .update_pi_irte = svm_update_pi_irte, .setup_mce = svm_setup_mce, @@ -4669,6 +4648,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .mem_enc_unreg_region = svm_unregister_enc_region, .vm_copy_enc_context_from = svm_vm_copy_asid_from, + .vm_move_enc_context_from = svm_vm_migrate_from, .can_emulate_instruction = svm_can_emulate_instruction, @@ -4680,6 +4660,244 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, }; +/* + * The default MMIO mask is a single bit (excluding the present bit), + * which could conflict with the memory encryption bit. Check for + * memory encryption support and override the default MMIO mask if + * memory encryption is enabled. + */ +static __init void svm_adjust_mmio_mask(void) +{ + unsigned int enc_bit, mask_bit; + u64 msr, mask; + + /* If there is no memory encryption support, use existing mask */ + if (cpuid_eax(0x80000000) < 0x8000001f) + return; + + /* If memory encryption is not enabled, use existing mask */ + rdmsrl(MSR_AMD64_SYSCFG, msr); + if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) + return; + + enc_bit = cpuid_ebx(0x8000001f) & 0x3f; + mask_bit = boot_cpu_data.x86_phys_bits; + + /* Increment the mask bit if it is the same as the encryption bit */ + if (enc_bit == mask_bit) + mask_bit++; + + /* + * If the mask bit location is below 52, then some bits above the + * physical addressing limit will always be reserved, so use the + * rsvd_bits() function to generate the mask. This mask, along with + * the present bit, will be used to generate a page fault with + * PFER.RSV = 1. + * + * If the mask bit location is 52 (or above), then clear the mask. + */ + mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0; + + kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK); +} + +static __init void svm_set_cpu_caps(void) +{ + kvm_set_cpu_caps(); + + supported_xss = 0; + + /* CPUID 0x80000001 and 0x8000000A (SVM features) */ + if (nested) { + kvm_cpu_cap_set(X86_FEATURE_SVM); + kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN); + + if (nrips) + kvm_cpu_cap_set(X86_FEATURE_NRIPS); + + if (npt_enabled) + kvm_cpu_cap_set(X86_FEATURE_NPT); + + if (tsc_scaling) + kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR); + + /* Nested VM can receive #VMEXIT instead of triggering #GP */ + kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); + } + + /* CPUID 0x80000008 */ + if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || + boot_cpu_has(X86_FEATURE_AMD_SSBD)) + kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); + + /* AMD PMU PERFCTR_CORE CPUID */ + if (enable_pmu && boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) + kvm_cpu_cap_set(X86_FEATURE_PERFCTR_CORE); + + /* CPUID 0x8000001F (SME/SEV features) */ + sev_set_cpu_caps(); +} + +static __init int svm_hardware_setup(void) +{ + int cpu; + struct page *iopm_pages; + void *iopm_va; + int r; + unsigned int order = get_order(IOPM_SIZE); + + /* + * NX is required for shadow paging and for NPT if the NX huge pages + * mitigation is enabled. + */ + if (!boot_cpu_has(X86_FEATURE_NX)) { + pr_err_ratelimited("NX (Execute Disable) not supported\n"); + return -EOPNOTSUPP; + } + kvm_enable_efer_bits(EFER_NX); + + iopm_pages = alloc_pages(GFP_KERNEL, order); + + if (!iopm_pages) + return -ENOMEM; + + iopm_va = page_address(iopm_pages); + memset(iopm_va, 0xff, PAGE_SIZE * (1 << order)); + iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; + + init_msrpm_offsets(); + + supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); + + if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) + kvm_enable_efer_bits(EFER_FFXSR); + + if (tsc_scaling) { + if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { + tsc_scaling = false; + } else { + pr_info("TSC scaling supported\n"); + kvm_has_tsc_control = true; + kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX; + kvm_tsc_scaling_ratio_frac_bits = 32; + } + } + + tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX); + + /* Check for pause filtering support */ + if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { + pause_filter_count = 0; + pause_filter_thresh = 0; + } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) { + pause_filter_thresh = 0; + } + + if (nested) { + printk(KERN_INFO "kvm: Nested Virtualization enabled\n"); + kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE); + } + + /* + * KVM's MMU doesn't support using 2-level paging for itself, and thus + * NPT isn't supported if the host is using 2-level paging since host + * CR4 is unchanged on VMRUN. + */ + if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE)) + npt_enabled = false; + + if (!boot_cpu_has(X86_FEATURE_NPT)) + npt_enabled = false; + + /* Force VM NPT level equal to the host's paging level */ + kvm_configure_mmu(npt_enabled, get_npt_level(), + get_npt_level(), PG_LEVEL_1G); + pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); + + /* Note, SEV setup consumes npt_enabled. */ + sev_hardware_setup(); + + svm_hv_hardware_setup(); + + svm_adjust_mmio_mask(); + + for_each_possible_cpu(cpu) { + r = svm_cpu_init(cpu); + if (r) + goto err; + } + + if (nrips) { + if (!boot_cpu_has(X86_FEATURE_NRIPS)) + nrips = false; + } + + enable_apicv = avic = avic && npt_enabled && boot_cpu_has(X86_FEATURE_AVIC); + + if (enable_apicv) { + pr_info("AVIC enabled\n"); + + amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); + } else { + svm_x86_ops.vcpu_blocking = NULL; + svm_x86_ops.vcpu_unblocking = NULL; + } + + if (vls) { + if (!npt_enabled || + !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) || + !IS_ENABLED(CONFIG_X86_64)) { + vls = false; + } else { + pr_info("Virtual VMLOAD VMSAVE supported\n"); + } + } + + if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK)) + svm_gp_erratum_intercept = false; + + if (vgif) { + if (!boot_cpu_has(X86_FEATURE_VGIF)) + vgif = false; + else + pr_info("Virtual GIF supported\n"); + } + + if (lbrv) { + if (!boot_cpu_has(X86_FEATURE_LBRV)) + lbrv = false; + else + pr_info("LBR virtualization supported\n"); + } + + if (!enable_pmu) + pr_info("PMU virtualization is disabled\n"); + + svm_set_cpu_caps(); + + /* + * It seems that on AMD processors PTE's accessed bit is + * being set by the CPU hardware before the NPF vmexit. + * This is not expected behaviour and our tests fail because + * of it. + * A workaround here is to disable support for + * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled. + * In this case userspace can know if there is support using + * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle + * it + * If future AMD CPU models change the behaviour described above, + * this variable can be changed accordingly + */ + allow_smaller_maxphyaddr = !npt_enabled; + + return 0; + +err: + svm_hardware_teardown(); + return r; +} + + static struct kvm_x86_init_ops svm_init_ops __initdata = { .cpu_has_kvm_support = has_svm, .disabled_by_bios = is_disabled, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index ff0855c03c..fa98d68447 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -79,7 +79,9 @@ struct kvm_sev_info { struct list_head regions_list; /* List of registered regions */ u64 ap_jump_table; /* SEV-ES AP Jump Table address */ struct kvm *enc_context_owner; /* Owner of copied encryption context */ + unsigned long num_mirrored_vms; /* Number of VMs sharing this ASID */ struct misc_cg *misc_cg; /* For misc cgroup accounting */ + atomic_t migration_in_progress; }; struct kvm_svm { @@ -103,6 +105,40 @@ struct kvm_vmcb_info { uint64_t asid_generation; }; +struct vmcb_save_area_cached { + u64 efer; + u64 cr4; + u64 cr3; + u64 cr0; + u64 dr7; + u64 dr6; +}; + +struct vmcb_ctrl_area_cached { + u32 intercepts[MAX_INTERCEPT]; + u16 pause_filter_thresh; + u16 pause_filter_count; + u64 iopm_base_pa; + u64 msrpm_base_pa; + u64 tsc_offset; + u32 asid; + u8 tlb_ctl; + u32 int_ctl; + u32 int_vector; + u32 int_state; + u32 exit_code; + u32 exit_code_hi; + u64 exit_info_1; + u64 exit_info_2; + u32 exit_int_info; + u32 exit_int_info_err; + u64 nested_ctl; + u32 event_inj; + u32 event_inj_err; + u64 nested_cr3; + u64 virt_ext; +}; + struct svm_nested_state { struct kvm_vmcb_info vmcb02; u64 hsave_msr; @@ -118,11 +154,31 @@ struct svm_nested_state { bool nested_run_pending; /* cache for control fields of the guest */ - struct vmcb_control_area ctl; + struct vmcb_ctrl_area_cached ctl; + + /* + * Note: this struct is not kept up-to-date while L2 runs; it is only + * valid within nested_svm_vmrun. + */ + struct vmcb_save_area_cached save; bool initialized; }; +struct vcpu_sev_es_state { + /* SEV-ES support */ + struct vmcb_save_area *vmsa; + struct ghcb *ghcb; + struct kvm_host_map ghcb_map; + bool received_first_sipi; + + /* SEV-ES scratch area support */ + void *ghcb_sa; + u32 ghcb_sa_len; + bool ghcb_sa_sync; + bool ghcb_sa_free; +}; + struct vcpu_svm { struct kvm_vcpu vcpu; /* vmcb always points at current_vmcb->ptr, it's purely a shorthand. */ @@ -140,6 +196,8 @@ struct vcpu_svm { u64 next_rip; u64 spec_ctrl; + + u64 tsc_ratio_msr; /* * Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be * translated into the appropriate L2_CFG bits on the host to @@ -160,13 +218,13 @@ struct vcpu_svm { unsigned long int3_rip; /* cached guest cpuid flags for faster access */ - bool nrips_enabled : 1; + bool nrips_enabled : 1; + bool tsc_scaling_enabled : 1; u32 ldr_reg; u32 dfr_reg; struct page *avic_backing_page; u64 *avic_physical_id_cache; - bool avic_is_running; /* * Per-vcpu list of struct amd_svm_iommu_ir: @@ -183,17 +241,7 @@ struct vcpu_svm { DECLARE_BITMAP(write, MAX_DIRECT_ACCESS_MSRS); } shadow_msr_intercept; - /* SEV-ES support */ - struct vmcb_save_area *vmsa; - struct ghcb *ghcb; - struct kvm_host_map ghcb_map; - bool received_first_sipi; - - /* SEV-ES scratch area support */ - void *ghcb_sa; - u32 ghcb_sa_len; - bool ghcb_sa_sync; - bool ghcb_sa_free; + struct vcpu_sev_es_state sev_es; bool guest_state_loaded; }; @@ -218,12 +266,12 @@ DECLARE_PER_CPU(struct svm_cpu_data *, svm_data); void recalc_intercepts(struct vcpu_svm *svm); -static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm) +static __always_inline struct kvm_svm *to_kvm_svm(struct kvm *kvm) { return container_of(kvm, struct kvm_svm, kvm); } -static inline bool sev_guest(struct kvm *kvm) +static __always_inline bool sev_guest(struct kvm *kvm) { #ifdef CONFIG_KVM_AMD_SEV struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; @@ -234,12 +282,12 @@ static inline bool sev_guest(struct kvm *kvm) #endif } -static inline bool sev_es_guest(struct kvm *kvm) +static __always_inline bool sev_es_guest(struct kvm *kvm) { #ifdef CONFIG_KVM_AMD_SEV struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; - return sev_guest(kvm) && sev->es_active; + return sev->es_active && !WARN_ON_ONCE(!sev->active); #else return false; #endif @@ -256,11 +304,6 @@ static inline void vmcb_mark_all_clean(struct vmcb *vmcb) & ~VMCB_ALWAYS_DIRTY_MASK; } -static inline bool vmcb_is_clean(struct vmcb *vmcb, int bit) -{ - return (vmcb->control.clean & (1 << bit)); -} - static inline void vmcb_mark_dirty(struct vmcb *vmcb, int bit) { vmcb->control.clean &= ~(1 << bit); @@ -271,11 +314,21 @@ static inline bool vmcb_is_dirty(struct vmcb *vmcb, int bit) return !test_bit(bit, (unsigned long *)&vmcb->control.clean); } -static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) +static __always_inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) { return container_of(vcpu, struct vcpu_svm, vcpu); } +/* + * Only the PDPTRs are loaded on demand into the shadow MMU. All other + * fields are synchronized in handle_exit, because accessing the VMCB is cheap. + * + * CR3 might be out of date in the VMCB but it is not marked dirty; instead, + * KVM_REQ_LOAD_MMU_PGD is always requested when the cached vcpu->arch.cr3 + * is changed. svm_load_mmu_pgd() then syncs the new CR3 value into the VMCB. + */ +#define SVM_REGS_LAZY_LOAD_SET (1 << VCPU_EXREG_PDPTR) + static inline void vmcb_set_intercept(struct vmcb_control_area *control, u32 bit) { WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT); @@ -294,6 +347,12 @@ static inline bool vmcb_is_intercept(struct vmcb_control_area *control, u32 bit) return test_bit(bit, (unsigned long *)&control->intercepts); } +static inline bool vmcb12_is_intercept(struct vmcb_ctrl_area_cached *control, u32 bit) +{ + WARN_ON_ONCE(bit >= 32 * MAX_INTERCEPT); + return test_bit(bit, (unsigned long *)&control->intercepts); +} + static inline void set_dr_intercepts(struct vcpu_svm *svm) { struct vmcb *vmcb = svm->vmcb01.ptr; @@ -430,6 +489,8 @@ void svm_set_gif(struct vcpu_svm *svm, bool value); int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code); void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr, int read, int write); +void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode, + int trig_mode, int vec); /* nested.c */ @@ -446,17 +507,17 @@ static inline bool nested_svm_virtualize_tpr(struct kvm_vcpu *vcpu) static inline bool nested_exit_on_smi(struct vcpu_svm *svm) { - return vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_SMI); + return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SMI); } static inline bool nested_exit_on_intr(struct vcpu_svm *svm) { - return vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_INTR); + return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INTR); } static inline bool nested_exit_on_nmi(struct vcpu_svm *svm) { - return vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_NMI); + return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_NMI); } int enter_svm_guest_mode(struct kvm_vcpu *vcpu, @@ -483,8 +544,12 @@ int nested_svm_check_permissions(struct kvm_vcpu *vcpu); int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, bool has_error_code, u32 error_code); int nested_svm_exit_special(struct vcpu_svm *svm); -void nested_load_control_from_vmcb12(struct vcpu_svm *svm, - struct vmcb_control_area *control); +void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu); +void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier); +void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, + struct vmcb_control_area *control); +void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, + struct vmcb_save_area *save); void nested_sync_control_from_vmcb02(struct vcpu_svm *svm); void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm); void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb); @@ -493,28 +558,6 @@ extern struct kvm_x86_nested_ops svm_nested_ops; /* avic.c */ -#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF) -#define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31 -#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31) - -#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK (0xFFULL) -#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12) -#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62) -#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63) - -#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL - -static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - u64 *entry = svm->avic_physical_id_cache; - - if (!entry) - return false; - - return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); -} - int avic_ga_log_notifier(u32 ga_tag); void avic_vm_destroy(struct kvm *kvm); int avic_vm_init(struct kvm *kvm); @@ -531,12 +574,12 @@ bool svm_check_apicv_inhibit_reasons(ulong bit); void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr); void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr); -int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec); bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu); int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, bool set); -void svm_vcpu_blocking(struct kvm_vcpu *vcpu); -void svm_vcpu_unblocking(struct kvm_vcpu *vcpu); +void avic_vcpu_blocking(struct kvm_vcpu *vcpu); +void avic_vcpu_unblocking(struct kvm_vcpu *vcpu); +void avic_ring_doorbell(struct kvm_vcpu *vcpu); /* sev.c */ @@ -553,6 +596,7 @@ int svm_register_enc_region(struct kvm *kvm, int svm_unregister_enc_region(struct kvm *kvm, struct kvm_enc_region *range); int svm_vm_copy_asid_from(struct kvm *kvm, unsigned int source_fd); +int svm_vm_migrate_from(struct kvm *kvm, unsigned int source_fd); void pre_sev_run(struct vcpu_svm *svm, int cpu); void __init sev_set_cpu_caps(void); void __init sev_hardware_setup(void); @@ -562,7 +606,7 @@ void sev_free_vcpu(struct kvm_vcpu *vcpu); int sev_handle_vmgexit(struct kvm_vcpu *vcpu); int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in); void sev_es_init_vmcb(struct vcpu_svm *svm); -void sev_es_create_vcpu(struct vcpu_svm *svm); +void sev_es_vcpu_reset(struct vcpu_svm *svm); void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector); void sev_es_prepare_guest_switch(struct vcpu_svm *svm, unsigned int cpu); void sev_es_unmap_ghcb(struct vcpu_svm *svm); diff --git a/arch/x86/kvm/svm/svm_onhyperv.h b/arch/x86/kvm/svm/svm_onhyperv.h index c53b8bf8d0..489ca56212 100644 --- a/arch/x86/kvm/svm/svm_onhyperv.h +++ b/arch/x86/kvm/svm/svm_onhyperv.h @@ -46,6 +46,9 @@ static inline void svm_hv_init_vmcb(struct vmcb *vmcb) if (npt_enabled && ms_hyperv.nested_features & HV_X64_NESTED_ENLIGHTENED_TLB) hve->hv_enlightenments_control.enlightened_npt_tlb = 1; + + if (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP) + hve->hv_enlightenments_control.msr_bitmap = 1; } static inline void svm_hv_hardware_setup(void) @@ -83,14 +86,7 @@ static inline void svm_hv_vmcb_dirty_nested_enlightenments( struct hv_enlightenments *hve = (struct hv_enlightenments *)vmcb->control.reserved_sw; - /* - * vmcb can be NULL if called during early vcpu init. - * And its okay not to mark vmcb dirty during vcpu init - * as we mark it dirty unconditionally towards end of vcpu - * init phase. - */ - if (vmcb_is_clean(vmcb, VMCB_HV_NESTED_ENLIGHTENMENTS) && - hve->hv_enlightenments_control.msr_bitmap) + if (hve->hv_enlightenments_control.msr_bitmap) vmcb_mark_dirty(vmcb, VMCB_HV_NESTED_ENLIGHTENMENTS); } diff --git a/arch/x86/kvm/svm/svm_ops.h b/arch/x86/kvm/svm/svm_ops.h index 22e2b019de..9430d6437c 100644 --- a/arch/x86/kvm/svm/svm_ops.h +++ b/arch/x86/kvm/svm/svm_ops.h @@ -56,12 +56,12 @@ static inline void invlpga(unsigned long addr, u32 asid) * VMSAVE, VMLOAD, etc... is still controlled by the effective address size, * hence 'unsigned long' instead of 'hpa_t'. */ -static inline void vmsave(unsigned long pa) +static __always_inline void vmsave(unsigned long pa) { svm_asm1(vmsave, "a" (pa), "memory"); } -static inline void vmload(unsigned long pa) +static __always_inline void vmload(unsigned long pa) { svm_asm1(vmload, "a" (pa), "memory"); } diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 03ebe36833..92e6f6702f 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -288,8 +288,8 @@ TRACE_EVENT(kvm_apic, #define TRACE_EVENT_KVM_EXIT(name) \ TRACE_EVENT(name, \ - TP_PROTO(unsigned int exit_reason, struct kvm_vcpu *vcpu, u32 isa), \ - TP_ARGS(exit_reason, vcpu, isa), \ + TP_PROTO(struct kvm_vcpu *vcpu, u32 isa), \ + TP_ARGS(vcpu, isa), \ \ TP_STRUCT__entry( \ __field( unsigned int, exit_reason ) \ @@ -303,11 +303,12 @@ TRACE_EVENT(name, \ ), \ \ TP_fast_assign( \ - __entry->exit_reason = exit_reason; \ __entry->guest_rip = kvm_rip_read(vcpu); \ __entry->isa = isa; \ __entry->vcpu_id = vcpu->vcpu_id; \ - static_call(kvm_x86_get_exit_info)(vcpu, &__entry->info1, \ + static_call(kvm_x86_get_exit_info)(vcpu, \ + &__entry->exit_reason, \ + &__entry->info1, \ &__entry->info2, \ &__entry->intr_info, \ &__entry->error_code); \ @@ -1355,6 +1356,30 @@ TRACE_EVENT(kvm_apicv_update_request, __entry->bit) ); +TRACE_EVENT(kvm_apicv_accept_irq, + TP_PROTO(__u32 apicid, __u16 dm, __u16 tm, __u8 vec), + TP_ARGS(apicid, dm, tm, vec), + + TP_STRUCT__entry( + __field( __u32, apicid ) + __field( __u16, dm ) + __field( __u16, tm ) + __field( __u8, vec ) + ), + + TP_fast_assign( + __entry->apicid = apicid; + __entry->dm = dm; + __entry->tm = tm; + __entry->vec = vec; + ), + + TP_printk("apicid %x vec %u (%s|%s)", + __entry->apicid, __entry->vec, + __print_symbolic((__entry->dm >> 8 & 0x7), kvm_deliver_mode), + __entry->tm ? "level" : "edge") +); + /* * Tracepoint for AMD AVIC */ diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index 4705ad55ab..3f430e2183 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -5,6 +5,7 @@ #include #include "lapic.h" +#include "x86.h" extern bool __read_mostly enable_vpid; extern bool __read_mostly flexpriority_enabled; @@ -53,7 +54,6 @@ struct nested_vmx_msrs { struct vmcs_config { int size; - int order; u32 basic_cap; u32 revision_id; u32 pin_based_exec_ctrl; @@ -312,6 +312,15 @@ static inline bool cpu_has_vmx_ept_1g_page(void) return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT; } +static inline int ept_caps_to_lpage_level(u32 ept_caps) +{ + if (ept_caps & VMX_EPT_1GB_PAGE_BIT) + return PG_LEVEL_1G; + if (ept_caps & VMX_EPT_2MB_PAGE_BIT) + return PG_LEVEL_2M; + return PG_LEVEL_4K; +} + static inline bool cpu_has_vmx_ept_ad_bits(void) { return vmx_capability.ept & VMX_EPT_AD_BIT; @@ -380,6 +389,9 @@ static inline u64 vmx_get_perf_capabilities(void) { u64 perf_cap = 0; + if (!enable_pmu) + return perf_cap; + if (boot_cpu_has(X86_FEATURE_PDCM)) rdmsrl(MSR_IA32_PERF_CAPABILITIES, perf_cap); diff --git a/arch/x86/kvm/vmx/evmcs.c b/arch/x86/kvm/vmx/evmcs.c index a7ed30d564..87e3dc10ed 100644 --- a/arch/x86/kvm/vmx/evmcs.c +++ b/arch/x86/kvm/vmx/evmcs.c @@ -12,8 +12,6 @@ DEFINE_STATIC_KEY_FALSE(enable_evmcs); -#if IS_ENABLED(CONFIG_HYPERV) - #define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x) #define EVMCS1_FIELD(number, name, clean_field)[ROL16(number, 6)] = \ {EVMCS1_OFFSET(name), clean_field} @@ -296,6 +294,7 @@ const struct evmcs_field vmcs_field_to_evmcs_1[] = { }; const unsigned int nr_evmcs_1_fields = ARRAY_SIZE(vmcs_field_to_evmcs_1); +#if IS_ENABLED(CONFIG_HYPERV) __init void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) { vmcs_conf->pin_based_exec_ctrl &= ~EVMCS1_UNSUPPORTED_PINCTRL; diff --git a/arch/x86/kvm/vmx/evmcs.h b/arch/x86/kvm/vmx/evmcs.h index b43976e4b9..8d70f9aea9 100644 --- a/arch/x86/kvm/vmx/evmcs.h +++ b/arch/x86/kvm/vmx/evmcs.h @@ -65,8 +65,6 @@ DECLARE_STATIC_KEY_FALSE(enable_evmcs); #define EVMCS1_UNSUPPORTED_VMENTRY_CTRL (VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) #define EVMCS1_UNSUPPORTED_VMFUNC (VMX_VMFUNC_EPTP_SWITCHING) -#if IS_ENABLED(CONFIG_HYPERV) - struct evmcs_field { u16 offset; u16 clean_field; @@ -75,27 +73,57 @@ struct evmcs_field { extern const struct evmcs_field vmcs_field_to_evmcs_1[]; extern const unsigned int nr_evmcs_1_fields; -static __always_inline int get_evmcs_offset(unsigned long field, - u16 *clean_field) +static __always_inline int evmcs_field_offset(unsigned long field, + u16 *clean_field) { unsigned int index = ROL16(field, 6); const struct evmcs_field *evmcs_field; - if (unlikely(index >= nr_evmcs_1_fields)) { - WARN_ONCE(1, "KVM: accessing unsupported EVMCS field %lx\n", - field); + if (unlikely(index >= nr_evmcs_1_fields)) return -ENOENT; - } evmcs_field = &vmcs_field_to_evmcs_1[index]; + /* + * Use offset=0 to detect holes in eVMCS. This offset belongs to + * 'revision_id' but this field has no encoding and is supposed to + * be accessed directly. + */ + if (unlikely(!evmcs_field->offset)) + return -ENOENT; + if (clean_field) *clean_field = evmcs_field->clean_field; return evmcs_field->offset; } -static inline void evmcs_write64(unsigned long field, u64 value) +static inline u64 evmcs_read_any(struct hv_enlightened_vmcs *evmcs, + unsigned long field, u16 offset) +{ + /* + * vmcs12_read_any() doesn't care whether the supplied structure + * is 'struct vmcs12' or 'struct hv_enlightened_vmcs' as it takes + * the exact offset of the required field, use it for convenience + * here. + */ + return vmcs12_read_any((void *)evmcs, field, offset); +} + +#if IS_ENABLED(CONFIG_HYPERV) + +static __always_inline int get_evmcs_offset(unsigned long field, + u16 *clean_field) +{ + int offset = evmcs_field_offset(field, clean_field); + + WARN_ONCE(offset < 0, "KVM: accessing unsupported EVMCS field %lx\n", + field); + + return offset; +} + +static __always_inline void evmcs_write64(unsigned long field, u64 value) { u16 clean_field; int offset = get_evmcs_offset(field, &clean_field); @@ -185,7 +213,7 @@ static inline void evmcs_load(u64 phys_addr) __init void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf); #else /* !IS_ENABLED(CONFIG_HYPERV) */ -static inline void evmcs_write64(unsigned long field, u64 value) {} +static __always_inline void evmcs_write64(unsigned long field, u64 value) {} static inline void evmcs_write32(unsigned long field, u32 value) {} static inline void evmcs_write16(unsigned long field, u16 value) {} static inline u64 evmcs_read64(unsigned long field) { return 0; } diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index a0193b11c3..dc822a1d40 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -7,6 +7,7 @@ #include #include "cpuid.h" +#include "evmcs.h" #include "hyperv.h" #include "mmu.h" #include "nested.h" @@ -191,7 +192,7 @@ static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) * failValid writes the error number to the current VMCS, which * can't be done if there isn't a current VMCS. */ - if (vmx->nested.current_vmptr == -1ull && + if (vmx->nested.current_vmptr == INVALID_GPA && !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) return nested_vmx_failInvalid(vcpu); @@ -218,7 +219,7 @@ static inline u64 vmx_control_msr(u32 low, u32 high) static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) { secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); - vmcs_write64(VMCS_LINK_POINTER, -1ull); + vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); vmx->nested.need_vmcs12_to_shadow_sync = false; } @@ -269,7 +270,13 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) vmx_sync_vmcs_host_state(vmx, prev); put_cpu(); - vmx_register_cache_reset(vcpu); + vcpu->arch.regs_avail = ~VMX_REGS_LAZY_LOAD_SET; + + /* + * All lazily updated registers will be reloaded from VMCS12 on both + * vmentry and vmexit. + */ + vcpu->arch.regs_dirty = 0; } /* @@ -290,9 +297,10 @@ static void free_nested(struct kvm_vcpu *vcpu) vmx->nested.vmxon = false; vmx->nested.smm.vmxon = false; + vmx->nested.vmxon_ptr = INVALID_GPA; free_vpid(vmx->nested.vpid02); vmx->nested.posted_intr_nv = -1; - vmx->nested.current_vmptr = -1ull; + vmx->nested.current_vmptr = INVALID_GPA; if (enable_shadow_vmcs) { vmx_disable_shadow_vmcs(vmx); vmcs_clear(vmx->vmcs01.shadow_vmcs); @@ -390,9 +398,11 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, static void nested_ept_new_eptp(struct kvm_vcpu *vcpu) { - kvm_init_shadow_ept_mmu(vcpu, - to_vmx(vcpu)->nested.msrs.ept_caps & - VMX_EPT_EXECUTE_ONLY_BIT, + struct vcpu_vmx *vmx = to_vmx(vcpu); + bool execonly = vmx->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT; + int ept_lpage_level = ept_caps_to_lpage_level(vmx->nested.msrs.ept_caps); + + kvm_init_shadow_ept_mmu(vcpu, execonly, ept_lpage_level, nested_ept_ad_enabled(vcpu), nested_ept_get_eptp(vcpu)); } @@ -524,44 +534,19 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, } /* - * If a msr is allowed by L0, we should check whether it is allowed by L1. - * The corresponding bit will be cleared unless both of L0 and L1 allow it. + * For x2APIC MSRs, ignore the vmcs01 bitmap. L1 can enable x2APIC without L1 + * itself utilizing x2APIC. All MSRs were previously set to be intercepted, + * only the "disable intercept" case needs to be handled. */ -static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1, - unsigned long *msr_bitmap_nested, - u32 msr, int type) +static void nested_vmx_disable_intercept_for_x2apic_msr(unsigned long *msr_bitmap_l1, + unsigned long *msr_bitmap_l0, + u32 msr, int type) { - int f = sizeof(unsigned long); + if (type & MSR_TYPE_R && !vmx_test_msr_bitmap_read(msr_bitmap_l1, msr)) + vmx_clear_msr_bitmap_read(msr_bitmap_l0, msr); - /* - * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals - * have the write-low and read-high bitmap offsets the wrong way round. - * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. - */ - if (msr <= 0x1fff) { - if (type & MSR_TYPE_R && - !test_bit(msr, msr_bitmap_l1 + 0x000 / f)) - /* read-low */ - __clear_bit(msr, msr_bitmap_nested + 0x000 / f); - - if (type & MSR_TYPE_W && - !test_bit(msr, msr_bitmap_l1 + 0x800 / f)) - /* write-low */ - __clear_bit(msr, msr_bitmap_nested + 0x800 / f); - - } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { - msr &= 0x1fff; - if (type & MSR_TYPE_R && - !test_bit(msr, msr_bitmap_l1 + 0x400 / f)) - /* read-high */ - __clear_bit(msr, msr_bitmap_nested + 0x400 / f); - - if (type & MSR_TYPE_W && - !test_bit(msr, msr_bitmap_l1 + 0xc00 / f)) - /* write-high */ - __clear_bit(msr, msr_bitmap_nested + 0xc00 / f); - - } + if (type & MSR_TYPE_W && !vmx_test_msr_bitmap_write(msr_bitmap_l1, msr)) + vmx_clear_msr_bitmap_write(msr_bitmap_l0, msr); } static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) @@ -615,6 +600,7 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, int msr; unsigned long *msr_bitmap_l1; unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; + struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; struct kvm_host_map *map = &vmx->nested.msr_bitmap_map; /* Nothing to do if the MSR bitmap is not in use. */ @@ -622,6 +608,19 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) return false; + /* + * MSR bitmap update can be skipped when: + * - MSR bitmap for L1 hasn't changed. + * - Nested hypervisor (L1) is attempting to launch the same L2 as + * before. + * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature + * and tells KVM (L0) there were no changes in MSR bitmap for L2. + */ + if (!vmx->nested.force_msr_bitmap_recalc && evmcs && + evmcs->hv_enlightenments_control.msr_bitmap && + evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP) + return true; + if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map)) return false; @@ -630,7 +629,7 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, /* * To keep the control flow simple, pay eight 8-byte writes (sixteen * 4-byte writes on 32-bit systems) up front to enable intercepts for - * the x2APIC MSR range and selectively disable them below. + * the x2APIC MSR range and selectively toggle those relevant to L2. */ enable_x2apic_msr_intercepts(msr_bitmap_l0); @@ -649,17 +648,17 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, } } - nested_vmx_disable_intercept_for_msr( + nested_vmx_disable_intercept_for_x2apic_msr( msr_bitmap_l1, msr_bitmap_l0, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_R | MSR_TYPE_W); if (nested_cpu_has_vid(vmcs12)) { - nested_vmx_disable_intercept_for_msr( + nested_vmx_disable_intercept_for_x2apic_msr( msr_bitmap_l1, msr_bitmap_l0, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); - nested_vmx_disable_intercept_for_msr( + nested_vmx_disable_intercept_for_x2apic_msr( msr_bitmap_l1, msr_bitmap_l0, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); @@ -688,39 +687,47 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, kvm_vcpu_unmap(vcpu, &vmx->nested.msr_bitmap_map, false); + vmx->nested.force_msr_bitmap_recalc = false; + return true; } static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { - struct kvm_host_map map; - struct vmcs12 *shadow; + struct vcpu_vmx *vmx = to_vmx(vcpu); + struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; if (!nested_cpu_has_shadow_vmcs(vmcs12) || - vmcs12->vmcs_link_pointer == -1ull) + vmcs12->vmcs_link_pointer == INVALID_GPA) return; - shadow = get_shadow_vmcs12(vcpu); - - if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map)) + if (ghc->gpa != vmcs12->vmcs_link_pointer && + kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, + vmcs12->vmcs_link_pointer, VMCS12_SIZE)) return; - memcpy(shadow, map.hva, VMCS12_SIZE); - kvm_vcpu_unmap(vcpu, &map, false); + kvm_read_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), + VMCS12_SIZE); } static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { struct vcpu_vmx *vmx = to_vmx(vcpu); + struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; if (!nested_cpu_has_shadow_vmcs(vmcs12) || - vmcs12->vmcs_link_pointer == -1ull) + vmcs12->vmcs_link_pointer == INVALID_GPA) return; - kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer, - get_shadow_vmcs12(vcpu), VMCS12_SIZE); + if (ghc->gpa != vmcs12->vmcs_link_pointer && + kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, + vmcs12->vmcs_link_pointer, VMCS12_SIZE)) + return; + + kvm_write_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu), + VMCS12_SIZE); } /* @@ -1113,7 +1120,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, * must not be dereferenced. */ if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) && - CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) { + CC(!load_pdptrs(vcpu, cr3))) { *entry_failure_code = ENTRY_FAIL_PDPTE; return -EINVAL; } @@ -1122,7 +1129,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, kvm_mmu_new_pgd(vcpu, cr3); vcpu->arch.cr3 = cr3; - kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); + kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ kvm_init_mmu(vcpu); @@ -1980,7 +1987,7 @@ static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( } if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { - vmx->nested.current_vmptr = -1ull; + vmx->nested.current_vmptr = INVALID_GPA; nested_release_evmcs(vcpu); @@ -2039,10 +2046,13 @@ static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( * Clean fields data can't be used on VMLAUNCH and when we switch * between different L2 guests as KVM keeps a single VMCS12 per L1. */ - if (from_launch || evmcs_gpa_changed) + if (from_launch || evmcs_gpa_changed) { vmx->nested.hv_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; + vmx->nested.force_msr_bitmap_recalc = true; + } + return EVMPTRLD_SUCCEEDED; } @@ -2164,7 +2174,7 @@ static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) } if (cpu_has_vmx_encls_vmexit()) - vmcs_write64(ENCLS_EXITING_BITMAP, -1ull); + vmcs_write64(ENCLS_EXITING_BITMAP, INVALID_GPA); /* * Set the MSR load/store lists to match L0's settings. Only the @@ -2183,7 +2193,7 @@ static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, { prepare_vmcs02_constant_state(vmx); - vmcs_write64(VMCS_LINK_POINTER, -1ull); + vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); if (enable_vpid) { if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) @@ -2942,27 +2952,31 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { - int r = 0; - struct vmcs12 *shadow; - struct kvm_host_map map; + struct vcpu_vmx *vmx = to_vmx(vcpu); + struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; + struct vmcs_hdr hdr; - if (vmcs12->vmcs_link_pointer == -1ull) + if (vmcs12->vmcs_link_pointer == INVALID_GPA) return 0; if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))) return -EINVAL; - if (CC(kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map))) + if (ghc->gpa != vmcs12->vmcs_link_pointer && + CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, + vmcs12->vmcs_link_pointer, VMCS12_SIZE))) + return -EINVAL; + + if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, + offsetof(struct vmcs12, hdr), + sizeof(hdr)))) return -EINVAL; - shadow = map.hva; + if (CC(hdr.revision_id != VMCS12_REVISION) || + CC(hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) + return -EINVAL; - if (CC(shadow->hdr.revision_id != VMCS12_REVISION) || - CC(shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) - r = -EINVAL; - - kvm_vcpu_unmap(vcpu, &map, false); - return r; + return 0; } /* @@ -3159,7 +3173,7 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) * the guest CR3 might be restored prior to setting the nested * state which can lead to a load of wrong PDPTRs. */ - if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3))) + if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) return false; } @@ -3213,7 +3227,7 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to * force VM-Entry to fail. */ - vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull); + vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, INVALID_GPA); } } @@ -3518,12 +3532,15 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) if (evmptrld_status == EVMPTRLD_ERROR) { kvm_queue_exception(vcpu, UD_VECTOR); return 1; - } else if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) { - return nested_vmx_failInvalid(vcpu); } + kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS); + + if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) + return nested_vmx_failInvalid(vcpu); + if (CC(!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) && - vmx->nested.current_vmptr == -1ull)) + vmx->nested.current_vmptr == INVALID_GPA)) return nested_vmx_failInvalid(vcpu); vmcs12 = get_vmcs12(vcpu); @@ -3617,7 +3634,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) !(nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING) && (vmcs12->guest_rflags & X86_EFLAGS_IF))) { vmx->nested.nested_run_pending = 0; - return kvm_vcpu_halt(vcpu); + return kvm_emulate_halt_noskip(vcpu); } break; case GUEST_ACTIVITY_WAIT_SIPI: @@ -4840,18 +4857,20 @@ static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; /* - * We should allocate a shadow vmcs for vmcs01 only when L1 - * executes VMXON and free it when L1 executes VMXOFF. - * As it is invalid to execute VMXON twice, we shouldn't reach - * here when vmcs01 already have an allocated shadow vmcs. + * KVM allocates a shadow VMCS only when L1 executes VMXON and frees it + * when L1 executes VMXOFF or the vCPU is forced out of nested + * operation. VMXON faults if the CPU is already post-VMXON, so it + * should be impossible to already have an allocated shadow VMCS. KVM + * doesn't support virtualization of VMCS shadowing, so vmcs01 should + * always be the loaded VMCS. */ - WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs); + if (WARN_ON(loaded_vmcs != &vmx->vmcs01 || loaded_vmcs->shadow_vmcs)) + return loaded_vmcs->shadow_vmcs; + + loaded_vmcs->shadow_vmcs = alloc_vmcs(true); + if (loaded_vmcs->shadow_vmcs) + vmcs_clear(loaded_vmcs->shadow_vmcs); - if (!loaded_vmcs->shadow_vmcs) { - loaded_vmcs->shadow_vmcs = alloc_vmcs(true); - if (loaded_vmcs->shadow_vmcs) - vmcs_clear(loaded_vmcs->shadow_vmcs); - } return loaded_vmcs->shadow_vmcs; } @@ -4868,6 +4887,7 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu) if (!vmx->nested.cached_vmcs12) goto out_cached_vmcs12; + vmx->nested.shadow_vmcs12_cache.gpa = INVALID_GPA; vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); if (!vmx->nested.cached_shadow_vmcs12) goto out_cached_shadow_vmcs12; @@ -4973,7 +4993,7 @@ static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (vmx->nested.current_vmptr == -1ull) + if (vmx->nested.current_vmptr == INVALID_GPA) return; copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); @@ -4993,7 +5013,7 @@ static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); - vmx->nested.current_vmptr = -1ull; + vmx->nested.current_vmptr = INVALID_GPA; } /* Emulate the VMXOFF instruction */ @@ -5087,27 +5107,49 @@ static int handle_vmread(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; - /* - * In VMX non-root operation, when the VMCS-link pointer is -1ull, - * any VMREAD sets the ALU flags for VMfailInvalid. - */ - if (vmx->nested.current_vmptr == -1ull || - (is_guest_mode(vcpu) && - get_vmcs12(vcpu)->vmcs_link_pointer == -1ull)) - return nested_vmx_failInvalid(vcpu); - /* Decode instruction info and find the field to read */ field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); - offset = vmcs_field_to_offset(field); - if (offset < 0) - return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); + if (!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { + /* + * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, + * any VMREAD sets the ALU flags for VMfailInvalid. + */ + if (vmx->nested.current_vmptr == INVALID_GPA || + (is_guest_mode(vcpu) && + get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) + return nested_vmx_failInvalid(vcpu); - if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field)) - copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); + offset = get_vmcs12_field_offset(field); + if (offset < 0) + return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); - /* Read the field, zero-extended to a u64 value */ - value = vmcs12_read_any(vmcs12, field, offset); + if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field)) + copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); + + /* Read the field, zero-extended to a u64 value */ + value = vmcs12_read_any(vmcs12, field, offset); + } else { + /* + * Hyper-V TLFS (as of 6.0b) explicitly states, that while an + * enlightened VMCS is active VMREAD/VMWRITE instructions are + * unsupported. Unfortunately, certain versions of Windows 11 + * don't comply with this requirement which is not enforced in + * genuine Hyper-V. Allow VMREAD from an enlightened VMCS as a + * workaround, as misbehaving guests will panic on VM-Fail. + * Note, enlightened VMCS is incompatible with shadow VMCS so + * all VMREADs from L2 should go to L1. + */ + if (WARN_ON_ONCE(is_guest_mode(vcpu))) + return nested_vmx_failInvalid(vcpu); + + offset = evmcs_field_offset(field, NULL); + if (offset < 0) + return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); + + /* Read the field, zero-extended to a u64 value */ + value = evmcs_read_any(vmx->nested.hv_evmcs, field, offset); + } /* * Now copy part of this value to register or memory, as requested. @@ -5180,12 +5222,12 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) return 1; /* - * In VMX non-root operation, when the VMCS-link pointer is -1ull, + * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, * any VMWRITE sets the ALU flags for VMfailInvalid. */ - if (vmx->nested.current_vmptr == -1ull || + if (vmx->nested.current_vmptr == INVALID_GPA || (is_guest_mode(vcpu) && - get_vmcs12(vcpu)->vmcs_link_pointer == -1ull)) + get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) return nested_vmx_failInvalid(vcpu); if (instr_info & BIT(10)) @@ -5202,7 +5244,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); - offset = vmcs_field_to_offset(field); + offset = get_vmcs12_field_offset(field); if (offset < 0) return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); @@ -5271,6 +5313,7 @@ static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) vmx->nested.need_vmcs12_to_shadow_sync = true; } vmx->nested.dirty_vmcs12 = true; + vmx->nested.force_msr_bitmap_recalc = true; } /* Emulate the VMPTRLD instruction */ @@ -5297,10 +5340,10 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) return 1; if (vmx->nested.current_vmptr != vmptr) { - struct kvm_host_map map; - struct vmcs12 *new_vmcs12; + struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; + struct vmcs_hdr hdr; - if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmptr), &map)) { + if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, vmptr, VMCS12_SIZE)) { /* * Reads from an unbacked page return all 1s, * which means that the 32 bits located at the @@ -5311,12 +5354,16 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); } - new_vmcs12 = map.hva; + if (kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, + offsetof(struct vmcs12, hdr), + sizeof(hdr))) { + return nested_vmx_fail(vcpu, + VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); + } - if (new_vmcs12->hdr.revision_id != VMCS12_REVISION || - (new_vmcs12->hdr.shadow_vmcs && + if (hdr.revision_id != VMCS12_REVISION || + (hdr.shadow_vmcs && !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { - kvm_vcpu_unmap(vcpu, &map, false); return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); } @@ -5327,8 +5374,11 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) * Load VMCS12 from guest memory since it is not already * cached. */ - memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE); - kvm_vcpu_unmap(vcpu, &map, false); + if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12, + VMCS12_SIZE)) { + return nested_vmx_fail(vcpu, + VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); + } set_current_vmptr(vmx, vmptr); } @@ -5376,7 +5426,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) struct { u64 eptp, gpa; } operand; - int i, r; + int i, r, gpr_index; if (!(vmx->nested.msrs.secondary_ctls_high & SECONDARY_EXEC_ENABLE_EPT) || @@ -5389,7 +5439,8 @@ static int handle_invept(struct kvm_vcpu *vcpu) return 1; vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); - type = kvm_register_read(vcpu, (vmx_instruction_info >> 28) & 0xf); + gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); + type = kvm_register_read(vcpu, gpr_index); types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; @@ -5456,7 +5507,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) u64 gla; } operand; u16 vpid02; - int r; + int r, gpr_index; if (!(vmx->nested.msrs.secondary_ctls_high & SECONDARY_EXEC_ENABLE_VPID) || @@ -5469,7 +5520,8 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) return 1; vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); - type = kvm_register_read(vcpu, (vmx_instruction_info >> 28) & 0xf); + gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); + type = kvm_register_read(vcpu, gpr_index); types = (vmx->nested.msrs.vpid_caps & VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; @@ -5628,7 +5680,7 @@ bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port, gpa_t bitmap, last_bitmap; u8 b; - last_bitmap = (gpa_t)-1; + last_bitmap = INVALID_GPA; b = -1; while (size > 0) { @@ -6063,7 +6115,7 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) goto reflect_vmexit; } - trace_kvm_nested_vmexit(exit_reason.full, vcpu, KVM_ISA_VMX); + trace_kvm_nested_vmexit(vcpu, KVM_ISA_VMX); /* If L0 (KVM) wants the exit, it trumps L1's desires. */ if (nested_vmx_l0_wants_exit(vcpu, exit_reason)) @@ -6104,8 +6156,8 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, .format = KVM_STATE_NESTED_FORMAT_VMX, .size = sizeof(kvm_state), .hdr.vmx.flags = 0, - .hdr.vmx.vmxon_pa = -1ull, - .hdr.vmx.vmcs12_pa = -1ull, + .hdr.vmx.vmxon_pa = INVALID_GPA, + .hdr.vmx.vmcs12_pa = INVALID_GPA, .hdr.vmx.preemption_timer_deadline = 0, }; struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = @@ -6131,7 +6183,7 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, if (is_guest_mode(vcpu) && nested_cpu_has_shadow_vmcs(vmcs12) && - vmcs12->vmcs_link_pointer != -1ull) + vmcs12->vmcs_link_pointer != INVALID_GPA) kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12); } @@ -6207,7 +6259,7 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, return -EFAULT; if (nested_cpu_has_shadow_vmcs(vmcs12) && - vmcs12->vmcs_link_pointer != -1ull) { + vmcs12->vmcs_link_pointer != INVALID_GPA) { if (copy_to_user(user_vmx_nested_state->shadow_vmcs12, get_shadow_vmcs12(vcpu), VMCS12_SIZE)) return -EFAULT; @@ -6242,11 +6294,11 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX) return -EINVAL; - if (kvm_state->hdr.vmx.vmxon_pa == -1ull) { + if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) { if (kvm_state->hdr.vmx.smm.flags) return -EINVAL; - if (kvm_state->hdr.vmx.vmcs12_pa != -1ull) + if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) return -EINVAL; /* @@ -6300,7 +6352,7 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, vmx_leave_nested(vcpu); - if (kvm_state->hdr.vmx.vmxon_pa == -1ull) + if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) return 0; vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa; @@ -6313,13 +6365,13 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, /* See vmx_has_valid_vmcs12. */ if ((kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE) || (kvm_state->flags & KVM_STATE_NESTED_EVMCS) || - (kvm_state->hdr.vmx.vmcs12_pa != -1ull)) + (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA)) return -EINVAL; else return 0; } - if (kvm_state->hdr.vmx.vmcs12_pa != -1ull) { + if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) { if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa || !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa)) return -EINVAL; @@ -6364,7 +6416,7 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, ret = -EINVAL; if (nested_cpu_has_shadow_vmcs(vmcs12) && - vmcs12->vmcs_link_pointer != -1ull) { + vmcs12->vmcs_link_pointer != INVALID_GPA) { struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); if (kvm_state->size < @@ -6397,6 +6449,7 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, goto error_guest_mode; vmx->nested.dirty_vmcs12 = true; + vmx->nested.force_msr_bitmap_recalc = true; ret = nested_vmx_enter_non_root_mode(vcpu, false); if (ret) goto error_guest_mode; @@ -6439,7 +6492,7 @@ static u64 nested_vmx_calc_vmcs_enum_msr(void) max_idx = 0; for (i = 0; i < nr_vmcs12_fields; i++) { /* The vmcs12 table is very, very sparsely populated. */ - if (!vmcs_field_to_offset_table[i]) + if (!vmcs12_field_offsets[i]) continue; idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i)); diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 6427d95de0..466d18fc0c 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -21,7 +21,6 @@ #define MSR_PMC_FULL_WIDTH_BIT (MSR_IA32_PMC0 - MSR_IA32_PERFCTR0) static struct kvm_event_hw_type_mapping intel_arch_events[] = { - /* Index must match CPUID 0x0A.EBX bit vector */ [0] = { 0x3c, 0x00, PERF_COUNT_HW_CPU_CYCLES }, [1] = { 0xc0, 0x00, PERF_COUNT_HW_INSTRUCTIONS }, [2] = { 0x3c, 0x01, PERF_COUNT_HW_BUS_CYCLES }, @@ -29,6 +28,7 @@ static struct kvm_event_hw_type_mapping intel_arch_events[] = { [4] = { 0x2e, 0x41, PERF_COUNT_HW_CACHE_MISSES }, [5] = { 0xc4, 0x00, PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, [6] = { 0xc5, 0x00, PERF_COUNT_HW_BRANCH_MISSES }, + /* The above index must match CPUID 0x0A.EBX bit vector */ [7] = { 0x00, 0x03, PERF_COUNT_HW_REF_CPU_CYCLES }, }; @@ -75,11 +75,17 @@ static unsigned int intel_pmc_perf_hw_id(struct kvm_pmc *pmc) u8 unit_mask = (pmc->eventsel & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; int i; - for (i = 0; i < ARRAY_SIZE(intel_arch_events); i++) - if (intel_arch_events[i].eventsel == event_select - && intel_arch_events[i].unit_mask == unit_mask - && (pmu->available_event_types & (1 << i))) - break; + for (i = 0; i < ARRAY_SIZE(intel_arch_events); i++) { + if (intel_arch_events[i].eventsel != event_select || + intel_arch_events[i].unit_mask != unit_mask) + continue; + + /* disable event that reported as not present by cpuid */ + if ((i < 7) && !(pmu->available_event_types & (1 << i))) + return PERF_COUNT_HW_MAX + 1; + + break; + } if (i == ARRAY_SIZE(intel_arch_events)) return PERF_COUNT_HW_MAX; @@ -87,18 +93,6 @@ static unsigned int intel_pmc_perf_hw_id(struct kvm_pmc *pmc) return intel_arch_events[i].event_type; } -static unsigned intel_find_fixed_event(int idx) -{ - u32 event; - size_t size = ARRAY_SIZE(fixed_pmc_events); - - if (idx >= size) - return PERF_COUNT_HW_MAX; - - event = fixed_pmc_events[array_index_nospec(idx, size)]; - return intel_arch_events[event].event_type; -} - /* check if a PMC is enabled by comparing it with globl_ctrl bits. */ static bool intel_pmc_is_enabled(struct kvm_pmc *pmc) { @@ -119,16 +113,15 @@ static struct kvm_pmc *intel_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx) } } -/* returns 0 if idx's corresponding MSR exists; otherwise returns 1. */ -static int intel_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) +static bool intel_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); bool fixed = idx & (1u << 30); idx &= ~(3u << 30); - return (!fixed && idx >= pmu->nr_arch_gp_counters) || - (fixed && idx >= pmu->nr_arch_fixed_counters); + return fixed ? idx < pmu->nr_arch_fixed_counters + : idx < pmu->nr_arch_gp_counters; } static struct kvm_pmc *intel_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu, @@ -366,7 +359,7 @@ static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) msr_info->data = pmu->global_ctrl; return 0; case MSR_CORE_PERF_GLOBAL_OVF_CTRL: - msr_info->data = pmu->global_ovf_ctrl; + msr_info->data = 0; return 0; default: if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || @@ -424,7 +417,6 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!(data & pmu->global_ovf_ctrl_mask)) { if (!msr_info->host_initiated) pmu->global_status &= ~data; - pmu->global_ovf_ctrl = data; return 0; } break; @@ -462,6 +454,21 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 1; } +static void setup_fixed_pmc_eventsel(struct kvm_pmu *pmu) +{ + size_t size = ARRAY_SIZE(fixed_pmc_events); + struct kvm_pmc *pmc; + u32 event; + int i; + + for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { + pmc = &pmu->fixed_counters[i]; + event = fixed_pmc_events[array_index_nospec(i, size)]; + pmc->eventsel = (intel_arch_events[event].unit_mask << 8) | + intel_arch_events[event].eventsel; + } +} + static void intel_pmu_refresh(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); @@ -480,7 +487,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->reserved_bits = 0xffffffff00200000ull; entry = kvm_find_cpuid_entry(vcpu, 0xa, 0); - if (!entry) + if (!entry || !enable_pmu) return; eax.full = entry->eax; edx.full = entry->edx; @@ -503,12 +510,14 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->nr_arch_fixed_counters = 0; } else { pmu->nr_arch_fixed_counters = - min_t(int, edx.split.num_counters_fixed, - x86_pmu.num_counters_fixed); + min3(ARRAY_SIZE(fixed_pmc_events), + (size_t) edx.split.num_counters_fixed, + (size_t) x86_pmu.num_counters_fixed); edx.split.bit_width_fixed = min_t(int, edx.split.bit_width_fixed, x86_pmu.bit_width_fixed); pmu->counter_bitmask[KVM_PMC_FIXED] = ((u64)1 << edx.split.bit_width_fixed) - 1; + setup_fixed_pmc_eventsel(pmu); } pmu->global_ctrl = ((1ull << pmu->nr_arch_gp_counters) - 1) | @@ -589,8 +598,7 @@ static void intel_pmu_reset(struct kvm_vcpu *vcpu) pmc->counter = 0; } - pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = - pmu->global_ovf_ctrl = 0; + pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0; intel_pmu_release_guest_lbr_event(vcpu); } @@ -708,7 +716,6 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) struct kvm_pmu_ops intel_pmu_ops = { .pmc_perf_hw_id = intel_pmc_perf_hw_id, - .find_fixed_event = intel_find_fixed_event, .pmc_is_enabled = intel_pmc_is_enabled, .pmc_idx_to_pmc = intel_pmc_idx_to_pmc, .rdpmc_ecx_to_pmc = intel_rdpmc_ecx_to_pmc, diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 46fb83d6a2..aa1fe9085d 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -11,58 +11,109 @@ #include "vmx.h" /* - * We maintain a per-CPU linked-list of vCPU, so in wakeup_handler() we - * can find which vCPU should be waken up. + * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler() + * when a WAKEUP_VECTOR interrupted is posted. vCPUs are added to the list when + * the vCPU is scheduled out and is blocking (e.g. in HLT) with IRQs enabled. + * The vCPUs posted interrupt descriptor is updated at the same time to set its + * notification vector to WAKEUP_VECTOR, so that posted interrupt from devices + * wake the target vCPUs. vCPUs are removed from the list and the notification + * vector is reset when the vCPU is scheduled in. */ -static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu); -static DEFINE_PER_CPU(raw_spinlock_t, blocked_vcpu_on_cpu_lock); +static DEFINE_PER_CPU(struct list_head, wakeup_vcpus_on_cpu); +/* + * Protect the per-CPU list with a per-CPU spinlock to handle task migration. + * When a blocking vCPU is awakened _and_ migrated to a different pCPU, the + * ->sched_in() path will need to take the vCPU off the list of the _previous_ + * CPU. IRQs must be disabled when taking this lock, otherwise deadlock will + * occur if a wakeup IRQ arrives and attempts to acquire the lock. + */ +static DEFINE_PER_CPU(raw_spinlock_t, wakeup_vcpus_on_cpu_lock); static inline struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu) { return &(to_vmx(vcpu)->pi_desc); } +static int pi_try_set_control(struct pi_desc *pi_desc, u64 old, u64 new) +{ + /* + * PID.ON can be set at any time by a different vCPU or by hardware, + * e.g. a device. PID.control must be written atomically, and the + * update must be retried with a fresh snapshot an ON change causes + * the cmpxchg to fail. + */ + if (cmpxchg64(&pi_desc->control, old, new) != old) + return -EBUSY; + + return 0; +} + void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); + struct vcpu_vmx *vmx = to_vmx(vcpu); struct pi_desc old, new; + unsigned long flags; unsigned int dest; /* - * In case of hot-plug or hot-unplug, we may have to undo - * vmx_vcpu_pi_put even if there is no assigned device. And we - * always keep PI.NDST up to date for simplicity: it makes the - * code easier, and CPU migration is not a fast path. + * To simplify hot-plug and dynamic toggling of APICv, keep PI.NDST and + * PI.SN up-to-date even if there is no assigned device or if APICv is + * deactivated due to a dynamic inhibit bit, e.g. for Hyper-V's SyncIC. */ - if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu) + if (!enable_apicv || !lapic_in_kernel(vcpu)) return; /* - * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change - * PI.NDST: pi_post_block is the one expected to change PID.NDST and the - * wakeup handler expects the vCPU to be on the blocked_vcpu_list that - * matches PI.NDST. Otherwise, a vcpu may not be able to be woken up - * correctly. + * If the vCPU wasn't on the wakeup list and wasn't migrated, then the + * full update can be skipped as neither the vector nor the destination + * needs to be changed. */ - if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR || vcpu->cpu == cpu) { - pi_clear_sn(pi_desc); - goto after_clear_sn; + if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR && vcpu->cpu == cpu) { + /* + * Clear SN if it was set due to being preempted. Again, do + * this even if there is no assigned device for simplicity. + */ + if (pi_test_and_clear_sn(pi_desc)) + goto after_clear_sn; + return; } - /* The full case. */ + local_irq_save(flags); + + /* + * If the vCPU was waiting for wakeup, remove the vCPU from the wakeup + * list of the _previous_ pCPU, which will not be the same as the + * current pCPU if the task was migrated. + */ + if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR) { + raw_spin_lock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu)); + list_del(&vmx->pi_wakeup_list); + raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu)); + } + + dest = cpu_physical_id(cpu); + if (!x2apic_mode) + dest = (dest << 8) & 0xFF00; + do { old.control = new.control = READ_ONCE(pi_desc->control); - dest = cpu_physical_id(cpu); - - if (x2apic_mode) - new.ndst = dest; - else - new.ndst = (dest << 8) & 0xFF00; - + /* + * Clear SN (as above) and refresh the destination APIC ID to + * handle task migration (@cpu != vcpu->cpu). + */ + new.ndst = dest; new.sn = 0; - } while (cmpxchg64(&pi_desc->control, old.control, - new.control) != old.control); + + /* + * Restore the notification vector; in the blocking case, the + * descriptor was modified on "put" to use the wakeup vector. + */ + new.nv = POSTED_INTR_VECTOR; + } while (pi_try_set_control(pi_desc, old.control, new.control)); + + local_irq_restore(flags); after_clear_sn: @@ -85,6 +136,47 @@ static bool vmx_can_use_vtd_pi(struct kvm *kvm) irq_remapping_cap(IRQ_POSTING_CAP); } +/* + * Put the vCPU on this pCPU's list of vCPUs that needs to be awakened and set + * WAKEUP as the notification vector in the PI descriptor. + */ +static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu) +{ + struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); + struct vcpu_vmx *vmx = to_vmx(vcpu); + struct pi_desc old, new; + unsigned long flags; + + local_irq_save(flags); + + raw_spin_lock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu)); + list_add_tail(&vmx->pi_wakeup_list, + &per_cpu(wakeup_vcpus_on_cpu, vcpu->cpu)); + raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu)); + + WARN(pi_desc->sn, "PI descriptor SN field set before blocking"); + + do { + old.control = new.control = READ_ONCE(pi_desc->control); + + /* set 'NV' to 'wakeup vector' */ + new.nv = POSTED_INTR_WAKEUP_VECTOR; + } while (pi_try_set_control(pi_desc, old.control, new.control)); + + /* + * Send a wakeup IPI to this CPU if an interrupt may have been posted + * before the notification vector was updated, in which case the IRQ + * will arrive on the non-wakeup vector. An IPI is needed as calling + * try_to_wake_up() from ->sched_out() isn't allowed (IRQs are not + * enabled until it is safe to call try_to_wake_up() on the task being + * scheduled out). + */ + if (pi_test_on(&new)) + apic->send_IPI_self(POSTED_INTR_WAKEUP_VECTOR); + + local_irq_restore(flags); +} + void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); @@ -92,145 +184,40 @@ void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) if (!vmx_can_use_vtd_pi(vcpu->kvm)) return; - /* Set SN when the vCPU is preempted */ + if (kvm_vcpu_is_blocking(vcpu) && !vmx_interrupt_blocked(vcpu)) + pi_enable_wakeup_handler(vcpu); + + /* + * Set SN when the vCPU is preempted. Note, the vCPU can both be seen + * as blocking and preempted, e.g. if it's preempted between setting + * its wait state and manually scheduling out. + */ if (vcpu->preempted) pi_set_sn(pi_desc); } -static void __pi_post_block(struct kvm_vcpu *vcpu) -{ - struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); - struct pi_desc old, new; - unsigned int dest; - - do { - old.control = new.control = READ_ONCE(pi_desc->control); - WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR, - "Wakeup handler not enabled while the VCPU is blocked\n"); - - dest = cpu_physical_id(vcpu->cpu); - - if (x2apic_mode) - new.ndst = dest; - else - new.ndst = (dest << 8) & 0xFF00; - - /* set 'NV' to 'notification vector' */ - new.nv = POSTED_INTR_VECTOR; - } while (cmpxchg64(&pi_desc->control, old.control, - new.control) != old.control); - - if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) { - raw_spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); - list_del(&vcpu->blocked_vcpu_list); - raw_spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); - vcpu->pre_pcpu = -1; - } -} - -/* - * This routine does the following things for vCPU which is going - * to be blocked if VT-d PI is enabled. - * - Store the vCPU to the wakeup list, so when interrupts happen - * we can find the right vCPU to wake up. - * - Change the Posted-interrupt descriptor as below: - * 'NDST' <-- vcpu->pre_pcpu - * 'NV' <-- POSTED_INTR_WAKEUP_VECTOR - * - If 'ON' is set during this process, which means at least one - * interrupt is posted for this vCPU, we cannot block it, in - * this case, return 1, otherwise, return 0. - * - */ -int pi_pre_block(struct kvm_vcpu *vcpu) -{ - unsigned int dest; - struct pi_desc old, new; - struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); - - if (!vmx_can_use_vtd_pi(vcpu->kvm) || - vmx_interrupt_blocked(vcpu)) - return 0; - - WARN_ON(irqs_disabled()); - local_irq_disable(); - if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) { - vcpu->pre_pcpu = vcpu->cpu; - raw_spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); - list_add_tail(&vcpu->blocked_vcpu_list, - &per_cpu(blocked_vcpu_on_cpu, - vcpu->pre_pcpu)); - raw_spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); - } - - do { - old.control = new.control = READ_ONCE(pi_desc->control); - - WARN((pi_desc->sn == 1), - "Warning: SN field of posted-interrupts " - "is set before blocking\n"); - - /* - * Since vCPU can be preempted during this process, - * vcpu->cpu could be different with pre_pcpu, we - * need to set pre_pcpu as the destination of wakeup - * notification event, then we can find the right vCPU - * to wakeup in wakeup handler if interrupts happen - * when the vCPU is in blocked state. - */ - dest = cpu_physical_id(vcpu->pre_pcpu); - - if (x2apic_mode) - new.ndst = dest; - else - new.ndst = (dest << 8) & 0xFF00; - - /* set 'NV' to 'wakeup vector' */ - new.nv = POSTED_INTR_WAKEUP_VECTOR; - } while (cmpxchg64(&pi_desc->control, old.control, - new.control) != old.control); - - /* We should not block the vCPU if an interrupt is posted for it. */ - if (pi_test_on(pi_desc) == 1) - __pi_post_block(vcpu); - - local_irq_enable(); - return (vcpu->pre_pcpu == -1); -} - -void pi_post_block(struct kvm_vcpu *vcpu) -{ - if (vcpu->pre_pcpu == -1) - return; - - WARN_ON(irqs_disabled()); - local_irq_disable(); - __pi_post_block(vcpu); - local_irq_enable(); -} - /* * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR. */ void pi_wakeup_handler(void) { - struct kvm_vcpu *vcpu; int cpu = smp_processor_id(); + struct vcpu_vmx *vmx; - raw_spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); - list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu), - blocked_vcpu_list) { - struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); + raw_spin_lock(&per_cpu(wakeup_vcpus_on_cpu_lock, cpu)); + list_for_each_entry(vmx, &per_cpu(wakeup_vcpus_on_cpu, cpu), + pi_wakeup_list) { - if (pi_test_on(pi_desc) == 1) - kvm_vcpu_kick(vcpu); + if (pi_test_on(&vmx->pi_desc)) + kvm_vcpu_wake_up(&vmx->vcpu); } - raw_spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); + raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, cpu)); } void __init pi_init_cpu(int cpu) { - INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu)); - raw_spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); + INIT_LIST_HEAD(&per_cpu(wakeup_vcpus_on_cpu, cpu)); + raw_spin_lock_init(&per_cpu(wakeup_vcpus_on_cpu_lock, cpu)); } bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu) @@ -246,7 +233,7 @@ bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu) * Bail out of the block loop if the VM has an assigned * device, but the blocking vCPU didn't reconfigure the * PI.NV to the wakeup vector, i.e. the assigned device - * came along after the initial check in pi_pre_block(). + * came along after the initial check in vmx_vcpu_pi_put(). */ void vmx_pi_start_assignment(struct kvm *kvm) { diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h index 7f7b2326ca..eb14e76b84 100644 --- a/arch/x86/kvm/vmx/posted_intr.h +++ b/arch/x86/kvm/vmx/posted_intr.h @@ -40,7 +40,13 @@ static inline bool pi_test_and_clear_on(struct pi_desc *pi_desc) (unsigned long *)&pi_desc->control); } -static inline int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc) +static inline bool pi_test_and_clear_sn(struct pi_desc *pi_desc) +{ + return test_and_clear_bit(POSTED_INTR_SN, + (unsigned long *)&pi_desc->control); +} + +static inline bool pi_test_and_set_pir(int vector, struct pi_desc *pi_desc) { return test_and_set_bit(vector, (unsigned long *)pi_desc->pir); } @@ -74,13 +80,13 @@ static inline void pi_clear_sn(struct pi_desc *pi_desc) (unsigned long *)&pi_desc->control); } -static inline int pi_test_on(struct pi_desc *pi_desc) +static inline bool pi_test_on(struct pi_desc *pi_desc) { return test_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control); } -static inline int pi_test_sn(struct pi_desc *pi_desc) +static inline bool pi_test_sn(struct pi_desc *pi_desc) { return test_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control); @@ -88,8 +94,6 @@ static inline int pi_test_sn(struct pi_desc *pi_desc) void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu); -int pi_pre_block(struct kvm_vcpu *vcpu); -void pi_post_block(struct kvm_vcpu *vcpu); void pi_wakeup_handler(void); void __init pi_init_cpu(int cpu); bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c index 6693ebdc07..35e7ec91ae 100644 --- a/arch/x86/kvm/vmx/sgx.c +++ b/arch/x86/kvm/vmx/sgx.c @@ -53,11 +53,9 @@ static int sgx_get_encls_gva(struct kvm_vcpu *vcpu, unsigned long offset, static void sgx_handle_emulation_failure(struct kvm_vcpu *vcpu, u64 addr, unsigned int size) { - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 2; - vcpu->run->internal.data[0] = addr; - vcpu->run->internal.data[1] = size; + uint64_t data[2] = { addr, size }; + + __kvm_prepare_emulation_failure_exit(vcpu, data, ARRAY_SIZE(data)); } static int sgx_read_hva(struct kvm_vcpu *vcpu, unsigned long hva, void *data, @@ -112,9 +110,7 @@ static int sgx_inject_fault(struct kvm_vcpu *vcpu, gva_t gva, int trapnr) * but the error code isn't (yet) plumbed through the ENCLS helpers. */ if (trapnr == PF_VECTOR && !boot_cpu_has(X86_FEATURE_SGX2)) { - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + kvm_prepare_emulation_failure_exit(vcpu); return 0; } @@ -155,9 +151,7 @@ static int __handle_encls_ecreate(struct kvm_vcpu *vcpu, sgx_12_0 = kvm_find_cpuid_entry(vcpu, 0x12, 0); sgx_12_1 = kvm_find_cpuid_entry(vcpu, 0x12, 1); if (!sgx_12_0 || !sgx_12_1) { - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + kvm_prepare_emulation_failure_exit(vcpu); return 0; } diff --git a/arch/x86/kvm/vmx/vmcs.h b/arch/x86/kvm/vmx/vmcs.h index 6e5de2e2b0..e325c290a8 100644 --- a/arch/x86/kvm/vmx/vmcs.h +++ b/arch/x86/kvm/vmx/vmcs.h @@ -129,6 +129,11 @@ static inline bool is_machine_check(u32 intr_info) return is_exception_n(intr_info, MC_VECTOR); } +static inline bool is_nm_fault(u32 intr_info) +{ + return is_exception_n(intr_info, NM_VECTOR); +} + /* Undocumented: icebp/int1 */ static inline bool is_icebp(u32 intr_info) { diff --git a/arch/x86/kvm/vmx/vmcs12.c b/arch/x86/kvm/vmx/vmcs12.c index cab6ba7a50..2251b60920 100644 --- a/arch/x86/kvm/vmx/vmcs12.c +++ b/arch/x86/kvm/vmx/vmcs12.c @@ -8,7 +8,7 @@ FIELD(number, name), \ [ROL16(number##_HIGH, 6)] = VMCS12_OFFSET(name) + sizeof(u32) -const unsigned short vmcs_field_to_offset_table[] = { +const unsigned short vmcs12_field_offsets[] = { FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id), FIELD(POSTED_INTR_NV, posted_intr_nv), FIELD(GUEST_ES_SELECTOR, guest_es_selector), @@ -151,4 +151,4 @@ const unsigned short vmcs_field_to_offset_table[] = { FIELD(HOST_RSP, host_rsp), FIELD(HOST_RIP, host_rip), }; -const unsigned int nr_vmcs12_fields = ARRAY_SIZE(vmcs_field_to_offset_table); +const unsigned int nr_vmcs12_fields = ARRAY_SIZE(vmcs12_field_offsets); diff --git a/arch/x86/kvm/vmx/vmcs12.h b/arch/x86/kvm/vmx/vmcs12.h index 2a45f026ee..746129ddd5 100644 --- a/arch/x86/kvm/vmx/vmcs12.h +++ b/arch/x86/kvm/vmx/vmcs12.h @@ -361,10 +361,10 @@ static inline void vmx_check_vmcs12_offsets(void) CHECK_OFFSET(guest_pml_index, 996); } -extern const unsigned short vmcs_field_to_offset_table[]; +extern const unsigned short vmcs12_field_offsets[]; extern const unsigned int nr_vmcs12_fields; -static inline short vmcs_field_to_offset(unsigned long field) +static inline short get_vmcs12_field_offset(unsigned long field) { unsigned short offset; unsigned int index; @@ -377,7 +377,7 @@ static inline short vmcs_field_to_offset(unsigned long field) return -ENOENT; index = array_index_nospec(index, nr_vmcs12_fields); - offset = vmcs_field_to_offset_table[index]; + offset = vmcs12_field_offsets[index]; if (offset == 0) return -ENOENT; return offset; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 322485ab92..b730d799c2 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -35,7 +35,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -161,6 +162,8 @@ static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = { MSR_FS_BASE, MSR_GS_BASE, MSR_KERNEL_GS_BASE, + MSR_IA32_XFD, + MSR_IA32_XFD_ERR, #endif MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, @@ -602,15 +605,13 @@ static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx, unsigned int slot = msr - vmx->guest_uret_msrs; int ret = 0; - u64 old_msr_data = msr->data; - msr->data = data; if (msr->load_into_hardware) { preempt_disable(); - ret = kvm_set_user_return_msr(slot, msr->data, msr->mask); + ret = kvm_set_user_return_msr(slot, data, msr->mask); preempt_enable(); - if (ret) - msr->data = old_msr_data; } + if (!ret) + msr->data = data; return ret; } @@ -763,6 +764,14 @@ void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu) vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, match); } + /* + * Disabling xfd interception indicates that dynamic xfeatures + * might be used in the guest. Always trap #NM in this case + * to save guest xfd_err timely. + */ + if (vcpu->arch.xfd_no_write_intercept) + eb |= (1u << NM_VECTOR); + vmcs_write32(EXCEPTION_BITMAP, eb); } @@ -1048,8 +1057,8 @@ static void pt_guest_enter(struct vcpu_vmx *vmx) rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { wrmsrl(MSR_IA32_RTIT_CTL, 0); - pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range); - pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range); + pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); + pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); } } @@ -1059,12 +1068,16 @@ static void pt_guest_exit(struct vcpu_vmx *vmx) return; if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { - pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range); - pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range); + pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); + pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); } - /* Reload host state (IA32_RTIT_CTL will be cleared on VM exit). */ - wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); + /* + * KVM requires VM_EXIT_CLEAR_IA32_RTIT_CTL to expose PT to the guest, + * i.e. RTIT_CTL is always cleared on VM-Exit. Restore it if necessary. + */ + if (vmx->pt_desc.host.ctl) + wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); } void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, @@ -1267,7 +1280,6 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, if (!already_loaded) { void *gdt = get_current_gdt_ro(); - unsigned long sysenter_esp; /* * Flush all EPTP/VPID contexts, the new pCPU may have stale @@ -1283,8 +1295,11 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss); vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */ - rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp); - vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */ + if (IS_ENABLED(CONFIG_IA32_EMULATION) || IS_ENABLED(CONFIG_X86_32)) { + /* 22.2.3 */ + vmcs_writel(HOST_IA32_SYSENTER_ESP, + (unsigned long)(cpu_entry_stack(cpu) + 1)); + } vmx->loaded_vmcs->cpu = cpu; } @@ -1450,26 +1465,27 @@ static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) * cause a #GP fault. */ value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET; - if ((value && (vmx->pt_desc.addr_range < 1)) || (value > 2)) + if ((value && (vmx->pt_desc.num_address_ranges < 1)) || (value > 2)) return 1; value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET; - if ((value && (vmx->pt_desc.addr_range < 2)) || (value > 2)) + if ((value && (vmx->pt_desc.num_address_ranges < 2)) || (value > 2)) return 1; value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET; - if ((value && (vmx->pt_desc.addr_range < 3)) || (value > 2)) + if ((value && (vmx->pt_desc.num_address_ranges < 3)) || (value > 2)) return 1; value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET; - if ((value && (vmx->pt_desc.addr_range < 4)) || (value > 2)) + if ((value && (vmx->pt_desc.num_address_ranges < 4)) || (value > 2)) return 1; return 0; } -static bool vmx_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int insn_len) +static bool vmx_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, + void *insn, int insn_len) { /* * Emulation of instructions in SGX enclaves is impossible as RIP does - * not point tthe failing instruction, and even if it did, the code + * not point at the failing instruction, and even if it did, the code * stream is inaccessible. Inject #UD instead of exiting to userspace * so that guest userspace can't DoS the guest simply by triggering * emulation (enclaves are CPL3 only). @@ -1749,7 +1765,7 @@ static int vmx_get_msr_feature(struct kvm_msr_entry *msr) } /* - * Reads an msr value (of 'msr_index') into 'pdata'. + * Reads an msr value (of 'msr_info->index') into 'msr_info->data'. * Returns 0 on success, non-0 otherwise. * Assumes vcpu_load() was already called. */ @@ -1880,8 +1896,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; if (!vmx_pt_mode_is_host_guest() || - (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps, - PT_CAP_num_address_ranges))) + (index >= 2 * vmx->pt_desc.num_address_ranges)) return 1; if (index % 2) msr_info->data = vmx->pt_desc.guest.addr_b[index / 2]; @@ -1957,6 +1972,24 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_KERNEL_GS_BASE: vmx_write_guest_kernel_gs_base(vmx, data); break; + case MSR_IA32_XFD: + ret = kvm_set_msr_common(vcpu, msr_info); + /* + * Always intercepting WRMSR could incur non-negligible + * overhead given xfd might be changed frequently in + * guest context switch. Disable write interception + * upon the first write with a non-zero value (indicating + * potential usage on dynamic xfeatures). Also update + * exception bitmap to trap #NM for proper virtualization + * of guest xfd_err. + */ + if (!ret && data) { + vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD, + MSR_TYPE_RW); + vcpu->arch.xfd_no_write_intercept = true; + vmx_update_exception_bitmap(vcpu); + } + break; #endif case MSR_IA32_SYSENTER_CS: if (is_guest_mode(vcpu)) @@ -2097,9 +2130,6 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) } ret = kvm_set_msr_common(vcpu, msr_info); break; - case MSR_IA32_TSC_ADJUST: - ret = kvm_set_msr_common(vcpu, msr_info); - break; case MSR_IA32_MCG_EXT_CTL: if ((!msr_info->host_initiated && !(to_vmx(vcpu)->msr_ia32_feature_control & @@ -2196,8 +2226,7 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!pt_can_write_msr(vmx)) return 1; index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; - if (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps, - PT_CAP_num_address_ranges)) + if (index >= 2 * vmx->pt_desc.num_address_ranges) return 1; if (is_noncanonical_address(data, vcpu)) return 1; @@ -2568,7 +2597,6 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, return -EIO; vmcs_conf->size = vmx_msr_high & 0x1fff; - vmcs_conf->order = get_order(vmcs_conf->size); vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff; vmcs_conf->revision_id = vmx_msr_low; @@ -2593,7 +2621,7 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) struct page *pages; struct vmcs *vmcs; - pages = __alloc_pages_node(node, flags, vmcs_config.order); + pages = __alloc_pages_node(node, flags, 0); if (!pages) return NULL; vmcs = page_address(pages); @@ -2612,7 +2640,7 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) void free_vmcs(struct vmcs *vmcs) { - free_pages((unsigned long)vmcs, vmcs_config.order); + free_page((unsigned long)vmcs); } /* @@ -2649,15 +2677,6 @@ int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) if (!loaded_vmcs->msr_bitmap) goto out_vmcs; memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); - - if (IS_ENABLED(CONFIG_HYPERV) && - static_branch_unlikely(&enable_evmcs) && - (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) { - struct hv_enlightened_vmcs *evmcs = - (struct hv_enlightened_vmcs *)loaded_vmcs->vmcs; - - evmcs->hv_enlightenments_control.msr_bitmap = 1; - } } memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state)); @@ -2992,7 +3011,7 @@ void ept_save_pdptrs(struct kvm_vcpu *vcpu) mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); - kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); + kvm_register_mark_available(vcpu, VCPU_EXREG_PDPTR); } #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \ @@ -3074,6 +3093,13 @@ void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) /* Note, vmx_set_cr4() consumes the new vcpu->arch.cr0. */ if ((old_cr0_pg ^ cr0) & X86_CR0_PG) vmx_set_cr4(vcpu, kvm_read_cr4(vcpu)); + + /* + * When !CR0_PG -> CR0_PG, vcpu->arch.cr3 becomes active, but + * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG. + */ + if (!(old_cr0_pg & X86_CR0_PG) && (cr0 & X86_CR0_PG)) + kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); } /* depends on vcpu->arch.cr0 to be set to a new value */ @@ -3117,9 +3143,9 @@ static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, if (!enable_unrestricted_guest && !is_paging(vcpu)) guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; - else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) + else if (kvm_register_is_dirty(vcpu, VCPU_EXREG_CR3)) guest_cr3 = vcpu->arch.cr3; - else /* vmcs01.GUEST_CR3 is already up-to-date. */ + else /* vmcs.GUEST_CR3 is already up-to-date. */ update_guest_cr3 = false; vmx_ept_load_pdptrs(vcpu); } else { @@ -3130,6 +3156,7 @@ static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, vmcs_writel(GUEST_CR3, guest_cr3); } + static bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { /* @@ -3694,6 +3721,19 @@ void free_vpid(int vpid) spin_unlock(&vmx_vpid_lock); } +static void vmx_msr_bitmap_l01_changed(struct vcpu_vmx *vmx) +{ + /* + * When KVM is a nested hypervisor on top of Hyper-V and uses + * 'Enlightened MSR Bitmap' feature L0 needs to know that MSR + * bitmap has changed. + */ + if (static_branch_unlikely(&enable_evmcs)) + evmcs_touch_msr_bitmap(); + + vmx->nested.force_msr_bitmap_recalc = true; +} + void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -3702,8 +3742,7 @@ void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) if (!cpu_has_vmx_msr_bitmap()) return; - if (static_branch_unlikely(&enable_evmcs)) - evmcs_touch_msr_bitmap(); + vmx_msr_bitmap_l01_changed(vmx); /* * Mark the desired intercept state in shadow bitmap, this is needed @@ -3747,8 +3786,7 @@ void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) if (!cpu_has_vmx_msr_bitmap()) return; - if (static_branch_unlikely(&enable_evmcs)) - evmcs_touch_msr_bitmap(); + vmx_msr_bitmap_l01_changed(vmx); /* * Mark the desired intercept state in shadow bitmap, this is needed @@ -3838,7 +3876,7 @@ void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu) vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_BASE, MSR_TYPE_RW, flag); vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_MASK, MSR_TYPE_RW, flag); vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_CR3_MATCH, MSR_TYPE_RW, flag); - for (i = 0; i < vmx->pt_desc.addr_range; i++) { + for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) { vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag); vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag); } @@ -3886,12 +3924,10 @@ static void vmx_msr_filter_changed(struct kvm_vcpu *vcpu) pt_update_intercept_for_msr(vcpu); } -static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, - bool nested) +static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, + int pi_vec) { #ifdef CONFIG_SMP - int pi_vec = nested ? POSTED_INTR_NESTED_VECTOR : POSTED_INTR_VECTOR; - if (vcpu->mode == IN_GUEST_MODE) { /* * The vector of interrupt to be delivered to vcpu had @@ -3919,10 +3955,15 @@ static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, */ apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec); - return true; + return; } #endif - return false; + /* + * The vCPU isn't in the guest; wake the vCPU in case it is blocking, + * otherwise do nothing as KVM will grab the highest priority pending + * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest(). + */ + kvm_vcpu_wake_up(vcpu); } static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, @@ -3938,9 +3979,21 @@ static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, */ vmx->nested.pi_pending = true; kvm_make_request(KVM_REQ_EVENT, vcpu); + + /* + * This pairs with the smp_mb_*() after setting vcpu->mode in + * vcpu_enter_guest() to guarantee the vCPU sees the event + * request if triggering a posted interrupt "fails" because + * vcpu->mode != IN_GUEST_MODE. The extra barrier is needed as + * the smb_wmb() in kvm_make_request() only ensures everything + * done before making the request is visible when the request + * is visible, it doesn't ensure ordering between the store to + * vcpu->requests and the load from vcpu->mode. + */ + smp_mb__after_atomic(); + /* the PIR and ON have been set by L1. */ - if (!kvm_vcpu_trigger_posted_interrupt(vcpu, true)) - kvm_vcpu_kick(vcpu); + kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_NESTED_VECTOR); return 0; } return -1; @@ -3971,12 +4024,31 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) if (pi_test_and_set_on(&vmx->pi_desc)) return 0; - if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false)) - kvm_vcpu_kick(vcpu); - + /* + * The implied barrier in pi_test_and_set_on() pairs with the smp_mb_*() + * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is + * guaranteed to see PID.ON=1 and sync the PIR to IRR if triggering a + * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE. + */ + kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); return 0; } +static void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, + int trig_mode, int vector) +{ + struct kvm_vcpu *vcpu = apic->vcpu; + + if (vmx_deliver_posted_interrupt(vcpu, vector)) { + kvm_lapic_set_irr(vector, apic); + kvm_make_request(KVM_REQ_EVENT, vcpu); + kvm_vcpu_kick(vcpu); + } else { + trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, + trig_mode, vector); + } +} + /* * Set up the vmcs's constant host-state fields, i.e., host-state fields that * will not change in the lifetime of the guest. @@ -4028,6 +4100,16 @@ void vmx_set_constant_host_state(struct vcpu_vmx *vmx) rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); vmcs_write32(HOST_IA32_SYSENTER_CS, low32); + + /* + * SYSENTER is used for 32-bit system calls on either 32-bit or + * 64-bit kernels. It is always zero If neither is allowed, otherwise + * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may + * have already done so!). + */ + if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32)) + vmcs_writel(HOST_IA32_SYSENTER_ESP, 0); + rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl); vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */ @@ -4046,8 +4128,10 @@ void set_cr4_guest_host_mask(struct vcpu_vmx *vmx) vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS & ~vcpu->arch.cr4_guest_rsvd_bits; - if (!enable_ept) - vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PGE; + if (!enable_ept) { + vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_TLBFLUSH_BITS; + vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PDPTR_BITS; + } if (is_guest_mode(&vmx->vcpu)) vcpu->arch.cr4_guest_owned_bits &= ~get_vmcs12(vcpu)->cr4_guest_host_mask; @@ -4286,10 +4370,6 @@ static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx) #define VMX_XSS_EXIT_BITMAP 0 -/* - * Noting that the initialization of Guest-state Area of VMCS is in - * vmx_vcpu_reset(). - */ static void init_vmcs(struct vcpu_vmx *vmx) { if (nested) @@ -4298,7 +4378,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) if (cpu_has_vmx_msr_bitmap()) vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); - vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */ + vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); /* 22.3.1.5 */ /* Control */ pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); @@ -4394,10 +4474,40 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmx_setup_uret_msrs(vmx); } +static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + init_vmcs(vmx); + + if (nested) + memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs)); + + vcpu_setup_sgx_lepubkeyhash(vcpu); + + vmx->nested.posted_intr_nv = -1; + vmx->nested.vmxon_ptr = INVALID_GPA; + vmx->nested.current_vmptr = INVALID_GPA; + vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; + + vcpu->arch.microcode_version = 0x100000000ULL; + vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; + + /* + * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR + * or POSTED_INTR_WAKEUP_VECTOR. + */ + vmx->pi_desc.nv = POSTED_INTR_VECTOR; + vmx->pi_desc.sn = 1; +} + static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) { struct vcpu_vmx *vmx = to_vmx(vcpu); + if (!init_event) + __vmx_vcpu_reset(vcpu); + vmx->rmode.vm86_active = 0; vmx->spec_ctrl = 0; @@ -4407,6 +4517,7 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) kvm_set_cr8(vcpu, 0); vmx_segment_cache_clear(vmx); + kvm_register_mark_available(vcpu, VCPU_EXREG_SEGMENTS); seg_setup(VCPU_SREG_CS); vmcs_write16(GUEST_CS_SELECTOR, 0xf000); @@ -4672,7 +4783,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, if (kvm_emulate_instruction(vcpu, 0)) { if (vcpu->arch.halt_request) { vcpu->arch.halt_request = 0; - return kvm_vcpu_halt(vcpu); + return kvm_emulate_halt_noskip(vcpu); } return 1; } @@ -4728,6 +4839,17 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) if (is_machine_check(intr_info) || is_nmi(intr_info)) return 1; /* handled by handle_exception_nmi_irqoff() */ + /* + * Queue the exception here instead of in handle_nm_fault_irqoff(). + * This ensures the nested_vmx check is not skipped so vmexit can + * be reflected to L1 (when it intercepts #NM) before reaching this + * point. + */ + if (is_nm_fault(intr_info)) { + kvm_queue_exception(vcpu, NM_VECTOR); + return 1; + } + if (is_invalid_opcode(intr_info)) return handle_ud(vcpu); @@ -5312,7 +5434,7 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu) { gpa_t gpa; - if (!vmx_can_emulate_instruction(vcpu, NULL, 0)) + if (!vmx_can_emulate_instruction(vcpu, EMULTYPE_PF, NULL, 0)) return 1; /* @@ -5341,6 +5463,14 @@ static int handle_nmi_window(struct kvm_vcpu *vcpu) return 1; } +static bool vmx_emulation_required_with_pending_exception(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + return vmx->emulation_required && !vmx->rmode.vm86_active && + vcpu->arch.exception.pending; +} + static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -5360,18 +5490,14 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) if (!kvm_emulate_instruction(vcpu, 0)) return 0; - if (vmx->emulation_required && !vmx->rmode.vm86_active && - vcpu->arch.exception.pending) { - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = - KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + if (vmx_emulation_required_with_pending_exception(vcpu)) { + kvm_prepare_emulation_failure_exit(vcpu); return 0; } if (vcpu->arch.halt_request) { vcpu->arch.halt_request = 0; - return kvm_vcpu_halt(vcpu); + return kvm_emulate_halt_noskip(vcpu); } /* @@ -5386,6 +5512,16 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) return 1; } +static int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu) +{ + if (vmx_emulation_required_with_pending_exception(vcpu)) { + kvm_prepare_emulation_failure_exit(vcpu); + return 0; + } + + return 1; +} + static void grow_ple_window(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -5451,6 +5587,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) u64 pcid; u64 gla; } operand; + int gpr_index; if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { kvm_queue_exception(vcpu, UD_VECTOR); @@ -5458,12 +5595,8 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) } vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); - type = kvm_register_read(vcpu, (vmx_instruction_info >> 28) & 0xf); - - if (type > 3) { - kvm_inject_gp(vcpu, 0); - return 1; - } + gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); + type = kvm_register_read(vcpu, gpr_index); /* According to the Intel instruction reference, the memory operand * is read even if it isn't needed (e.g., for type==all) @@ -5616,11 +5749,13 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { static const int kvm_vmx_max_exit_handlers = ARRAY_SIZE(kvm_vmx_exit_handlers); -static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2, +static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, + u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { struct vcpu_vmx *vmx = to_vmx(vcpu); + *reason = vmx->exit_reason.full; *info1 = vmx_get_exit_qual(vcpu); if (!(vmx->exit_reason.failed_vmentry)) { *info2 = vmx->idt_vectoring_info; @@ -6353,11 +6488,33 @@ void vmx_do_interrupt_nmi_irqoff(unsigned long entry); static void handle_interrupt_nmi_irqoff(struct kvm_vcpu *vcpu, unsigned long entry) { - kvm_before_interrupt(vcpu); + bool is_nmi = entry == (unsigned long)asm_exc_nmi_noist; + + kvm_before_interrupt(vcpu, is_nmi ? KVM_HANDLING_NMI : KVM_HANDLING_IRQ); vmx_do_interrupt_nmi_irqoff(entry); kvm_after_interrupt(vcpu); } +static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) +{ + /* + * Save xfd_err to guest_fpu before interrupt is enabled, so the + * MSR value is not clobbered by the host activity before the guest + * has chance to consume it. + * + * Do not blindly read xfd_err here, since this exception might + * be caused by L1 interception on a platform which doesn't + * support xfd at all. + * + * Do it conditionally upon guest_fpu::xfd. xfd_err matters + * only when xfd contains a non-zero value. + * + * Queuing exception is done in vmx_handle_exit. See comment there. + */ + if (vcpu->arch.guest_fpu.fpstate->xfd) + rdmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); +} + static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) { const unsigned long nmi_entry = (unsigned long)asm_exc_nmi_noist; @@ -6366,6 +6523,9 @@ static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) /* if exit due to PF check for async PF */ if (is_page_fault(intr_info)) vmx->vcpu.arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); + /* if exit due to NM, handle before interrupts are enabled */ + else if (is_nm_fault(intr_info)) + handle_nm_fault_irqoff(&vmx->vcpu); /* Handle machine checks before interrupts are enabled */ else if (is_machine_check(intr_info)) kvm_machine_check(); @@ -6416,6 +6576,7 @@ static bool vmx_has_emulated_msr(struct kvm *kvm, u32 index) case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: return nested; case MSR_AMD64_VIRT_SPEC_CTRL: + case MSR_AMD64_TSC_RATIO: /* This is AMD only. */ return false; default: @@ -6601,7 +6762,7 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) { - kvm_guest_enter_irqoff(); + guest_state_enter_irqoff(); /* L1D Flush includes CPU buffer clear to mitigate MDS */ if (static_branch_unlikely(&vmx_l1d_should_flush)) @@ -6617,7 +6778,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, vcpu->arch.cr2 = native_read_cr2(); - kvm_guest_exit_irqoff(); + guest_state_exit_irqoff(); } static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) @@ -6664,7 +6825,15 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP)) vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); + vcpu->arch.regs_dirty = 0; + /* + * Refresh vmcs.HOST_CR3 if necessary. This must be done immediately + * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time + * it switches back to the current->mm, which can occur in KVM context + * when switching to a temporary mm to patch kernel code, e.g. if KVM + * toggles a static key while handling a VM-Exit. + */ cr3 = __get_current_cr3_fast(); if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { vmcs_writel(HOST_CR3, cr3); @@ -6758,7 +6927,7 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) loadsegment(es, __USER_DS); #endif - vmx_register_cache_reset(vcpu); + vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET; pt_guest_exit(vmx); @@ -6790,7 +6959,7 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) if (likely(!vmx->exit_reason.failed_vmentry)) vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); - trace_kvm_exit(vmx->exit_reason.full, vcpu, KVM_ISA_VMX); + trace_kvm_exit(vcpu, KVM_ISA_VMX); if (unlikely(vmx->exit_reason.failed_vmentry)) return EXIT_FASTPATH_NONE; @@ -6821,11 +6990,13 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) { struct vmx_uret_msr *tsx_ctrl; struct vcpu_vmx *vmx; - int i, cpu, err; + int i, err; BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0); vmx = to_vmx(vcpu); + INIT_LIST_HEAD(&vmx->pi_wakeup_list); + err = -ENOMEM; vmx->vpid = allocate_vpid(); @@ -6842,10 +7013,8 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) goto free_vpid; } - for (i = 0; i < kvm_nr_uret_msrs; ++i) { - vmx->guest_uret_msrs[i].data = 0; + for (i = 0; i < kvm_nr_uret_msrs; ++i) vmx->guest_uret_msrs[i].mask = -1ull; - } if (boot_cpu_has(X86_FEATURE_RTM)) { /* * TSX_CTRL_CPUID_CLEAR is handled in the CPUID interception. @@ -6861,6 +7030,19 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) if (err < 0) goto free_pml; + /* + * Use Hyper-V 'Enlightened MSR Bitmap' feature when KVM runs as a + * nested (L1) hypervisor and Hyper-V in L0 supports it. Enable the + * feature only for vmcs01, KVM currently isn't equipped to realize any + * performance benefits from enabling it for vmcs02. + */ + if (IS_ENABLED(CONFIG_HYPERV) && static_branch_unlikely(&enable_evmcs) && + (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) { + struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; + + evmcs->hv_enlightenments_control.msr_bitmap = 1; + } + /* The MSR bitmap starts with all ones */ bitmap_fill(vmx->shadow_msr_intercept.read, MAX_POSSIBLE_PASSTHROUGH_MSRS); bitmap_fill(vmx->shadow_msr_intercept.write, MAX_POSSIBLE_PASSTHROUGH_MSRS); @@ -6882,12 +7064,7 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) } vmx->loaded_vmcs = &vmx->vmcs01; - cpu = get_cpu(); - vmx_vcpu_load(vcpu, cpu); - vcpu->cpu = cpu; - init_vmcs(vmx); - vmx_vcpu_put(vcpu); - put_cpu(); + if (cpu_need_virtualize_apic_accesses(vcpu)) { err = alloc_apic_access_page(vcpu->kvm); if (err) @@ -6900,27 +7077,6 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) goto free_vmcs; } - if (nested) - memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs)); - else - memset(&vmx->nested.msrs, 0, sizeof(vmx->nested.msrs)); - - vcpu_setup_sgx_lepubkeyhash(vcpu); - - vmx->nested.posted_intr_nv = -1; - vmx->nested.current_vmptr = -1ull; - vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; - - vcpu->arch.microcode_version = 0x100000000ULL; - vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; - - /* - * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR - * or POSTED_INTR_WAKEUP_VECTOR. - */ - vmx->pi_desc.nv = POSTED_INTR_VECTOR; - vmx->pi_desc.sn = 1; - return 0; free_vmcs: @@ -6992,7 +7148,6 @@ static int __init vmx_check_processor_compat(void) static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) { u8 cache; - u64 ipat = 0; /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in * memory aliases with conflicting memory types and sometimes MCEs. @@ -7012,30 +7167,22 @@ static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) * EPT memory type is used to emulate guest CD/MTRR. */ - if (is_mmio) { - cache = MTRR_TYPE_UNCACHABLE; - goto exit; - } + if (is_mmio) + return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT; - if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) { - ipat = VMX_EPT_IPAT_BIT; - cache = MTRR_TYPE_WRBACK; - goto exit; - } + if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) + return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; if (kvm_read_cr0(vcpu) & X86_CR0_CD) { - ipat = VMX_EPT_IPAT_BIT; if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) cache = MTRR_TYPE_WRBACK; else cache = MTRR_TYPE_UNCACHABLE; - goto exit; + + return (cache << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; } - cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn); - -exit: - return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat; + return kvm_mtrr_get_guest_memory_type(vcpu, gfn) << VMX_EPT_MT_EPTE_SHIFT; } static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx, u32 new_ctl) @@ -7135,12 +7282,13 @@ static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) } /* Get the number of configurable Address Ranges for filtering */ - vmx->pt_desc.addr_range = intel_pt_validate_cap(vmx->pt_desc.caps, + vmx->pt_desc.num_address_ranges = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_num_address_ranges); /* Initialize and clear the no dependency bits */ vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS | - RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC); + RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC | + RTIT_CTL_BRANCH_EN); /* * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise @@ -7158,12 +7306,11 @@ static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ); /* - * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn BranchEn and - * MTCFreq can be set + * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn and MTCFreq can be set */ if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc)) vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN | - RTIT_CTL_BRANCH_EN | RTIT_CTL_MTC_RANGE); + RTIT_CTL_MTC_RANGE); /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */ if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite)) @@ -7183,7 +7330,7 @@ static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN; /* unmask address range configure area */ - for (i = 0; i < vmx->pt_desc.addr_range; i++) + for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4)); } @@ -7227,6 +7374,11 @@ static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) } } + if (kvm_cpu_cap_has(X86_FEATURE_XFD)) + vmx_set_intercept_for_msr(vcpu, MSR_IA32_XFD_ERR, MSR_TYPE_R, + !guest_cpuid_has(vcpu, X86_FEATURE_XFD)); + + set_cr4_guest_host_mask(vmx); vmx_write_encls_bitmap(vcpu, NULL); @@ -7466,25 +7618,6 @@ void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); } -static int vmx_pre_block(struct kvm_vcpu *vcpu) -{ - if (pi_pre_block(vcpu)) - return 1; - - if (kvm_lapic_hv_timer_in_use(vcpu)) - kvm_lapic_switch_to_sw_timer(vcpu); - - return 0; -} - -static void vmx_post_block(struct kvm_vcpu *vcpu) -{ - if (kvm_x86_ops.set_hv_timer) - kvm_lapic_switch_to_hv_timer(vcpu); - - pi_post_block(vcpu); -} - static void vmx_setup_mce(struct kvm_vcpu *vcpu) { if (vcpu->arch.mcg_cap & MCG_LMCE_P) @@ -7571,12 +7704,16 @@ static void hardware_unsetup(void) static bool vmx_check_apicv_inhibit_reasons(ulong bit) { ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | - BIT(APICV_INHIBIT_REASON_HYPERV); + BIT(APICV_INHIBIT_REASON_ABSENT) | + BIT(APICV_INHIBIT_REASON_HYPERV) | + BIT(APICV_INHIBIT_REASON_BLOCKIRQ); return supported & BIT(bit); } static struct kvm_x86_ops vmx_x86_ops __initdata = { + .name = "kvm_intel", + .hardware_unsetup = hardware_unsetup, .hardware_enable = hardware_enable, @@ -7624,6 +7761,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .tlb_flush_gva = vmx_flush_tlb_gva, .tlb_flush_guest = vmx_flush_tlb_guest, + .vcpu_pre_run = vmx_vcpu_pre_run, .run = vmx_vcpu_run, .handle_exit = vmx_handle_exit, .skip_emulated_instruction = vmx_skip_emulated_instruction, @@ -7652,7 +7790,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .hwapic_isr_update = vmx_hwapic_isr_update, .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt, .sync_pir_to_irr = vmx_sync_pir_to_irr, - .deliver_posted_interrupt = vmx_deliver_posted_interrupt, + .deliver_interrupt = vmx_deliver_interrupt, .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt, .set_tss_addr = vmx_set_tss_addr, @@ -7682,9 +7820,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .cpu_dirty_log_size = PML_ENTITY_NUM, .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging, - .pre_block = vmx_pre_block, - .post_block = vmx_post_block, - .pmu_ops = &intel_pmu_ops, .nested_ops = &vmx_nested_ops, @@ -7713,6 +7848,20 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, }; +static unsigned int vmx_handle_intel_pt_intr(void) +{ + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); + + /* '0' on failure so that the !PT case can use a RET0 static call. */ + if (!kvm_arch_pmi_in_guest(vcpu)) + return 0; + + kvm_make_request(KVM_REQ_PMI, vcpu); + __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, + (unsigned long *)&vcpu->arch.pmu.global_status); + return 1; +} + static __init void vmx_setup_user_return_msrs(void) { @@ -7739,11 +7888,13 @@ static __init void vmx_setup_user_return_msrs(void) kvm_add_user_return_msr(vmx_uret_msrs_list[i]); } +static struct kvm_x86_init_ops vmx_init_ops __initdata; + static __init int hardware_setup(void) { unsigned long host_bndcfgs; struct desc_ptr dt; - int r, ept_lpage_level; + int r; store_idt(&dt); host_idt_base = dt.address; @@ -7840,16 +7991,8 @@ static __init int hardware_setup(void) kvm_mmu_set_ept_masks(enable_ept_ad_bits, cpu_has_vmx_ept_execute_only()); - if (!enable_ept) - ept_lpage_level = 0; - else if (cpu_has_vmx_ept_1g_page()) - ept_lpage_level = PG_LEVEL_1G; - else if (cpu_has_vmx_ept_2m_page()) - ept_lpage_level = PG_LEVEL_2M; - else - ept_lpage_level = PG_LEVEL_4K; kvm_configure_mmu(enable_ept, 0, vmx_get_max_tdp_level(), - ept_lpage_level); + ept_caps_to_lpage_level(vmx_capability.ept)); /* * Only enable PML when hardware supports PML feature, and both EPT @@ -7897,6 +8040,10 @@ static __init int hardware_setup(void) return -EINVAL; if (!enable_ept || !cpu_has_vmx_intel_pt()) pt_mode = PT_MODE_SYSTEM; + if (pt_mode == PT_MODE_HOST_GUEST) + vmx_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr; + else + vmx_init_ops.handle_intel_pt_intr = NULL; setup_default_sgx_lepubkeyhash(); @@ -7925,7 +8072,7 @@ static struct kvm_x86_init_ops vmx_init_ops __initdata = { .disabled_by_bios = vmx_disabled_by_bios, .check_processor_compatibility = vmx_check_processor_compat, .hardware_setup = hardware_setup, - .intel_pt_intr_in_guest = vmx_pt_mode_is_host_guest, + .handle_intel_pt_intr = NULL, .runtime_ops = &vmx_x86_ops, }; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 3f9c854862..9c6bfcd840 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -62,7 +62,7 @@ struct pt_ctx { struct pt_desc { u64 ctl_bitmask; - u32 addr_range; + u32 num_address_ranges; u32 caps[PT_CPUID_REGS_NUM * PT_CPUID_LEAVES]; struct pt_ctx host; struct pt_ctx guest; @@ -141,6 +141,16 @@ struct nested_vmx { */ struct vmcs12 *cached_shadow_vmcs12; + /* + * GPA to HVA cache for accessing vmcs12->vmcs_link_pointer + */ + struct gfn_to_hva_cache shadow_vmcs12_cache; + + /* + * GPA to HVA cache for VMCS12 + */ + struct gfn_to_hva_cache vmcs12_cache; + /* * Indicates if the shadow vmcs or enlightened vmcs must be updated * with the data held by struct vmcs12. @@ -148,6 +158,15 @@ struct nested_vmx { bool need_vmcs12_to_shadow_sync; bool dirty_vmcs12; + /* + * Indicates whether MSR bitmap for L2 needs to be rebuilt due to + * changes in MSR bitmap for L1 or switching to a different L2. Note, + * this flag can only be used reliably in conjunction with a paravirt L1 + * which informs L0 whether any changes to MSR bitmap for L2 were done + * on its side. + */ + bool force_msr_bitmap_recalc; + /* * Indicates lazily loaded guest state has not yet been decached from * vmcs02. @@ -298,6 +317,9 @@ struct vcpu_vmx { /* Posted interrupt descriptor */ struct pi_desc pi_desc; + /* Used if this vCPU is waiting for PI notification wakeup. */ + struct list_head pi_wakeup_list; + /* Support for a guest hypervisor (nested VMX) */ struct nested_vmx nested; @@ -330,7 +352,7 @@ struct vcpu_vmx { struct lbr_desc lbr_desc; /* Save desired MSR intercept (read: pass-through) state */ -#define MAX_POSSIBLE_PASSTHROUGH_MSRS 13 +#define MAX_POSSIBLE_PASSTHROUGH_MSRS 15 struct { DECLARE_BITMAP(read, MAX_POSSIBLE_PASSTHROUGH_MSRS); DECLARE_BITMAP(write, MAX_POSSIBLE_PASSTHROUGH_MSRS); @@ -400,68 +422,33 @@ static inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); -static inline bool vmx_test_msr_bitmap_read(ulong *msr_bitmap, u32 msr) -{ - int f = sizeof(unsigned long); - - if (msr <= 0x1fff) - return test_bit(msr, msr_bitmap + 0x000 / f); - else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) - return test_bit(msr & 0x1fff, msr_bitmap + 0x400 / f); - return true; -} - -static inline bool vmx_test_msr_bitmap_write(ulong *msr_bitmap, u32 msr) -{ - int f = sizeof(unsigned long); - - if (msr <= 0x1fff) - return test_bit(msr, msr_bitmap + 0x800 / f); - else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) - return test_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f); - return true; -} - -static inline void vmx_clear_msr_bitmap_read(ulong *msr_bitmap, u32 msr) -{ - int f = sizeof(unsigned long); - - if (msr <= 0x1fff) - __clear_bit(msr, msr_bitmap + 0x000 / f); - else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) - __clear_bit(msr & 0x1fff, msr_bitmap + 0x400 / f); -} - -static inline void vmx_clear_msr_bitmap_write(ulong *msr_bitmap, u32 msr) -{ - int f = sizeof(unsigned long); - - if (msr <= 0x1fff) - __clear_bit(msr, msr_bitmap + 0x800 / f); - else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) - __clear_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f); -} - -static inline void vmx_set_msr_bitmap_read(ulong *msr_bitmap, u32 msr) -{ - int f = sizeof(unsigned long); - - if (msr <= 0x1fff) - __set_bit(msr, msr_bitmap + 0x000 / f); - else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) - __set_bit(msr & 0x1fff, msr_bitmap + 0x400 / f); -} - -static inline void vmx_set_msr_bitmap_write(ulong *msr_bitmap, u32 msr) -{ - int f = sizeof(unsigned long); - - if (msr <= 0x1fff) - __set_bit(msr, msr_bitmap + 0x800 / f); - else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) - __set_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f); +/* + * Note, early Intel manuals have the write-low and read-high bitmap offsets + * the wrong way round. The bitmaps control MSRs 0x00000000-0x00001fff and + * 0xc0000000-0xc0001fff. The former (low) uses bytes 0-0x3ff for reads and + * 0x800-0xbff for writes. The latter (high) uses 0x400-0x7ff for reads and + * 0xc00-0xfff for writes. MSRs not covered by either of the ranges always + * VM-Exit. + */ +#define __BUILD_VMX_MSR_BITMAP_HELPER(rtype, action, bitop, access, base) \ +static inline rtype vmx_##action##_msr_bitmap_##access(unsigned long *bitmap, \ + u32 msr) \ +{ \ + int f = sizeof(unsigned long); \ + \ + if (msr <= 0x1fff) \ + return bitop##_bit(msr, bitmap + base / f); \ + else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) \ + return bitop##_bit(msr & 0x1fff, bitmap + (base + 0x400) / f); \ + return (rtype)true; \ } +#define BUILD_VMX_MSR_BITMAP_HELPERS(ret_type, action, bitop) \ + __BUILD_VMX_MSR_BITMAP_HELPER(ret_type, action, bitop, read, 0x0) \ + __BUILD_VMX_MSR_BITMAP_HELPER(ret_type, action, bitop, write, 0x800) +BUILD_VMX_MSR_BITMAP_HELPERS(bool, test, test) +BUILD_VMX_MSR_BITMAP_HELPERS(void, clear, __clear) +BUILD_VMX_MSR_BITMAP_HELPERS(void, set, __set) static inline u8 vmx_get_rvi(void) { @@ -498,19 +485,21 @@ BUILD_CONTROLS_SHADOW(pin, PIN_BASED_VM_EXEC_CONTROL) BUILD_CONTROLS_SHADOW(exec, CPU_BASED_VM_EXEC_CONTROL) BUILD_CONTROLS_SHADOW(secondary_exec, SECONDARY_VM_EXEC_CONTROL) -static inline void vmx_register_cache_reset(struct kvm_vcpu *vcpu) -{ - vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP) - | (1 << VCPU_EXREG_RFLAGS) - | (1 << VCPU_EXREG_PDPTR) - | (1 << VCPU_EXREG_SEGMENTS) - | (1 << VCPU_EXREG_CR0) - | (1 << VCPU_EXREG_CR3) - | (1 << VCPU_EXREG_CR4) - | (1 << VCPU_EXREG_EXIT_INFO_1) - | (1 << VCPU_EXREG_EXIT_INFO_2)); - vcpu->arch.regs_dirty = 0; -} +/* + * VMX_REGS_LAZY_LOAD_SET - The set of registers that will be updated in the + * cache on demand. Other registers not listed here are synced to + * the cache immediately after VM-Exit. + */ +#define VMX_REGS_LAZY_LOAD_SET ((1 << VCPU_REGS_RIP) | \ + (1 << VCPU_REGS_RSP) | \ + (1 << VCPU_EXREG_RFLAGS) | \ + (1 << VCPU_EXREG_PDPTR) | \ + (1 << VCPU_EXREG_SEGMENTS) | \ + (1 << VCPU_EXREG_CR0) | \ + (1 << VCPU_EXREG_CR3) | \ + (1 << VCPU_EXREG_CR4) | \ + (1 << VCPU_EXREG_EXIT_INFO_1) | \ + (1 << VCPU_EXREG_EXIT_INFO_2)) static inline struct kvm_vmx *to_kvm_vmx(struct kvm *kvm) { @@ -585,4 +574,9 @@ static inline bool vmx_guest_state_valid(struct kvm_vcpu *vcpu) void dump_vmcs(struct kvm_vcpu *vcpu); +static inline int vmx_get_instr_info_reg2(u32 vmx_instr_info) +{ + return (vmx_instr_info >> 28) & 0xf; +} + #endif /* __KVM_X86_VMX_H */ diff --git a/arch/x86/kvm/vmx/vmx_ops.h b/arch/x86/kvm/vmx/vmx_ops.h index 9e9ef47e98..5e7f412257 100644 --- a/arch/x86/kvm/vmx/vmx_ops.h +++ b/arch/x86/kvm/vmx/vmx_ops.h @@ -71,6 +71,31 @@ static __always_inline unsigned long __vmcs_readl(unsigned long field) { unsigned long value; +#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT + + asm_volatile_goto("1: vmread %[field], %[output]\n\t" + "jna %l[do_fail]\n\t" + + _ASM_EXTABLE(1b, %l[do_exception]) + + : [output] "=r" (value) + : [field] "r" (field) + : "cc" + : do_fail, do_exception); + + return value; + +do_fail: + WARN_ONCE(1, "kvm: vmread failed: field=%lx\n", field); + pr_warn_ratelimited("kvm: vmread failed: field=%lx\n", field); + return 0; + +do_exception: + kvm_spurious_fault(); + return 0; + +#else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ + asm volatile("1: vmread %2, %1\n\t" ".byte 0x3e\n\t" /* branch taken hint */ "ja 3f\n\t" @@ -80,9 +105,11 @@ static __always_inline unsigned long __vmcs_readl(unsigned long field) * @field, and bounce through the trampoline to preserve * volatile registers. */ - "push $0\n\t" + "xorl %k1, %k1\n\t" + "2:\n\t" + "push %1\n\t" "push %2\n\t" - "2:call vmread_error_trampoline\n\t" + "call vmread_error_trampoline\n\t" /* * Unwind the stack. Note, the trampoline zeros out the @@ -93,14 +120,12 @@ static __always_inline unsigned long __vmcs_readl(unsigned long field) "3:\n\t" /* VMREAD faulted. As above, except push '1' for @fault. */ - ".pushsection .fixup, \"ax\"\n\t" - "4: push $1\n\t" - "push %2\n\t" - "jmp 2b\n\t" - ".popsection\n\t" - _ASM_EXTABLE(1b, 4b) - : ASM_CALL_CONSTRAINT, "=r"(value) : "r"(field) : "cc"); + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_ONE_REG, %1) + + : ASM_CALL_CONSTRAINT, "=&r"(value) : "r"(field) : "cc"); return value; + +#endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ } static __always_inline u16 vmcs_read16(unsigned long field) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 61bc54748f..eb4029660b 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -68,7 +68,9 @@ #include #include #include -#include /* Ugh! */ +#include +#include +#include #include #include #include @@ -88,6 +90,8 @@ u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P; EXPORT_SYMBOL_GPL(kvm_mce_cap_supported); +#define ERR_PTR_USR(e) ((void __user *)ERR_PTR(e)) + #define emul_to_vcpu(ctxt) \ ((struct kvm_vcpu *)(ctxt)->vcpu) @@ -116,6 +120,7 @@ static void enter_smm(struct kvm_vcpu *vcpu); static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); static void store_regs(struct kvm_vcpu *vcpu); static int sync_regs(struct kvm_vcpu *vcpu); +static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu); static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); @@ -184,6 +189,11 @@ module_param(force_emulation_prefix, bool, S_IRUGO); int __read_mostly pi_inject_timer = -1; module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR); +/* Enable/disable PMU virtualization */ +bool __read_mostly enable_pmu = true; +EXPORT_SYMBOL_GPL(enable_pmu); +module_param(enable_pmu, bool, 0444); + /* * Restoring the host value for MSRs that are only consumed when running in * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU @@ -208,7 +218,7 @@ static struct kvm_user_return_msrs __percpu *user_return_msrs; #define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ - | XFEATURE_MASK_PKRU) + | XFEATURE_MASK_PKRU | XFEATURE_MASK_XTILE) u64 __read_mostly host_efer; EXPORT_SYMBOL_GPL(host_efer); @@ -293,8 +303,6 @@ u64 __read_mostly host_xcr0; u64 __read_mostly supported_xcr0; EXPORT_SYMBOL_GPL(supported_xcr0); -static struct kmem_cache *x86_fpu_cache; - static struct kmem_cache *x86_emulator_cache; /* @@ -710,6 +718,17 @@ int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) } EXPORT_SYMBOL_GPL(kvm_complete_insn_gp); +static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err) +{ + if (err) { + kvm_inject_gp(vcpu, 0); + return 1; + } + + return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP | + EMULTYPE_COMPLETE_USER_EXIT); +} + void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) { ++vcpu->stat.pf_guest; @@ -790,30 +809,6 @@ bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) } EXPORT_SYMBOL_GPL(kvm_require_dr); -/* - * This function will be used to read from the physical memory of the currently - * running guest. The difference to kvm_vcpu_read_guest_page is that this function - * can read from guest physical or from the guest's guest physical memory. - */ -int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, - gfn_t ngfn, void *data, int offset, int len, - u32 access) -{ - struct x86_exception exception; - gfn_t real_gfn; - gpa_t ngpa; - - ngpa = gfn_to_gpa(ngfn); - real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception); - if (real_gfn == UNMAPPED_GVA) - return -EFAULT; - - real_gfn = gpa_to_gfn(real_gfn); - - return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len); -} -EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu); - static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) { return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); @@ -822,38 +817,50 @@ static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) /* * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise. */ -int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3) +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) { + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; - unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; + gpa_t real_gpa; int i; int ret; u64 pdpte[ARRAY_SIZE(mmu->pdptrs)]; - ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte, - offset * sizeof(u64), sizeof(pdpte), - PFERR_USER_MASK|PFERR_WRITE_MASK); - if (ret < 0) { - ret = 0; - goto out; - } + /* + * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated + * to an L1 GPA. + */ + real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn), + PFERR_USER_MASK | PFERR_WRITE_MASK, NULL); + if (real_gpa == UNMAPPED_GVA) + return 0; + + /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */ + ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, + cr3 & GENMASK(11, 5), sizeof(pdpte)); + if (ret < 0) + return 0; + for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { if ((pdpte[i] & PT_PRESENT_MASK) && (pdpte[i] & pdptr_rsvd_bits(vcpu))) { - ret = 0; - goto out; + return 0; } } - ret = 1; + + /* + * Marking VCPU_EXREG_PDPTR dirty doesn't work for !tdp_enabled. + * Shadow page roots need to be reconstructed instead. + */ + if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs))) + kvm_mmu_free_roots(vcpu, mmu, KVM_MMU_ROOT_CURRENT); memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)); kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); vcpu->arch.pdptrs_from_userspace = false; -out: - - return ret; + return 1; } EXPORT_SYMBOL_GPL(load_pdptrs); @@ -877,7 +884,6 @@ EXPORT_SYMBOL_GPL(kvm_post_set_cr0); int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { unsigned long old_cr0 = kvm_read_cr0(vcpu); - unsigned long pdptr_bits = X86_CR0_CD | X86_CR0_NW | X86_CR0_PG; cr0 |= X86_CR0_ET; @@ -907,11 +913,12 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } #endif if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && - is_pae(vcpu) && ((cr0 ^ old_cr0) & pdptr_bits) && - !load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu))) + is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && + !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) return 1; - if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)) + if (!(cr0 & X86_CR0_PG) && + (is_64_bit_mode(vcpu) || kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))) return 1; static_call(kvm_x86_set_cr0)(vcpu, cr0); @@ -977,6 +984,18 @@ void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state); +static inline u64 kvm_guest_supported_xcr0(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.guest_fpu.fpstate->user_xfeatures; +} + +#ifdef CONFIG_X86_64 +static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu) +{ + return kvm_guest_supported_xcr0(vcpu) & XFEATURE_MASK_USER_DYNAMIC; +} +#endif + static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) { u64 xcr0 = xcr; @@ -994,9 +1013,9 @@ static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) /* * Do not allow the guest to set bits that we do not support * saving. However, xcr0 bit 0 is always set, even if the - * emulated CPU does not support XSAVE (see fx_init). + * emulated CPU does not support XSAVE (see kvm_vcpu_reset()). */ - valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP; + valid_bits = kvm_guest_supported_xcr0(vcpu) | XFEATURE_MASK_FP; if (xcr0 & ~valid_bits) return 1; @@ -1010,6 +1029,11 @@ static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512) return 1; } + + if ((xcr0 & XFEATURE_MASK_XTILE) && + ((xcr0 & XFEATURE_MASK_XTILE) != XFEATURE_MASK_XTILE)) + return 1; + vcpu->arch.xcr0 = xcr0; if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND) @@ -1043,17 +1067,34 @@ EXPORT_SYMBOL_GPL(kvm_is_valid_cr4); void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) { - if (((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) || - (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) + /* + * If any role bit is changed, the MMU needs to be reset. + * + * If CR4.PCIDE is changed 1 -> 0, the guest TLB must be flushed. + * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB + * according to the SDM; however, stale prev_roots could be reused + * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we + * free them all. KVM_REQ_MMU_RELOAD is fit for the both cases; it + * is slow, but changing CR4.PCIDE is a rare case. + * + * If CR4.PGE is changed, the guest TLB must be flushed. + * + * Note: resetting MMU is a superset of KVM_REQ_MMU_RELOAD and + * KVM_REQ_MMU_RELOAD is a superset of KVM_REQ_TLB_FLUSH_GUEST, hence + * the usage of "else if". + */ + if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) kvm_mmu_reset_context(vcpu); + else if ((cr4 ^ old_cr4) & X86_CR4_PCIDE) + kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); + else if ((cr4 ^ old_cr4) & X86_CR4_PGE) + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); } EXPORT_SYMBOL_GPL(kvm_post_set_cr4); int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { unsigned long old_cr4 = kvm_read_cr4(vcpu); - unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE | - X86_CR4_SMEP; if (!kvm_is_valid_cr4(vcpu, cr4)) return 1; @@ -1064,9 +1105,8 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) if ((cr4 ^ old_cr4) & X86_CR4_LA57) return 1; } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) - && ((cr4 ^ old_cr4) & pdptr_bits) - && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, - kvm_read_cr3(vcpu))) + && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS) + && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) return 1; if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { @@ -1114,6 +1154,14 @@ static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } + /* + * If PCID is disabled, there is no need to free prev_roots even if the + * PCIDs for them are also 0, because MOV to CR3 always flushes the TLB + * with PCIDE=0. + */ + if (!kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)) + return; + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid) roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); @@ -1147,14 +1195,15 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (kvm_vcpu_is_illegal_gpa(vcpu, cr3)) return 1; - if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) + if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) return 1; if (cr3 != kvm_read_cr3(vcpu)) kvm_mmu_new_pgd(vcpu, cr3); vcpu->arch.cr3 = cr3; - kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); + kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); + /* Do not call post_set_cr3, we do not get here for confidential guests. */ handle_tlb_flush: /* @@ -1352,6 +1401,7 @@ static const u32 msrs_to_save_all[] = { MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5, MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2, MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5, + MSR_IA32_XFD, MSR_IA32_XFD_ERR, }; static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)]; @@ -1394,6 +1444,7 @@ static const u32 emulated_msrs_all[] = { MSR_PLATFORM_INFO, MSR_MISC_FEATURES_ENABLES, MSR_AMD64_VIRT_SPEC_CTRL, + MSR_AMD64_TSC_RATIO, MSR_IA32_POWER_CTL, MSR_IA32_UCODE_REV, @@ -1807,22 +1858,36 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data) } EXPORT_SYMBOL_GPL(kvm_set_msr); -static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu) +static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu) { - int err = vcpu->run->msr.error; - if (!err) { + if (!vcpu->run->msr.error) { kvm_rax_write(vcpu, (u32)vcpu->run->msr.data); kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32); } - - return static_call(kvm_x86_complete_emulated_msr)(vcpu, err); } -static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu) +static int complete_emulated_msr_access(struct kvm_vcpu *vcpu) +{ + return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error); +} + +static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu) +{ + complete_userspace_rdmsr(vcpu); + return complete_emulated_msr_access(vcpu); +} + +static int complete_fast_msr_access(struct kvm_vcpu *vcpu) { return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error); } +static int complete_fast_rdmsr(struct kvm_vcpu *vcpu) +{ + complete_userspace_rdmsr(vcpu); + return complete_fast_msr_access(vcpu); +} + static u64 kvm_msr_reason(int r) { switch (r) { @@ -1857,18 +1922,6 @@ static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index, return 1; } -static int kvm_get_msr_user_space(struct kvm_vcpu *vcpu, u32 index, int r) -{ - return kvm_msr_user_space(vcpu, index, KVM_EXIT_X86_RDMSR, 0, - complete_emulated_rdmsr, r); -} - -static int kvm_set_msr_user_space(struct kvm_vcpu *vcpu, u32 index, u64 data, int r) -{ - return kvm_msr_user_space(vcpu, index, KVM_EXIT_X86_WRMSR, data, - complete_emulated_wrmsr, r); -} - int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) { u32 ecx = kvm_rcx_read(vcpu); @@ -1877,18 +1930,16 @@ int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) r = kvm_get_msr(vcpu, ecx, &data); - /* MSR read failed? See if we should ask user space */ - if (r && kvm_get_msr_user_space(vcpu, ecx, r)) { - /* Bounce to user space */ - return 0; - } - if (!r) { trace_kvm_msr_read(ecx, data); kvm_rax_write(vcpu, data & -1u); kvm_rdx_write(vcpu, (data >> 32) & -1u); } else { + /* MSR read failed? See if we should ask user space */ + if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0, + complete_fast_rdmsr, r)) + return 0; trace_kvm_msr_read_ex(ecx); } @@ -1904,19 +1955,18 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) r = kvm_set_msr(vcpu, ecx, data); - /* MSR write failed? See if we should ask user space */ - if (r && kvm_set_msr_user_space(vcpu, ecx, data, r)) - /* Bounce to user space */ - return 0; - - /* Signal all other negative errors to userspace */ - if (r < 0) - return r; - - if (!r) + if (!r) { trace_kvm_msr_write(ecx, data); - else + } else { + /* MSR write failed? See if we should ask user space */ + if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data, + complete_fast_msr_access, r)) + return 0; + /* Signal all other negative errors to userspace */ + if (r < 0) + return r; trace_kvm_msr_write_ex(ecx, data); + } return static_call(kvm_x86_complete_emulated_msr)(vcpu, r); } @@ -2111,7 +2161,7 @@ static s64 get_kvmclock_base_ns(void) } #endif -void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs) +static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs) { int version; int r; @@ -2313,10 +2363,12 @@ static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) return tsc; } +#ifdef CONFIG_X86_64 static inline int gtod_is_based_on_tsc(int mode) { return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK; } +#endif static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) { @@ -2467,13 +2519,64 @@ static inline bool kvm_check_tsc_unstable(void) return check_tsc_unstable(); } +/* + * Infers attempts to synchronize the guest's tsc from host writes. Sets the + * offset for the vcpu and tracks the TSC matching generation that the vcpu + * participates in. + */ +static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, + u64 ns, bool matched) +{ + struct kvm *kvm = vcpu->kvm; + + lockdep_assert_held(&kvm->arch.tsc_write_lock); + + /* + * We also track th most recent recorded KHZ, write and time to + * allow the matching interval to be extended at each write. + */ + kvm->arch.last_tsc_nsec = ns; + kvm->arch.last_tsc_write = tsc; + kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; + kvm->arch.last_tsc_offset = offset; + + vcpu->arch.last_guest_tsc = tsc; + + kvm_vcpu_write_tsc_offset(vcpu, offset); + + if (!matched) { + /* + * We split periods of matched TSC writes into generations. + * For each generation, we track the original measured + * nanosecond time, offset, and write, so if TSCs are in + * sync, we can match exact offset, and if not, we can match + * exact software computation in compute_guest_tsc() + * + * These values are tracked in kvm->arch.cur_xxx variables. + */ + kvm->arch.cur_tsc_generation++; + kvm->arch.cur_tsc_nsec = ns; + kvm->arch.cur_tsc_write = tsc; + kvm->arch.cur_tsc_offset = offset; + kvm->arch.nr_vcpus_matched_tsc = 0; + } else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) { + kvm->arch.nr_vcpus_matched_tsc++; + } + + /* Keep track of which generation this VCPU has synchronized to */ + vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation; + vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec; + vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write; + + kvm_track_tsc_matching(vcpu); +} + static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data) { struct kvm *kvm = vcpu->kvm; u64 offset, ns, elapsed; unsigned long flags; - bool matched; - bool already_matched; + bool matched = false; bool synchronizing = false; raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); @@ -2519,51 +2622,10 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data) offset = kvm_compute_l1_tsc_offset(vcpu, data); } matched = true; - already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation); - } else { - /* - * We split periods of matched TSC writes into generations. - * For each generation, we track the original measured - * nanosecond time, offset, and write, so if TSCs are in - * sync, we can match exact offset, and if not, we can match - * exact software computation in compute_guest_tsc() - * - * These values are tracked in kvm->arch.cur_xxx variables. - */ - kvm->arch.cur_tsc_generation++; - kvm->arch.cur_tsc_nsec = ns; - kvm->arch.cur_tsc_write = data; - kvm->arch.cur_tsc_offset = offset; - matched = false; } - /* - * We also track th most recent recorded KHZ, write and time to - * allow the matching interval to be extended at each write. - */ - kvm->arch.last_tsc_nsec = ns; - kvm->arch.last_tsc_write = data; - kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; - - vcpu->arch.last_guest_tsc = data; - - /* Keep track of which generation this VCPU has synchronized to */ - vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation; - vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec; - vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write; - - kvm_vcpu_write_tsc_offset(vcpu, offset); + __kvm_synchronize_tsc(vcpu, offset, data, ns, matched); raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); - - raw_spin_lock_irqsave(&kvm->arch.pvclock_gtod_sync_lock, flags); - if (!matched) { - kvm->arch.nr_vcpus_matched_tsc = 0; - } else if (!already_matched) { - kvm->arch.nr_vcpus_matched_tsc++; - } - - kvm_track_tsc_matching(vcpu); - raw_spin_unlock_irqrestore(&kvm->arch.pvclock_gtod_sync_lock, flags); } static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, @@ -2751,6 +2813,7 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm) int vclock_mode; bool host_tsc_clocksource, vcpus_matched; + lockdep_assert_held(&kvm->arch.tsc_write_lock); vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == atomic_read(&kvm->online_vcpus)); @@ -2775,68 +2838,101 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm) #endif } -void kvm_make_mclock_inprogress_request(struct kvm *kvm) +static void kvm_make_mclock_inprogress_request(struct kvm *kvm) { kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS); } -static void kvm_gen_update_masterclock(struct kvm *kvm) +static void __kvm_start_pvclock_update(struct kvm *kvm) { -#ifdef CONFIG_X86_64 - int i; - struct kvm_vcpu *vcpu; - struct kvm_arch *ka = &kvm->arch; - unsigned long flags; - - kvm_hv_invalidate_tsc_page(kvm); + raw_spin_lock_irq(&kvm->arch.tsc_write_lock); + write_seqcount_begin(&kvm->arch.pvclock_sc); +} +static void kvm_start_pvclock_update(struct kvm *kvm) +{ kvm_make_mclock_inprogress_request(kvm); /* no guest entries from this point */ - raw_spin_lock_irqsave(&ka->pvclock_gtod_sync_lock, flags); - pvclock_update_vm_gtod_copy(kvm); - raw_spin_unlock_irqrestore(&ka->pvclock_gtod_sync_lock, flags); + __kvm_start_pvclock_update(kvm); +} +static void kvm_end_pvclock_update(struct kvm *kvm) +{ + struct kvm_arch *ka = &kvm->arch; + struct kvm_vcpu *vcpu; + unsigned long i; + + write_seqcount_end(&ka->pvclock_sc); + raw_spin_unlock_irq(&ka->tsc_write_lock); kvm_for_each_vcpu(i, vcpu, kvm) kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); /* guest entries allowed */ kvm_for_each_vcpu(i, vcpu, kvm) kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu); -#endif } -u64 get_kvmclock_ns(struct kvm *kvm) +static void kvm_update_masterclock(struct kvm *kvm) +{ + kvm_hv_invalidate_tsc_page(kvm); + kvm_start_pvclock_update(kvm); + pvclock_update_vm_gtod_copy(kvm); + kvm_end_pvclock_update(kvm); +} + +/* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */ +static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) { struct kvm_arch *ka = &kvm->arch; struct pvclock_vcpu_time_info hv_clock; - unsigned long flags; - u64 ret; - - raw_spin_lock_irqsave(&ka->pvclock_gtod_sync_lock, flags); - if (!ka->use_master_clock) { - raw_spin_unlock_irqrestore(&ka->pvclock_gtod_sync_lock, flags); - return get_kvmclock_base_ns() + ka->kvmclock_offset; - } - - hv_clock.tsc_timestamp = ka->master_cycle_now; - hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - raw_spin_unlock_irqrestore(&ka->pvclock_gtod_sync_lock, flags); /* both __this_cpu_read() and rdtsc() should be on the same cpu */ get_cpu(); - if (__this_cpu_read(cpu_tsc_khz)) { + data->flags = 0; + if (ka->use_master_clock && __this_cpu_read(cpu_tsc_khz)) { +#ifdef CONFIG_X86_64 + struct timespec64 ts; + + if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) { + data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; + data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC; + } else +#endif + data->host_tsc = rdtsc(); + + data->flags |= KVM_CLOCK_TSC_STABLE; + hv_clock.tsc_timestamp = ka->master_cycle_now; + hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL, &hv_clock.tsc_shift, &hv_clock.tsc_to_system_mul); - ret = __pvclock_read_cycles(&hv_clock, rdtsc()); - } else - ret = get_kvmclock_base_ns() + ka->kvmclock_offset; + data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); + } else { + data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; + } put_cpu(); +} - return ret; +static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) +{ + struct kvm_arch *ka = &kvm->arch; + unsigned seq; + + do { + seq = read_seqcount_begin(&ka->pvclock_sc); + __get_kvmclock(kvm, data); + } while (read_seqcount_retry(&ka->pvclock_sc, seq)); +} + +u64 get_kvmclock_ns(struct kvm *kvm) +{ + struct kvm_clock_data data; + + get_kvmclock(kvm, &data); + return data.clock; } static void kvm_setup_pvclock_page(struct kvm_vcpu *v, @@ -2901,6 +2997,7 @@ static void kvm_setup_pvclock_page(struct kvm_vcpu *v, static int kvm_guest_time_update(struct kvm_vcpu *v) { unsigned long flags, tgt_tsc_khz; + unsigned seq; struct kvm_vcpu_arch *vcpu = &v->arch; struct kvm_arch *ka = &v->kvm->arch; s64 kernel_ns; @@ -2915,13 +3012,14 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) * If the host uses TSC clock, then passthrough TSC as stable * to the guest. */ - raw_spin_lock_irqsave(&ka->pvclock_gtod_sync_lock, flags); - use_master_clock = ka->use_master_clock; - if (use_master_clock) { - host_tsc = ka->master_cycle_now; - kernel_ns = ka->master_kernel_ns; - } - raw_spin_unlock_irqrestore(&ka->pvclock_gtod_sync_lock, flags); + do { + seq = read_seqcount_begin(&ka->pvclock_sc); + use_master_clock = ka->use_master_clock; + if (use_master_clock) { + host_tsc = ka->master_cycle_now; + kernel_ns = ka->master_kernel_ns; + } + } while (read_seqcount_retry(&ka->pvclock_sc, seq)); /* Keep irq disabled to prevent changes to the clock */ local_irq_save(flags); @@ -3012,7 +3110,7 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) static void kvmclock_update_fn(struct work_struct *work) { - int i; + unsigned long i; struct delayed_work *dwork = to_delayed_work(work); struct kvm_arch *ka = container_of(dwork, struct kvm_arch, kvmclock_update_work); @@ -3192,15 +3290,14 @@ static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu) ++vcpu->stat.tlb_flush; if (!tdp_enabled) { - /* + /* * A TLB flush on behalf of the guest is equivalent to * INVPCID(all), toggling CR4.PGE, etc., which requires - * a forced sync of the shadow page tables. Unload the - * entire MMU here and the subsequent load will sync the - * shadow page tables, and also flush the TLB. + * a forced sync of the shadow page tables. Ensure all the + * roots are synced and the guest TLB in hardware is clean. */ - kvm_mmu_unload(vcpu); - return; + kvm_mmu_sync_roots(vcpu); + kvm_mmu_sync_prev_roots(vcpu); } static_call(kvm_x86_tlb_flush_guest)(vcpu); @@ -3531,7 +3628,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) return 1; - if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8))) + if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8))) return 1; break; @@ -3617,6 +3714,28 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 1; vcpu->arch.msr_misc_features_enables = data; break; +#ifdef CONFIG_X86_64 + case MSR_IA32_XFD: + if (!msr_info->host_initiated && + !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) + return 1; + + if (data & ~kvm_guest_supported_xfd(vcpu)) + return 1; + + fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data); + break; + case MSR_IA32_XFD_ERR: + if (!msr_info->host_initiated && + !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) + return 1; + + if (data & ~kvm_guest_supported_xfd(vcpu)) + return 1; + + vcpu->arch.guest_fpu.xfd_err = data; + break; +#endif default: if (kvm_pmu_is_valid_msr(vcpu, msr)) return kvm_pmu_set_msr(vcpu, msr_info); @@ -3937,6 +4056,22 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_K7_HWCR: msr_info->data = vcpu->arch.msr_hwcr; break; +#ifdef CONFIG_X86_64 + case MSR_IA32_XFD: + if (!msr_info->host_initiated && + !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) + return 1; + + msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd; + break; + case MSR_IA32_XFD_ERR: + if (!msr_info->host_initiated && + !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) + return 1; + + msr_info->data = vcpu->arch.guest_fpu.xfd_err; + break; +#endif default: if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) return kvm_pmu_get_msr(vcpu, msr_info); @@ -4105,8 +4240,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_SGX_ATTRIBUTE: #endif case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: + case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM: case KVM_CAP_SREGS2: case KVM_CAP_EXIT_ON_EMULATION_FAILURE: + case KVM_CAP_VCPU_ATTRIBUTES: + case KVM_CAP_SYS_ATTRIBUTES: + case KVM_CAP_ENABLE_CAP: r = 1; break; case KVM_CAP_EXIT_HYPERCALL: @@ -4118,7 +4257,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_XEN_HVM: r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR | KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL | - KVM_XEN_HVM_CONFIG_SHARED_INFO; + KVM_XEN_HVM_CONFIG_SHARED_INFO | + KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL; if (sched_info_on()) r |= KVM_XEN_HVM_CONFIG_RUNSTATE; break; @@ -4127,7 +4267,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = KVM_SYNC_X86_VALID_FIELDS; break; case KVM_CAP_ADJUST_CLOCK: - r = KVM_CLOCK_TSC_STABLE; + r = KVM_CLOCK_VALID_FLAGS; break; case KVM_CAP_X86_DISABLE_EXITS: r |= KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE | @@ -4150,13 +4290,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = !static_call(kvm_x86_cpu_has_accelerated_tpr)(); break; case KVM_CAP_NR_VCPUS: - r = KVM_SOFT_MAX_VCPUS; + r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS); break; case KVM_CAP_MAX_VCPUS: r = KVM_MAX_VCPUS; break; case KVM_CAP_MAX_VCPU_ID: - r = KVM_MAX_VCPU_ID; + r = KVM_MAX_VCPU_IDS; break; case KVM_CAP_PV_MMU: /* obsolete */ r = 0; @@ -4196,11 +4336,61 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) else r = 0; break; + case KVM_CAP_XSAVE2: { + u64 guest_perm = xstate_get_guest_group_perm(); + + r = xstate_required_size(supported_xcr0 & guest_perm, false); + if (r < sizeof(struct kvm_xsave)) + r = sizeof(struct kvm_xsave); + break; + } default: break; } return r; +} +static inline void __user *kvm_get_attr_addr(struct kvm_device_attr *attr) +{ + void __user *uaddr = (void __user*)(unsigned long)attr->addr; + + if ((u64)(unsigned long)uaddr != attr->addr) + return ERR_PTR_USR(-EFAULT); + return uaddr; +} + +static int kvm_x86_dev_get_attr(struct kvm_device_attr *attr) +{ + u64 __user *uaddr = kvm_get_attr_addr(attr); + + if (attr->group) + return -ENXIO; + + if (IS_ERR(uaddr)) + return PTR_ERR(uaddr); + + switch (attr->attr) { + case KVM_X86_XCOMP_GUEST_SUPP: + if (put_user(supported_xcr0, uaddr)) + return -EFAULT; + return 0; + default: + return -ENXIO; + break; + } +} + +static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr) +{ + if (attr->group) + return -ENXIO; + + switch (attr->attr) { + case KVM_X86_XCOMP_GUEST_SUPP: + return 0; + default: + return -ENXIO; + } } long kvm_arch_dev_ioctl(struct file *filp, @@ -4291,6 +4481,22 @@ long kvm_arch_dev_ioctl(struct file *filp, case KVM_GET_SUPPORTED_HV_CPUID: r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp); break; + case KVM_GET_DEVICE_ATTR: { + struct kvm_device_attr attr; + r = -EFAULT; + if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) + break; + r = kvm_x86_dev_get_attr(&attr); + break; + } + case KVM_HAS_DEVICE_ATTR: { + struct kvm_device_attr attr; + r = -EFAULT; + if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) + break; + r = kvm_x86_dev_has_attr(&attr); + break; + } default: r = -EINVAL; break; @@ -4789,144 +4995,37 @@ static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, return 0; } -#define XSTATE_COMPACTION_ENABLED (1ULL << 63) - -static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu) -{ - struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave; - u64 xstate_bv = xsave->header.xfeatures; - u64 valid; - - /* - * Copy legacy XSAVE area, to avoid complications with CPUID - * leaves 0 and 1 in the loop below. - */ - memcpy(dest, xsave, XSAVE_HDR_OFFSET); - - /* Set XSTATE_BV */ - xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE; - *(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv; - - /* - * Copy each region from the possibly compacted offset to the - * non-compacted offset. - */ - valid = xstate_bv & ~XFEATURE_MASK_FPSSE; - while (valid) { - u32 size, offset, ecx, edx; - u64 xfeature_mask = valid & -valid; - int xfeature_nr = fls64(xfeature_mask) - 1; - void *src; - - cpuid_count(XSTATE_CPUID, xfeature_nr, - &size, &offset, &ecx, &edx); - - if (xfeature_nr == XFEATURE_PKRU) { - memcpy(dest + offset, &vcpu->arch.pkru, - sizeof(vcpu->arch.pkru)); - } else { - src = get_xsave_addr(xsave, xfeature_nr); - if (src) - memcpy(dest + offset, src, size); - } - - valid -= xfeature_mask; - } -} - -static void load_xsave(struct kvm_vcpu *vcpu, u8 *src) -{ - struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave; - u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET); - u64 valid; - - /* - * Copy legacy XSAVE area, to avoid complications with CPUID - * leaves 0 and 1 in the loop below. - */ - memcpy(xsave, src, XSAVE_HDR_OFFSET); - - /* Set XSTATE_BV and possibly XCOMP_BV. */ - xsave->header.xfeatures = xstate_bv; - if (boot_cpu_has(X86_FEATURE_XSAVES)) - xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED; - - /* - * Copy each region from the non-compacted offset to the - * possibly compacted offset. - */ - valid = xstate_bv & ~XFEATURE_MASK_FPSSE; - while (valid) { - u32 size, offset, ecx, edx; - u64 xfeature_mask = valid & -valid; - int xfeature_nr = fls64(xfeature_mask) - 1; - - cpuid_count(XSTATE_CPUID, xfeature_nr, - &size, &offset, &ecx, &edx); - - if (xfeature_nr == XFEATURE_PKRU) { - memcpy(&vcpu->arch.pkru, src + offset, - sizeof(vcpu->arch.pkru)); - } else { - void *dest = get_xsave_addr(xsave, xfeature_nr); - - if (dest) - memcpy(dest, src + offset, size); - } - - valid -= xfeature_mask; - } -} - static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, struct kvm_xsave *guest_xsave) { - if (!vcpu->arch.guest_fpu) + if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) return; - if (boot_cpu_has(X86_FEATURE_XSAVE)) { - memset(guest_xsave, 0, sizeof(struct kvm_xsave)); - fill_xsave((u8 *) guest_xsave->region, vcpu); - } else { - memcpy(guest_xsave->region, - &vcpu->arch.guest_fpu->state.fxsave, - sizeof(struct fxregs_state)); - *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] = - XFEATURE_MASK_FPSSE; - } + fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, + guest_xsave->region, + sizeof(guest_xsave->region), + vcpu->arch.pkru); } -#define XSAVE_MXCSR_OFFSET 24 +static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu, + u8 *state, unsigned int size) +{ + if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) + return; + + fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, + state, size, vcpu->arch.pkru); +} static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, struct kvm_xsave *guest_xsave) { - u64 xstate_bv; - u32 mxcsr; - - if (!vcpu->arch.guest_fpu) + if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) return 0; - xstate_bv = *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)]; - mxcsr = *(u32 *)&guest_xsave->region[XSAVE_MXCSR_OFFSET / sizeof(u32)]; - - if (boot_cpu_has(X86_FEATURE_XSAVE)) { - /* - * Here we allow setting states that are not present in - * CPUID leaf 0xD, index 0, EDX:EAX. This is for compatibility - * with old userspace. - */ - if (xstate_bv & ~supported_xcr0 || mxcsr & ~mxcsr_feature_mask) - return -EINVAL; - load_xsave(vcpu, (u8 *)guest_xsave->region); - } else { - if (xstate_bv & ~XFEATURE_MASK_FPSSE || - mxcsr & ~mxcsr_feature_mask) - return -EINVAL; - memcpy(&vcpu->arch.guest_fpu->state.fxsave, - guest_xsave->region, sizeof(struct fxregs_state)); - } - return 0; + return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu, + guest_xsave->region, + supported_xcr0, &vcpu->arch.pkru); } static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu, @@ -4981,6 +5080,115 @@ static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) return 0; } +static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int r; + + switch (attr->attr) { + case KVM_VCPU_TSC_OFFSET: + r = 0; + break; + default: + r = -ENXIO; + } + + return r; +} + +static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + u64 __user *uaddr = kvm_get_attr_addr(attr); + int r; + + if (IS_ERR(uaddr)) + return PTR_ERR(uaddr); + + switch (attr->attr) { + case KVM_VCPU_TSC_OFFSET: + r = -EFAULT; + if (put_user(vcpu->arch.l1_tsc_offset, uaddr)) + break; + r = 0; + break; + default: + r = -ENXIO; + } + + return r; +} + +static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + u64 __user *uaddr = kvm_get_attr_addr(attr); + struct kvm *kvm = vcpu->kvm; + int r; + + if (IS_ERR(uaddr)) + return PTR_ERR(uaddr); + + switch (attr->attr) { + case KVM_VCPU_TSC_OFFSET: { + u64 offset, tsc, ns; + unsigned long flags; + bool matched; + + r = -EFAULT; + if (get_user(offset, uaddr)) + break; + + raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); + + matched = (vcpu->arch.virtual_tsc_khz && + kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz && + kvm->arch.last_tsc_offset == offset); + + tsc = kvm_scale_tsc(vcpu, rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset; + ns = get_kvmclock_base_ns(); + + __kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched); + raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); + + r = 0; + break; + } + default: + r = -ENXIO; + } + + return r; +} + +static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu, + unsigned int ioctl, + void __user *argp) +{ + struct kvm_device_attr attr; + int r; + + if (copy_from_user(&attr, argp, sizeof(attr))) + return -EFAULT; + + if (attr.group != KVM_VCPU_TSC_CTRL) + return -ENXIO; + + switch (ioctl) { + case KVM_HAS_DEVICE_ATTR: + r = kvm_arch_tsc_has_attr(vcpu, &attr); + break; + case KVM_GET_DEVICE_ATTR: + r = kvm_arch_tsc_get_attr(vcpu, &attr); + break; + case KVM_SET_DEVICE_ATTR: + r = kvm_arch_tsc_set_attr(vcpu, &attr); + break; + } + + return r; +} + static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, struct kvm_enable_cap *cap) { @@ -5243,6 +5451,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; } case KVM_GET_XSAVE: { + r = -EINVAL; + if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave)) + break; + u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT); r = -ENOMEM; if (!u.xsave) @@ -5257,7 +5469,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; } case KVM_SET_XSAVE: { - u.xsave = memdup_user(argp, sizeof(*u.xsave)); + int size = vcpu->arch.guest_fpu.uabi_size; + + u.xsave = memdup_user(argp, size); if (IS_ERR(u.xsave)) { r = PTR_ERR(u.xsave); goto out_nofree; @@ -5266,6 +5480,25 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave); break; } + + case KVM_GET_XSAVE2: { + int size = vcpu->arch.guest_fpu.uabi_size; + + u.xsave = kzalloc(size, GFP_KERNEL_ACCOUNT); + r = -ENOMEM; + if (!u.xsave) + break; + + kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size); + + r = -EFAULT; + if (copy_to_user(argp, u.xsave, size)) + break; + + r = 0; + break; + } + case KVM_GET_XCRS: { u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT); r = -ENOMEM; @@ -5435,6 +5668,11 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = __set_sregs2(vcpu, u.sregs2); break; } + case KVM_HAS_DEVICE_ATTR: + case KVM_GET_DEVICE_ATTR: + case KVM_SET_DEVICE_ATTR: + r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp); + break; default: r = -EINVAL; } @@ -5625,7 +5863,7 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) * VM-Exit. */ struct kvm_vcpu *vcpu; - int i; + unsigned long i; kvm_for_each_vcpu(i, vcpu, kvm) kvm_vcpu_kick(vcpu); @@ -5673,6 +5911,7 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, smp_wmb(); kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT; kvm->arch.nr_reserved_ioapic_pins = cap->args[0]; + kvm_request_apicv_update(kvm, true, APICV_INHIBIT_REASON_ABSENT); r = 0; split_irqchip_unlock: mutex_unlock(&kvm->lock); @@ -5754,6 +5993,12 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, if (kvm_x86_ops.vm_copy_enc_context_from) r = kvm_x86_ops.vm_copy_enc_context_from(kvm, cap->args[0]); return r; + case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM: + r = -EINVAL; + if (kvm_x86_ops.vm_move_enc_context_from) + r = kvm_x86_ops.vm_move_enc_context_from( + kvm, cap->args[0]); + return r; case KVM_CAP_EXIT_HYPERCALL: if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) { r = -EINVAL; @@ -5887,7 +6132,8 @@ static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, void __user *argp) static int kvm_arch_suspend_notifier(struct kvm *kvm) { struct kvm_vcpu *vcpu; - int i, ret = 0; + unsigned long i; + int ret = 0; mutex_lock(&kvm->lock); kvm_for_each_vcpu(i, vcpu, kvm) { @@ -5918,6 +6164,63 @@ int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state) } #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */ +static int kvm_vm_ioctl_get_clock(struct kvm *kvm, void __user *argp) +{ + struct kvm_clock_data data = { 0 }; + + get_kvmclock(kvm, &data); + if (copy_to_user(argp, &data, sizeof(data))) + return -EFAULT; + + return 0; +} + +static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp) +{ + struct kvm_arch *ka = &kvm->arch; + struct kvm_clock_data data; + u64 now_raw_ns; + + if (copy_from_user(&data, argp, sizeof(data))) + return -EFAULT; + + /* + * Only KVM_CLOCK_REALTIME is used, but allow passing the + * result of KVM_GET_CLOCK back to KVM_SET_CLOCK. + */ + if (data.flags & ~KVM_CLOCK_VALID_FLAGS) + return -EINVAL; + + kvm_hv_invalidate_tsc_page(kvm); + kvm_start_pvclock_update(kvm); + pvclock_update_vm_gtod_copy(kvm); + + /* + * This pairs with kvm_guest_time_update(): when masterclock is + * in use, we use master_kernel_ns + kvmclock_offset to set + * unsigned 'system_time' so if we use get_kvmclock_ns() (which + * is slightly ahead) here we risk going negative on unsigned + * 'system_time' when 'data.clock' is very small. + */ + if (data.flags & KVM_CLOCK_REALTIME) { + u64 now_real_ns = ktime_get_real_ns(); + + /* + * Avoid stepping the kvmclock backwards. + */ + if (now_real_ns > data.realtime) + data.clock += now_real_ns - data.realtime; + } + + if (ka->use_master_clock) + now_raw_ns = ka->master_kernel_ns; + else + now_raw_ns = get_kvmclock_base_ns(); + ka->kvmclock_offset = data.clock - now_raw_ns; + kvm_end_pvclock_update(kvm); + return 0; +} + long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -5990,6 +6293,7 @@ long kvm_arch_vm_ioctl(struct file *filp, /* Write kvm->irq_routing before enabling irqchip_in_kernel. */ smp_wmb(); kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL; + kvm_request_apicv_update(kvm, true, APICV_INHIBIT_REASON_ABSENT); create_irqchip_unlock: mutex_unlock(&kvm->lock); break; @@ -6161,60 +6465,12 @@ long kvm_arch_vm_ioctl(struct file *filp, break; } #endif - case KVM_SET_CLOCK: { - struct kvm_arch *ka = &kvm->arch; - struct kvm_clock_data user_ns; - u64 now_ns; - - r = -EFAULT; - if (copy_from_user(&user_ns, argp, sizeof(user_ns))) - goto out; - - r = -EINVAL; - if (user_ns.flags) - goto out; - - r = 0; - /* - * TODO: userspace has to take care of races with VCPU_RUN, so - * kvm_gen_update_masterclock() can be cut down to locked - * pvclock_update_vm_gtod_copy(). - */ - kvm_gen_update_masterclock(kvm); - - /* - * This pairs with kvm_guest_time_update(): when masterclock is - * in use, we use master_kernel_ns + kvmclock_offset to set - * unsigned 'system_time' so if we use get_kvmclock_ns() (which - * is slightly ahead) here we risk going negative on unsigned - * 'system_time' when 'user_ns.clock' is very small. - */ - raw_spin_lock_irq(&ka->pvclock_gtod_sync_lock); - if (kvm->arch.use_master_clock) - now_ns = ka->master_kernel_ns; - else - now_ns = get_kvmclock_base_ns(); - ka->kvmclock_offset = user_ns.clock - now_ns; - raw_spin_unlock_irq(&ka->pvclock_gtod_sync_lock); - - kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE); + case KVM_SET_CLOCK: + r = kvm_vm_ioctl_set_clock(kvm, argp); break; - } - case KVM_GET_CLOCK: { - struct kvm_clock_data user_ns; - u64 now_ns; - - now_ns = get_kvmclock_ns(kvm); - user_ns.clock = now_ns; - user_ns.flags = kvm->arch.use_master_clock ? KVM_CLOCK_TSC_STABLE : 0; - memset(&user_ns.pad, 0, sizeof(user_ns.pad)); - - r = -EFAULT; - if (copy_to_user(argp, &user_ns, sizeof(user_ns))) - goto out; - r = 0; + case KVM_GET_CLOCK: + r = kvm_vm_ioctl_get_clock(kvm, argp); break; - } case KVM_MEMORY_ENCRYPT_OP: { r = -ENOTTY; if (kvm_x86_ops.mem_enc_op) @@ -6337,6 +6593,11 @@ static void kvm_init_msr_list(void) min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp)) continue; break; + case MSR_IA32_XFD: + case MSR_IA32_XFD_ERR: + if (!kvm_cpu_cap_has(X86_FEATURE_XFD)) + continue; + break; default: break; } @@ -6420,13 +6681,14 @@ void kvm_get_segment(struct kvm_vcpu *vcpu, gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, struct x86_exception *exception) { + struct kvm_mmu *mmu = vcpu->arch.mmu; gpa_t t_gpa; BUG_ON(!mmu_is_nested(vcpu)); /* NPT walks are always user-walks */ access |= PFERR_USER_MASK; - t_gpa = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception); + t_gpa = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception); return t_gpa; } @@ -6434,25 +6696,31 @@ gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception) { + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; + u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); + return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); } EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_read); gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception) { + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; + u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_FETCH_MASK; - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); + return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); } gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception) { + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; + u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_WRITE_MASK; - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); + return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); } EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_write); @@ -6460,19 +6728,21 @@ EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_write); gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception) { - return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception); + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; + + return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception); } static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, struct kvm_vcpu *vcpu, u32 access, struct x86_exception *exception) { + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; void *data = val; int r = X86EMUL_CONTINUE; while (bytes) { - gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access, - exception); + gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception); unsigned offset = addr & (PAGE_SIZE-1); unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset); int ret; @@ -6500,13 +6770,14 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, struct x86_exception *exception) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; unsigned offset; int ret; /* Inline kvm_read_guest_virt_helper for speed. */ - gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK, - exception); + gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK, + exception); if (unlikely(gpa == UNMAPPED_GVA)) return X86EMUL_PROPAGATE_FAULT; @@ -6565,13 +6836,12 @@ static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes struct kvm_vcpu *vcpu, u32 access, struct x86_exception *exception) { + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; void *data = val; int r = X86EMUL_CONTINUE; while (bytes) { - gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, - access, - exception); + gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception); unsigned offset = addr & (PAGE_SIZE-1); unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset); int ret; @@ -6617,6 +6887,13 @@ int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val, } EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system); +static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type, + void *insn, int insn_len) +{ + return static_call(kvm_x86_can_emulate_instruction)(vcpu, emul_type, + insn, insn_len); +} + int handle_ud(struct kvm_vcpu *vcpu) { static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX }; @@ -6624,7 +6901,7 @@ int handle_ud(struct kvm_vcpu *vcpu) char sig[5]; /* ud2; .ascii "kvm" */ struct x86_exception e; - if (unlikely(!static_call(kvm_x86_can_emulate_instruction)(vcpu, NULL, 0))) + if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0))) return 1; if (force_emulation_prefix && @@ -6658,6 +6935,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, gpa_t *gpa, struct x86_exception *exception, bool write) { + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; u32 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0) | (write ? PFERR_WRITE_MASK : 0); @@ -6675,7 +6953,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, return 1; } - *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); + *gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); if (*gpa == UNMAPPED_GVA) return -1; @@ -7309,7 +7587,8 @@ static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, r = kvm_get_msr(vcpu, msr_index, pdata); - if (r && kvm_get_msr_user_space(vcpu, msr_index, r)) { + if (r && kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0, + complete_emulated_rdmsr, r)) { /* Bounce to user space */ return X86EMUL_IO_NEEDED; } @@ -7325,7 +7604,8 @@ static int emulator_set_msr(struct x86_emulate_ctxt *ctxt, r = kvm_set_msr(vcpu, msr_index, data); - if (r && kvm_set_msr_user_space(vcpu, msr_index, data, r)) { + if (r && kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data, + complete_emulated_msr_access, r)) { /* Bounce to user space */ return X86EMUL_IO_NEEDED; } @@ -7350,7 +7630,9 @@ static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase) static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt, u32 pmc) { - return kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc); + if (kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc)) + return 0; + return -EINVAL; } static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt, @@ -7586,29 +7868,78 @@ void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) } EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt); -static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu) +static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, + u8 ndata, u8 *insn_bytes, u8 insn_size) { - struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; - u32 insn_size = ctxt->fetch.end - ctxt->fetch.data; struct kvm_run *run = vcpu->run; + u64 info[5]; + u8 info_start; + + /* + * Zero the whole array used to retrieve the exit info, as casting to + * u32 for select entries will leave some chunks uninitialized. + */ + memset(&info, 0, sizeof(info)); + + static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1], + &info[2], (u32 *)&info[3], + (u32 *)&info[4]); run->exit_reason = KVM_EXIT_INTERNAL_ERROR; run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION; - run->emulation_failure.ndata = 0; + + /* + * There's currently space for 13 entries, but 5 are used for the exit + * reason and info. Restrict to 4 to reduce the maintenance burden + * when expanding kvm_run.emulation_failure in the future. + */ + if (WARN_ON_ONCE(ndata > 4)) + ndata = 4; + + /* Always include the flags as a 'data' entry. */ + info_start = 1; run->emulation_failure.flags = 0; if (insn_size) { - run->emulation_failure.ndata = 3; + BUILD_BUG_ON((sizeof(run->emulation_failure.insn_size) + + sizeof(run->emulation_failure.insn_bytes) != 16)); + info_start += 2; run->emulation_failure.flags |= KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES; run->emulation_failure.insn_size = insn_size; memset(run->emulation_failure.insn_bytes, 0x90, sizeof(run->emulation_failure.insn_bytes)); - memcpy(run->emulation_failure.insn_bytes, - ctxt->fetch.data, insn_size); + memcpy(run->emulation_failure.insn_bytes, insn_bytes, insn_size); } + + memcpy(&run->internal.data[info_start], info, sizeof(info)); + memcpy(&run->internal.data[info_start + ARRAY_SIZE(info)], data, + ndata * sizeof(data[0])); + + run->emulation_failure.ndata = info_start + ARRAY_SIZE(info) + ndata; } +static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu) +{ + struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; + + prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data, + ctxt->fetch.end - ctxt->fetch.data); +} + +void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, + u8 ndata) +{ + prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0); +} +EXPORT_SYMBOL_GPL(__kvm_prepare_emulation_failure_exit); + +void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu) +{ + __kvm_prepare_emulation_failure_exit(vcpu, NULL, 0); +} +EXPORT_SYMBOL_GPL(kvm_prepare_emulation_failure_exit); + static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) { struct kvm *kvm = vcpu->kvm; @@ -7623,16 +7954,14 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) if (kvm->arch.exit_on_emulation_error || (emulation_type & EMULTYPE_SKIP)) { - prepare_emulation_failure_exit(vcpu); + prepare_emulation_ctxt_failure_exit(vcpu); return 0; } kvm_queue_exception(vcpu, UD_VECTOR); if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) { - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + prepare_emulation_ctxt_failure_exit(vcpu); return 0; } @@ -7827,6 +8156,8 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) if (unlikely(!r)) return 0; + kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS); + /* * rflags is the old, "raw" value of the flags. The new value has * not been saved yet. @@ -7946,7 +8277,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, bool writeback = true; bool write_fault_to_spt; - if (unlikely(!static_call(kvm_x86_can_emulate_instruction)(vcpu, insn, insn_len))) + if (unlikely(!kvm_can_emulate_insn(vcpu, emulation_type, insn, insn_len))) return 1; vcpu->arch.l1tf_flush_l1d = true; @@ -7994,9 +8325,10 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, } /* - * Note, EMULTYPE_SKIP is intended for use *only* by vendor callbacks - * for kvm_skip_emulated_instruction(). The caller is responsible for - * updating interruptibility state and injecting single-step #DBs. + * EMULTYPE_SKIP without EMULTYPE_COMPLETE_USER_EXIT is intended for + * use *only* by vendor callbacks for kvm_skip_emulated_instruction(). + * The caller is responsible for updating interruptibility state and + * injecting single-step #DBs. */ if (emulation_type & EMULTYPE_SKIP) { if (ctxt->mode != X86EMUL_MODE_PROT64) @@ -8004,6 +8336,11 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, else ctxt->eip = ctxt->_eip; + if (emulation_type & EMULTYPE_COMPLETE_USER_EXIT) { + r = 1; + goto writeback; + } + kvm_rip_write(vcpu, ctxt->eip); if (ctxt->eflags & X86_EFLAGS_RF) kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF); @@ -8076,12 +8413,16 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, else r = 1; +writeback: if (writeback) { unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu); toggle_interruptibility(vcpu, ctxt->interruptibility); vcpu->arch.emulate_regs_need_sync_to_vcpu = false; if (!ctxt->have_exception || exception_type(ctxt->exception.vector) == EXCPT_TRAP) { + kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS); + if (ctxt->is_branch) + kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS); kvm_rip_write(vcpu, ctxt->eip); if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP))) r = kvm_vcpu_do_singlestep(vcpu); @@ -8240,14 +8581,13 @@ static void tsc_khz_changed(void *data) static void kvm_hyperv_tsc_notifier(void) { struct kvm *kvm; - struct kvm_vcpu *vcpu; int cpu; - unsigned long flags; mutex_lock(&kvm_lock); list_for_each_entry(kvm, &vm_list, vm_list) kvm_make_mclock_inprogress_request(kvm); + /* no guest entries from this point */ hyperv_stop_tsc_emulation(); /* TSC frequency always matches when on Hyper-V */ @@ -8256,18 +8596,11 @@ static void kvm_hyperv_tsc_notifier(void) kvm_max_guest_tsc_khz = tsc_khz; list_for_each_entry(kvm, &vm_list, vm_list) { - struct kvm_arch *ka = &kvm->arch; - - raw_spin_lock_irqsave(&ka->pvclock_gtod_sync_lock, flags); + __kvm_start_pvclock_update(kvm); pvclock_update_vm_gtod_copy(kvm); - raw_spin_unlock_irqrestore(&ka->pvclock_gtod_sync_lock, flags); - - kvm_for_each_vcpu(cpu, vcpu, kvm) - kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); - - kvm_for_each_vcpu(cpu, vcpu, kvm) - kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu); + kvm_end_pvclock_update(kvm); } + mutex_unlock(&kvm_lock); } #endif @@ -8276,7 +8609,8 @@ static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu) { struct kvm *kvm; struct kvm_vcpu *vcpu; - int i, send_ipi = 0; + int send_ipi = 0; + unsigned long i; /* * We allow guests to temporarily run on slowing clocks, @@ -8401,57 +8735,12 @@ static void kvm_timer_init(void) kvmclock_cpu_online, kvmclock_cpu_down_prep); } -DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu); -EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu); - -int kvm_is_in_guest(void) -{ - return __this_cpu_read(current_vcpu) != NULL; -} - -static int kvm_is_user_mode(void) -{ - int user_mode = 3; - - if (__this_cpu_read(current_vcpu)) - user_mode = static_call(kvm_x86_get_cpl)(__this_cpu_read(current_vcpu)); - - return user_mode != 0; -} - -static unsigned long kvm_get_guest_ip(void) -{ - unsigned long ip = 0; - - if (__this_cpu_read(current_vcpu)) - ip = kvm_rip_read(__this_cpu_read(current_vcpu)); - - return ip; -} - -static void kvm_handle_intel_pt_intr(void) -{ - struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu); - - kvm_make_request(KVM_REQ_PMI, vcpu); - __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, - (unsigned long *)&vcpu->arch.pmu.global_status); -} - -static struct perf_guest_info_callbacks kvm_guest_cbs = { - .is_in_guest = kvm_is_in_guest, - .is_user_mode = kvm_is_user_mode, - .get_guest_ip = kvm_get_guest_ip, - .handle_intel_pt_intr = NULL, -}; - #ifdef CONFIG_X86_64 static void pvclock_gtod_update_fn(struct work_struct *work) { struct kvm *kvm; - struct kvm_vcpu *vcpu; - int i; + unsigned long i; mutex_lock(&kvm_lock); list_for_each_entry(kvm, &vm_list, vm_list) @@ -8508,18 +8797,20 @@ int kvm_arch_init(void *opaque) int r; if (kvm_x86_ops.hardware_enable) { - printk(KERN_ERR "kvm: already loaded the other module\n"); + pr_err("kvm: already loaded vendor module '%s'\n", kvm_x86_ops.name); r = -EEXIST; goto out; } if (!ops->cpu_has_kvm_support()) { - pr_err_ratelimited("kvm: no hardware support\n"); + pr_err_ratelimited("kvm: no hardware support for '%s'\n", + ops->runtime_ops->name); r = -EOPNOTSUPP; goto out; } if (ops->disabled_by_bios()) { - pr_err_ratelimited("kvm: disabled by bios\n"); + pr_err_ratelimited("kvm: support for '%s' disabled by bios\n", + ops->runtime_ops->name); r = -EOPNOTSUPP; goto out; } @@ -8536,18 +8827,11 @@ int kvm_arch_init(void *opaque) } r = -ENOMEM; - x86_fpu_cache = kmem_cache_create("x86_fpu", sizeof(struct fpu), - __alignof__(struct fpu), SLAB_ACCOUNT, - NULL); - if (!x86_fpu_cache) { - printk(KERN_ERR "kvm: failed to allocate cache for x86 fpu\n"); - goto out; - } x86_emulator_cache = kvm_alloc_emulator_cache(); if (!x86_emulator_cache) { pr_err("kvm: failed to allocate cache for x86 emulator\n"); - goto out_free_x86_fpu_cache; + goto out; } user_return_msrs = alloc_percpu(struct kvm_user_return_msrs); @@ -8583,8 +8867,6 @@ int kvm_arch_init(void *opaque) free_percpu(user_return_msrs); out_free_x86_emulator_cache: kmem_cache_destroy(x86_emulator_cache); -out_free_x86_fpu_cache: - kmem_cache_destroy(x86_fpu_cache); out: return r; } @@ -8610,15 +8892,21 @@ void kvm_arch_exit(void) kvm_mmu_module_exit(); free_percpu(user_return_msrs); kmem_cache_destroy(x86_emulator_cache); - kmem_cache_destroy(x86_fpu_cache); #ifdef CONFIG_KVM_XEN static_key_deferred_flush(&kvm_xen_enabled); WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key)); #endif } -static int __kvm_vcpu_halt(struct kvm_vcpu *vcpu, int state, int reason) +static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason) { + /* + * The vCPU has halted, e.g. executed HLT. Update the run state if the + * local APIC is in-kernel, the run loop will detect the non-runnable + * state and halt the vCPU. Exit to userspace if the local APIC is + * managed by userspace, in which case userspace is responsible for + * handling wake events. + */ ++vcpu->stat.halt_exits; if (lapic_in_kernel(vcpu)) { vcpu->arch.mp_state = state; @@ -8629,11 +8917,11 @@ static int __kvm_vcpu_halt(struct kvm_vcpu *vcpu, int state, int reason) } } -int kvm_vcpu_halt(struct kvm_vcpu *vcpu) +int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu) { - return __kvm_vcpu_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT); + return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT); } -EXPORT_SYMBOL_GPL(kvm_vcpu_halt); +EXPORT_SYMBOL_GPL(kvm_emulate_halt_noskip); int kvm_emulate_halt(struct kvm_vcpu *vcpu) { @@ -8642,7 +8930,7 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu) * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered * KVM_EXIT_DEBUG here. */ - return kvm_vcpu_halt(vcpu) && ret; + return kvm_emulate_halt_noskip(vcpu) && ret; } EXPORT_SYMBOL_GPL(kvm_emulate_halt); @@ -8650,7 +8938,8 @@ int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu) { int ret = kvm_skip_emulated_instruction(vcpu); - return __kvm_vcpu_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD, KVM_EXIT_AP_RESET_HOLD) && ret; + return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD, + KVM_EXIT_AP_RESET_HOLD) && ret; } EXPORT_SYMBOL_GPL(kvm_emulate_ap_reset_hold); @@ -8718,12 +9007,11 @@ EXPORT_SYMBOL_GPL(kvm_apicv_activated); static void kvm_apicv_init(struct kvm *kvm) { - mutex_init(&kvm->arch.apicv_update_lock); + init_rwsem(&kvm->arch.apicv_update_lock); - if (enable_apicv) - clear_bit(APICV_INHIBIT_REASON_DISABLE, - &kvm->arch.apicv_inhibit_reasons); - else + set_bit(APICV_INHIBIT_REASON_ABSENT, + &kvm->arch.apicv_inhibit_reasons); + if (!enable_apicv) set_bit(APICV_INHIBIT_REASON_DISABLE, &kvm->arch.apicv_inhibit_reasons); } @@ -8892,6 +9180,7 @@ static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu) likely(!pic_in_kernel(vcpu->kvm)); } +/* Called within kvm->srcu read side. */ static void post_kvm_run_save(struct kvm_vcpu *vcpu) { struct kvm_run *kvm_run = vcpu->run; @@ -8900,16 +9189,9 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu) kvm_run->cr8 = kvm_get_cr8(vcpu); kvm_run->apic_base = kvm_get_apic_base(vcpu); - /* - * The call to kvm_ready_for_interrupt_injection() may end up in - * kvm_xen_has_interrupt() which may require the srcu lock to be - * held, to protect against changes in the vcpu_info address. - */ - vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); kvm_run->ready_for_interrupt_injection = pic_in_kernel(vcpu->kvm) || kvm_vcpu_ready_for_interrupt_injection(vcpu); - srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); if (is_smm(vcpu)) kvm_run->flags |= KVM_RUN_X86_SMM; @@ -9366,14 +9648,7 @@ static void process_smi(struct kvm_vcpu *vcpu) void kvm_make_scan_ioapic_request_mask(struct kvm *kvm, unsigned long *vcpu_bitmap) { - cpumask_var_t cpus; - - zalloc_cpumask_var(&cpus, GFP_ATOMIC); - - kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, - NULL, vcpu_bitmap, cpus); - - free_cpumask_var(cpus); + kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, vcpu_bitmap); } void kvm_make_scan_ioapic_request(struct kvm *kvm) @@ -9388,7 +9663,7 @@ void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) if (!lapic_in_kernel(vcpu)) return; - mutex_lock(&vcpu->kvm->arch.apicv_update_lock); + down_read(&vcpu->kvm->arch.apicv_update_lock); activate = kvm_apicv_activated(vcpu->kvm); if (vcpu->arch.apicv_active == activate) @@ -9408,7 +9683,7 @@ void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) kvm_make_request(KVM_REQ_EVENT, vcpu); out: - mutex_unlock(&vcpu->kvm->arch.apicv_update_lock); + up_read(&vcpu->kvm->arch.apicv_update_lock); } EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv); @@ -9416,6 +9691,8 @@ void __kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) { unsigned long old, new; + lockdep_assert_held_write(&kvm->arch.apicv_update_lock); + if (!kvm_x86_ops.check_apicv_inhibit_reasons || !static_call(kvm_x86_check_apicv_inhibit_reasons)(bit)) return; @@ -9429,6 +9706,18 @@ void __kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) if (!!old != !!new) { trace_kvm_apicv_update_request(activate, bit); + /* + * Kick all vCPUs before setting apicv_inhibit_reasons to avoid + * false positives in the sanity check WARN in svm_vcpu_run(). + * This task will wait for all vCPUs to ack the kick IRQ before + * updating apicv_inhibit_reasons, and all other vCPUs will + * block on acquiring apicv_update_lock so that vCPUs can't + * redo svm_vcpu_run() without seeing the new inhibit state. + * + * Note, holding apicv_update_lock and taking it in the read + * side (handling the request) also prevents other vCPUs from + * servicing the request with a stale apicv_inhibit_reasons. + */ kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE); kvm->arch.apicv_inhibit_reasons = new; if (new) { @@ -9442,9 +9731,9 @@ EXPORT_SYMBOL_GPL(__kvm_request_apicv_update); void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) { - mutex_lock(&kvm->arch.apicv_update_lock); + down_write(&kvm->arch.apicv_update_lock); __kvm_request_apicv_update(kvm, activate, bit); - mutex_unlock(&kvm->arch.apicv_update_lock); + up_write(&kvm->arch.apicv_update_lock); } EXPORT_SYMBOL_GPL(kvm_request_apicv_update); @@ -9502,7 +9791,7 @@ void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD); } -void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) +static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) { if (!lapic_in_kernel(vcpu)) return; @@ -9520,6 +9809,7 @@ void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu) EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit); /* + * Called within kvm->srcu read side. * Returns 1 to let vcpu_run() continue the guest execution loop without * exiting to the userspace. Otherwise, the value will be returned to the * userspace. @@ -9544,7 +9834,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) } if (kvm_request_pending(vcpu)) { - if (kvm_check_request(KVM_REQ_VM_BUGGED, vcpu)) { + if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) { r = -EIO; goto out; } @@ -9559,7 +9849,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu)) __kvm_migrate_timers(vcpu); if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) - kvm_gen_update_masterclock(vcpu->kvm); + kvm_update_masterclock(vcpu->kvm); if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu)) kvm_gen_kvmclock_update(vcpu); if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) { @@ -9708,7 +9998,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) * result in virtual interrupt delivery. */ local_irq_disable(); - vcpu->mode = IN_GUEST_MODE; + + /* Store vcpu->apicv_active before vcpu->mode. */ + smp_store_release(&vcpu->mode, IN_GUEST_MODE); srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); @@ -9727,10 +10019,11 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) smp_mb__after_srcu_read_unlock(); /* - * This handles the case where a posted interrupt was - * notified with kvm_vcpu_kick. Assigned devices can - * use the POSTED_INTR_VECTOR even if APICv is disabled, - * so do it even if APICv is disabled on this vCPU. + * Process pending posted interrupts to handle the case where the + * notification IRQ arrived in the host, or was never sent (because the + * target vCPU wasn't running). Do this regardless of the vCPU's APICv + * status, KVM doesn't update assigned devices when APICv is inhibited, + * i.e. they can post interrupts even if APICv is temporarily disabled. */ if (kvm_lapic_enabled(vcpu)) static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu); @@ -9754,6 +10047,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (test_thread_flag(TIF_NEED_FPU_LOAD)) switch_fpu_return(); + if (vcpu->arch.guest_fpu.xfd_err) + wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); + if (unlikely(vcpu->arch.switch_db_regs)) { set_debugreg(0, 7); set_debugreg(vcpu->arch.eff_db[0], 0); @@ -9764,7 +10060,17 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) set_debugreg(0, 7); } + guest_timing_enter_irqoff(); + for (;;) { + /* + * Assert that vCPU vs. VM APICv state is consistent. An APICv + * update must kick and wait for all vCPUs before toggling the + * per-VM state, and responsing vCPUs must wait for the update + * to complete before servicing KVM_REQ_APICV_UPDATE. + */ + WARN_ON_ONCE(kvm_apicv_activated(vcpu->kvm) != kvm_vcpu_apicv_active(vcpu)); + exit_fastpath = static_call(kvm_x86_run)(vcpu); if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST)) break; @@ -9807,8 +10113,19 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu->mode = OUTSIDE_GUEST_MODE; smp_wmb(); + /* + * Sync xfd before calling handle_exit_irqoff() which may + * rely on the fact that guest_fpu::xfd is up-to-date (e.g. + * in #NM irqoff handler). + */ + if (vcpu->arch.xfd_no_write_intercept) + fpu_sync_guest_vmexit_xfd_state(); + static_call(kvm_x86_handle_exit_irqoff)(vcpu); + if (vcpu->arch.guest_fpu.xfd_err) + wrmsrl(MSR_IA32_XFD_ERR, 0); + /* * Consume any pending interrupts, including the possible source of * VM-Exit on SVM and any ticks that occur between VM-Exit and now. @@ -9816,7 +10133,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) * interrupts on processors that implement an interrupt shadow, the * stat.exits increment will do nicely. */ - kvm_before_interrupt(vcpu); + kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); local_irq_enable(); ++vcpu->stat.exits; local_irq_disable(); @@ -9829,7 +10146,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) * of accounting via context tracking, but the loss of accuracy is * acceptable for all known use cases. */ - vtime_account_guest_exit(); + guest_timing_exit_irqoff(); if (lapic_in_kernel(vcpu)) { s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta; @@ -9871,16 +10188,32 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) return r; } +/* Called within kvm->srcu read side. */ static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu) { - if (!kvm_arch_vcpu_runnable(vcpu) && - (!kvm_x86_ops.pre_block || static_call(kvm_x86_pre_block)(vcpu) == 0)) { + bool hv_timer; + + if (!kvm_arch_vcpu_runnable(vcpu)) { + /* + * Switch to the software timer before halt-polling/blocking as + * the guest's timer may be a break event for the vCPU, and the + * hypervisor timer runs only when the CPU is in guest mode. + * Switch before halt-polling so that KVM recognizes an expired + * timer before blocking. + */ + hv_timer = kvm_lapic_hv_timer_in_use(vcpu); + if (hv_timer) + kvm_lapic_switch_to_sw_timer(vcpu); + srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); - kvm_vcpu_block(vcpu); + if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) + kvm_vcpu_halt(vcpu); + else + kvm_vcpu_block(vcpu); vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); - if (kvm_x86_ops.post_block) - static_call(kvm_x86_post_block)(vcpu); + if (hv_timer) + kvm_lapic_switch_to_hv_timer(vcpu); if (!kvm_check_request(KVM_REQ_UNHALT, vcpu)) return 1; @@ -9915,12 +10248,12 @@ static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu) !vcpu->arch.apf.halted); } +/* Called within kvm->srcu read side. */ static int vcpu_run(struct kvm_vcpu *vcpu) { int r; struct kvm *kvm = vcpu->kvm; - vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); vcpu->arch.l1tf_flush_l1d = true; for (;;) { @@ -9948,14 +10281,12 @@ static int vcpu_run(struct kvm_vcpu *vcpu) if (__xfer_to_guest_mode_work_pending()) { srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); r = xfer_to_guest_mode_handle_work(vcpu); + vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); if (r) return r; - vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); } } - srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); - return r; } @@ -10039,58 +10370,21 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu) return 0; } -static void kvm_save_current_fpu(struct fpu *fpu) -{ - /* - * If the target FPU state is not resident in the CPU registers, just - * memcpy() from current, else save CPU state directly to the target. - */ - if (test_thread_flag(TIF_NEED_FPU_LOAD)) - memcpy(&fpu->state, ¤t->thread.fpu.state, - fpu_kernel_xstate_size); - else - save_fpregs_to_fpstate(fpu); -} - /* Swap (qemu) user FPU context for the guest FPU context. */ static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) { - fpregs_lock(); - - kvm_save_current_fpu(vcpu->arch.user_fpu); - /* - * Guests with protected state can't have it set by the hypervisor, - * so skip trying to set it. + * Exclude PKRU from restore as restored separately in + * kvm_x86_ops.run(). */ - if (vcpu->arch.guest_fpu) - /* PKRU is separately restored in kvm_x86_ops.run. */ - __restore_fpregs_from_fpstate(&vcpu->arch.guest_fpu->state, - ~XFEATURE_MASK_PKRU); - - fpregs_mark_activate(); - fpregs_unlock(); - + fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true); trace_kvm_fpu(1); } /* When vcpu_run ends, restore user space FPU context. */ static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) { - fpregs_lock(); - - /* - * Guests with protected state can't have it read by the hypervisor, - * so skip trying to save it. - */ - if (vcpu->arch.guest_fpu) - kvm_save_current_fpu(vcpu->arch.guest_fpu); - - restore_fpregs_from_fpstate(&vcpu->arch.user_fpu->state); - - fpregs_mark_activate(); - fpregs_unlock(); - + fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false); ++vcpu->stat.fpu_reload; trace_kvm_fpu(0); } @@ -10098,6 +10392,7 @@ static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { struct kvm_run *kvm_run = vcpu->run; + struct kvm *kvm = vcpu->kvm; int r; vcpu_load(vcpu); @@ -10105,12 +10400,22 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_run->flags = 0; kvm_load_guest_fpu(vcpu); + vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { if (kvm_run->immediate_exit) { r = -EINTR; goto out; } + /* + * It should be impossible for the hypervisor timer to be in + * use before KVM has ever run the vCPU. + */ + WARN_ON_ONCE(kvm_lapic_hv_timer_in_use(vcpu)); + + srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); kvm_vcpu_block(vcpu); + vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); + if (kvm_apic_accept_events(vcpu) < 0) { r = 0; goto out; @@ -10154,18 +10459,25 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) } else WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed); - if (kvm_run->immediate_exit) + if (kvm_run->immediate_exit) { r = -EINTR; - else - r = vcpu_run(vcpu); + goto out; + } + + r = static_call(kvm_x86_vcpu_pre_run)(vcpu); + if (r <= 0) + goto out; + + r = vcpu_run(vcpu); out: kvm_put_guest_fpu(vcpu); if (kvm_run->kvm_valid_regs) store_regs(vcpu); post_kvm_run_save(vcpu); - kvm_sigset_deactivate(vcpu); + srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); + kvm_sigset_deactivate(vcpu); vcpu_put(vcpu); return r; } @@ -10473,7 +10785,8 @@ static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, vcpu->arch.cr2 = sregs->cr2; *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; vcpu->arch.cr3 = sregs->cr3; - kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); + kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); + static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3); kvm_set_cr8(vcpu, sregs->cr8); @@ -10490,7 +10803,7 @@ static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, if (update_pdptrs) { idx = srcu_read_lock(&vcpu->kvm->srcu); if (is_pae_paging(vcpu)) { - load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); + load_pdptrs(vcpu, kvm_read_cr3(vcpu)); *mmu_reset_needed = 1; } srcu_read_unlock(&vcpu->kvm->srcu, idx); @@ -10584,6 +10897,24 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, return ret; } +static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm) +{ + bool inhibit = false; + struct kvm_vcpu *vcpu; + unsigned long i; + + down_write(&kvm->arch.apicv_update_lock); + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) { + inhibit = true; + break; + } + } + __kvm_request_apicv_update(kvm, !inhibit, APICV_INHIBIT_REASON_BLOCKIRQ); + up_write(&kvm->arch.apicv_update_lock); +} + int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) { @@ -10636,6 +10967,8 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, static_call(kvm_x86_update_exception_bitmap)(vcpu); + kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm); + r = 0; out: @@ -10671,12 +11004,12 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) { struct fxregs_state *fxsave; - if (!vcpu->arch.guest_fpu) + if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) return 0; vcpu_load(vcpu); - fxsave = &vcpu->arch.guest_fpu->state.fxsave; + fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; memcpy(fpu->fpr, fxsave->st_space, 128); fpu->fcw = fxsave->cwd; fpu->fsw = fxsave->swd; @@ -10694,12 +11027,12 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) { struct fxregs_state *fxsave; - if (!vcpu->arch.guest_fpu) + if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) return 0; vcpu_load(vcpu); - fxsave = &vcpu->arch.guest_fpu->state.fxsave; + fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; memcpy(fxsave->st_space, fpu->fpr, 128); fxsave->cwd = fpu->fcw; @@ -10750,33 +11083,6 @@ static int sync_regs(struct kvm_vcpu *vcpu) return 0; } -static void fx_init(struct kvm_vcpu *vcpu) -{ - if (!vcpu->arch.guest_fpu) - return; - - fpstate_init(&vcpu->arch.guest_fpu->state); - if (boot_cpu_has(X86_FEATURE_XSAVES)) - vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv = - host_xcr0 | XSTATE_COMPACTION_ENABLED; - - /* - * Ensure guest xcr0 is valid for loading - */ - vcpu->arch.xcr0 = XFEATURE_MASK_FP; - - vcpu->arch.cr0 |= X86_CR0_ET; -} - -void kvm_free_guest_fpu(struct kvm_vcpu *vcpu) -{ - if (vcpu->arch.guest_fpu) { - kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu); - vcpu->arch.guest_fpu = NULL; - } -} -EXPORT_SYMBOL_GPL(kvm_free_guest_fpu); - int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) { if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0) @@ -10833,21 +11139,11 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) if (!alloc_emulate_ctxt(vcpu)) goto free_wbinvd_dirty_mask; - vcpu->arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache, - GFP_KERNEL_ACCOUNT); - if (!vcpu->arch.user_fpu) { - pr_err("kvm: failed to allocate userspace's fpu\n"); + if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) { + pr_err("kvm: failed to allocate vcpu's fpu\n"); goto free_emulate_ctxt; } - vcpu->arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache, - GFP_KERNEL_ACCOUNT); - if (!vcpu->arch.guest_fpu) { - pr_err("kvm: failed to allocate vcpu's fpu\n"); - goto free_user_fpu; - } - fx_init(vcpu); - vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu); vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu); @@ -10878,9 +11174,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) return 0; free_guest_fpu: - kvm_free_guest_fpu(vcpu); -free_user_fpu: - kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu); + fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); free_emulate_ctxt: kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); free_wbinvd_dirty_mask: @@ -10926,8 +11220,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); - kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu); - kvm_free_guest_fpu(vcpu); + fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); kvm_hv_vcpu_uninit(vcpu); kvm_pmu_destroy(vcpu); @@ -10944,9 +11237,19 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) { + struct kvm_cpuid_entry2 *cpuid_0x1; unsigned long old_cr0 = kvm_read_cr0(vcpu); unsigned long new_cr0; - u32 eax, dummy; + + /* + * Several of the "set" flows, e.g. ->set_cr0(), read other registers + * to handle side effects. RESET emulation hits those flows and relies + * on emulated/virtualized registers, including those that are loaded + * into hardware, to be zeroed at vCPU creation. Use CRs as a sentinel + * to detect improper or missing initialization. + */ + WARN_ON_ONCE(!init_event && + (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu))); kvm_lapic_reset(vcpu, init_event); @@ -10979,8 +11282,8 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) kvm_async_pf_hash_reset(vcpu); vcpu->arch.apf.halted = false; - if (vcpu->arch.guest_fpu && kvm_mpx_supported()) { - void *mpx_state_buffer; + if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) { + struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate; /* * To avoid have the INIT path from kvm_apic_has_events() that be @@ -10988,14 +11291,10 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) */ if (init_event) kvm_put_guest_fpu(vcpu); - mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave, - XFEATURE_BNDREGS); - if (mpx_state_buffer) - memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state)); - mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave, - XFEATURE_BNDCSR); - if (mpx_state_buffer) - memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr)); + + fpstate_clear_xstate_component(fpstate, XFEATURE_BNDREGS); + fpstate_clear_xstate_component(fpstate, XFEATURE_BNDCSR); + if (init_event) kvm_load_guest_fpu(vcpu); } @@ -11010,21 +11309,19 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) __kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true); } + /* All GPRs except RDX (handled below) are zeroed on RESET/INIT. */ memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs)); - vcpu->arch.regs_avail = ~0; - vcpu->arch.regs_dirty = ~0; + kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP); /* * Fall back to KVM's default Family/Model/Stepping of 0x600 (P6/Athlon) * if no CPUID match is found. Note, it's impossible to get a match at * RESET since KVM emulates RESET before exposing the vCPU to userspace, - * i.e. it'simpossible for kvm_cpuid() to find a valid entry on RESET. - * But, go through the motions in case that's ever remedied. + * i.e. it's impossible for kvm_find_cpuid_entry() to find a valid entry + * on RESET. But, go through the motions in case that's ever remedied. */ - eax = 1; - if (!kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true)) - eax = 0x600; - kvm_rdx_write(vcpu, eax); + cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1, 0); + kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600); static_call(kvm_x86_vcpu_reset)(vcpu, init_event); @@ -11091,7 +11388,7 @@ int kvm_arch_hardware_enable(void) { struct kvm *kvm; struct kvm_vcpu *vcpu; - int i; + unsigned long i; int ret; u64 local_tsc; u64 max_tsc = 0; @@ -11201,9 +11498,7 @@ int kvm_arch_hardware_setup(void *opaque) memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); kvm_ops_static_call_update(); - if (ops->intel_pt_intr_in_guest && ops->intel_pt_intr_in_guest()) - kvm_guest_cbs.handle_intel_pt_intr = kvm_handle_intel_pt_intr; - perf_register_guest_info_callbacks(&kvm_guest_cbs); + kvm_register_perf_callbacks(ops->handle_intel_pt_intr); if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) supported_xss = 0; @@ -11232,8 +11527,7 @@ int kvm_arch_hardware_setup(void *opaque) void kvm_arch_hardware_unsetup(void) { - perf_unregister_guest_info_callbacks(&kvm_guest_cbs); - kvm_guest_cbs.handle_intel_pt_intr = NULL; + kvm_unregister_perf_callbacks(); static_call(kvm_x86_hardware_unsetup)(); } @@ -11281,13 +11575,14 @@ void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) void kvm_arch_free_vm(struct kvm *kvm) { kfree(to_kvm_hv(kvm)->hv_pa_pg); - vfree(kvm); + __kvm_arch_free_vm(kvm); } int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { int ret; + unsigned long flags; if (type) return -EINVAL; @@ -11311,10 +11606,12 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) raw_spin_lock_init(&kvm->arch.tsc_write_lock); mutex_init(&kvm->arch.apic_map_lock); - raw_spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock); - + seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock); kvm->arch.kvmclock_offset = -get_kvmclock_base_ns(); + + raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); pvclock_update_vm_gtod_copy(kvm); + raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); kvm->arch.guest_can_read_msr_platform_info = true; @@ -11348,7 +11645,7 @@ static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) static void kvm_free_vcpus(struct kvm *kvm) { - unsigned int i; + unsigned long i; struct kvm_vcpu *vcpu; /* @@ -11358,15 +11655,8 @@ static void kvm_free_vcpus(struct kvm *kvm) kvm_clear_async_pf_completion_queue(vcpu); kvm_unload_vcpu_mmu(vcpu); } - kvm_for_each_vcpu(i, vcpu, kvm) - kvm_vcpu_destroy(vcpu); - mutex_lock(&kvm->lock); - for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) - kvm->vcpus[i] = NULL; - - atomic_set(&kvm->online_vcpus, 0); - mutex_unlock(&kvm->lock); + kvm_destroy_vcpus(kvm); } void kvm_arch_sync_events(struct kvm *kvm) @@ -11376,8 +11666,6 @@ void kvm_arch_sync_events(struct kvm *kvm) kvm_free_pit(kvm); } -#define ERR_PTR_USR(e) ((void __user *)ERR_PTR(e)) - /** * __x86_set_memory_region: Setup KVM internal memory slot * @@ -11511,8 +11799,7 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) kvm_page_track_free_memslot(slot); } -static int memslot_rmap_alloc(struct kvm_memory_slot *slot, - unsigned long npages) +int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages) { const int sz = sizeof(*slot->arch.rmap[0]); int i; @@ -11534,54 +11821,10 @@ static int memslot_rmap_alloc(struct kvm_memory_slot *slot, return 0; } -int alloc_all_memslots_rmaps(struct kvm *kvm) -{ - struct kvm_memslots *slots; - struct kvm_memory_slot *slot; - int r, i; - - /* - * Check if memslots alreday have rmaps early before acquiring - * the slots_arch_lock below. - */ - if (kvm_memslots_have_rmaps(kvm)) - return 0; - - mutex_lock(&kvm->slots_arch_lock); - - /* - * Read memslots_have_rmaps again, under the slots arch lock, - * before allocating the rmaps - */ - if (kvm_memslots_have_rmaps(kvm)) { - mutex_unlock(&kvm->slots_arch_lock); - return 0; - } - - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { - slots = __kvm_memslots(kvm, i); - kvm_for_each_memslot(slot, slots) { - r = memslot_rmap_alloc(slot, slot->npages); - if (r) { - mutex_unlock(&kvm->slots_arch_lock); - return r; - } - } - } - - /* - * Ensure that memslots_have_rmaps becomes true strictly after - * all the rmap pointers are set. - */ - smp_store_release(&kvm->arch.memslots_have_rmaps, true); - mutex_unlock(&kvm->slots_arch_lock); - return 0; -} - static int kvm_alloc_memslot_metadata(struct kvm *kvm, - struct kvm_memory_slot *slot, - unsigned long npages) + struct kvm_memory_slot *slot) { + unsigned long npages = slot->npages; int i, r; /* @@ -11628,7 +11871,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm, } } - if (kvm_page_track_create_memslot(slot, npages)) + if (kvm_page_track_create_memslot(kvm, slot, npages)) goto out_free; return 0; @@ -11646,7 +11889,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm, void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) { struct kvm_vcpu *vcpu; - int i; + unsigned long i; /* * memslots->generation has been incremented. @@ -11660,13 +11903,18 @@ void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) } int kvm_arch_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, - enum kvm_mr_change change) + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) { if (change == KVM_MR_CREATE || change == KVM_MR_MOVE) - return kvm_alloc_memslot_metadata(kvm, memslot, - mem->memory_size >> PAGE_SHIFT); + return kvm_alloc_memslot_metadata(kvm, new); + + if (change == KVM_MR_FLAGS_ONLY) + memcpy(&new->arch, &old->arch, sizeof(old->arch)); + else if (WARN_ON_ONCE(change != KVM_MR_DELETE)) + return -EIO; + return 0; } @@ -11690,13 +11938,15 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, const struct kvm_memory_slot *new, enum kvm_mr_change change) { - bool log_dirty_pages = new->flags & KVM_MEM_LOG_DIRTY_PAGES; + u32 old_flags = old ? old->flags : 0; + u32 new_flags = new ? new->flags : 0; + bool log_dirty_pages = new_flags & KVM_MEM_LOG_DIRTY_PAGES; /* * Update CPU dirty logging if dirty logging is being toggled. This * applies to all operations. */ - if ((old->flags ^ new->flags) & KVM_MEM_LOG_DIRTY_PAGES) + if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES) kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages); /* @@ -11714,7 +11964,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, * MOVE/DELETE: The old mappings will already have been cleaned up by * kvm_arch_flush_shadow_memslot(). */ - if ((change != KVM_MR_FLAGS_ONLY) || (new->flags & KVM_MEM_READONLY)) + if ((change != KVM_MR_FLAGS_ONLY) || (new_flags & KVM_MEM_READONLY)) return; /* @@ -11722,7 +11972,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty * logging isn't being toggled on or off. */ - if (WARN_ON_ONCE(!((old->flags ^ new->flags) & KVM_MEM_LOG_DIRTY_PAGES))) + if (WARN_ON_ONCE(!((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES))) return; if (!log_dirty_pages) { @@ -11758,14 +12008,18 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, } void kvm_arch_commit_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { - if (!kvm->arch.n_requested_mmu_pages) - kvm_mmu_change_mmu_pages(kvm, - kvm_mmu_calculate_default_mmu_pages(kvm)); + if (!kvm->arch.n_requested_mmu_pages && + (change == KVM_MR_CREATE || change == KVM_MR_DELETE)) { + unsigned long nr_mmu_pages; + + nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO; + nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES); + kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); + } kvm_mmu_slot_apply_flags(kvm, old, new, change); @@ -11866,6 +12120,11 @@ bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) return vcpu->arch.preempted_in_kernel; } +unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) +{ + return kvm_rip_read(vcpu); +} + int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) { return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; @@ -12226,6 +12485,15 @@ int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq, return static_call(kvm_x86_update_pi_irte)(kvm, host_irq, guest_irq, set); } +bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old, + struct kvm_kernel_irq_routing_entry *new) +{ + if (new->type != KVM_IRQ_ROUTING_MSI) + return true; + + return !!memcmp(&old->msi, &new->msi, sizeof(new->msi)); +} + bool kvm_vector_hashing_enabled(void) { return vector_hashing; @@ -12266,12 +12534,13 @@ EXPORT_SYMBOL_GPL(kvm_spec_ctrl_test_value); void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code) { + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; struct x86_exception fault; u32 access = error_code & (PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK); if (!(error_code & PFERR_PRESENT_MASK) || - vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, &fault) != UNMAPPED_GVA) { + mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != UNMAPPED_GVA) { /* * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page * tables probably do not match the TLB. Just proceed @@ -12307,9 +12576,7 @@ int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r, * doesn't seem to be a real use-case behind such requests, just return * KVM_EXIT_INTERNAL_ERROR for now. */ - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + kvm_prepare_emulation_failure_exit(vcpu); return 0; } @@ -12369,7 +12636,8 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) return kvm_skip_emulated_instruction(vcpu); default: - BUG(); /* We have already checked above that type <= 3 */ + kvm_inject_gp(vcpu, 0); + return 1; } } EXPORT_SYMBOL_GPL(kvm_handle_invpcid); @@ -12609,6 +12877,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_update_request); +EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_enter); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index bc7ef6820a..767ec7f995 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -10,51 +10,6 @@ void kvm_spurious_fault(void); -static __always_inline void kvm_guest_enter_irqoff(void) -{ - /* - * VMENTER enables interrupts (host state), but the kernel state is - * interrupts disabled when this is invoked. Also tell RCU about - * it. This is the same logic as for exit_to_user_mode(). - * - * This ensures that e.g. latency analysis on the host observes - * guest mode as interrupt enabled. - * - * guest_enter_irqoff() informs context tracking about the - * transition to guest mode and if enabled adjusts RCU state - * accordingly. - */ - instrumentation_begin(); - trace_hardirqs_on_prepare(); - lockdep_hardirqs_on_prepare(CALLER_ADDR0); - instrumentation_end(); - - guest_enter_irqoff(); - lockdep_hardirqs_on(CALLER_ADDR0); -} - -static __always_inline void kvm_guest_exit_irqoff(void) -{ - /* - * VMEXIT disables interrupts (host state), but tracing and lockdep - * have them in state 'on' as recorded before entering guest mode. - * Same as enter_from_user_mode(). - * - * context_tracking_guest_exit() restores host context and reinstates - * RCU if enabled and required. - * - * This needs to be done immediately after VM-Exit, before any code - * that might contain tracepoints or call out to the greater world, - * e.g. before x86_spec_ctrl_restore_host(). - */ - lockdep_hardirqs_off(CALLER_ADDR0); - context_tracking_guest_exit(); - - instrumentation_begin(); - trace_hardirqs_off_finish(); - instrumentation_end(); -} - #define KVM_NESTED_VMENTER_CONSISTENCY_CHECK(consistency_check) \ ({ \ bool failed = (consistency_check); \ @@ -301,7 +256,6 @@ static inline bool kvm_vcpu_latch_init(struct kvm_vcpu *vcpu) return is_smm(vcpu) || static_call(kvm_x86_apic_init_signal_blocked)(vcpu); } -void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs); void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip); u64 get_kvmclock_ns(struct kvm *kvm); @@ -337,6 +291,7 @@ extern u64 host_xcr0; extern u64 supported_xcr0; extern u64 host_xss; extern u64 supported_xss; +extern bool enable_pmu; static inline bool kvm_mpx_supported(void) { @@ -350,8 +305,6 @@ extern bool enable_vmware_backdoor; extern int pi_inject_timer; -extern struct static_key kvm_no_apic_vcpu; - extern bool report_ignored_msrs; static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec) @@ -394,18 +347,27 @@ static inline bool kvm_cstate_in_guest(struct kvm *kvm) return kvm->arch.cstate_in_guest; } -DECLARE_PER_CPU(struct kvm_vcpu *, current_vcpu); +enum kvm_intr_type { + /* Values are arbitrary, but must be non-zero. */ + KVM_HANDLING_IRQ = 1, + KVM_HANDLING_NMI, +}; -static inline void kvm_before_interrupt(struct kvm_vcpu *vcpu) +static inline void kvm_before_interrupt(struct kvm_vcpu *vcpu, + enum kvm_intr_type intr) { - __this_cpu_write(current_vcpu, vcpu); + WRITE_ONCE(vcpu->arch.handling_intr_from_guest, (u8)intr); } static inline void kvm_after_interrupt(struct kvm_vcpu *vcpu) { - __this_cpu_write(current_vcpu, NULL); + WRITE_ONCE(vcpu->arch.handling_intr_from_guest, 0); } +static inline bool kvm_handling_nmi_from_guest(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.handling_intr_from_guest == KVM_HANDLING_NMI; +} static inline bool kvm_pat_valid(u64 data) { diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index ab9f88de6d..74be1fda58 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "trace.h" @@ -23,38 +24,77 @@ DEFINE_STATIC_KEY_DEFERRED_FALSE(kvm_xen_enabled, HZ); static int kvm_xen_shared_info_init(struct kvm *kvm, gfn_t gfn) { + struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; + struct pvclock_wall_clock *wc; gpa_t gpa = gfn_to_gpa(gfn); - int wc_ofs, sec_hi_ofs; + u32 *wc_sec_hi; + u32 wc_version; + u64 wall_nsec; int ret = 0; int idx = srcu_read_lock(&kvm->srcu); - if (kvm_is_error_hva(gfn_to_hva(kvm, gfn))) { - ret = -EFAULT; + if (gfn == GPA_INVALID) { + kvm_gfn_to_pfn_cache_destroy(kvm, gpc); goto out; } - kvm->arch.xen.shinfo_gfn = gfn; + + do { + ret = kvm_gfn_to_pfn_cache_init(kvm, gpc, NULL, false, true, + gpa, PAGE_SIZE, false); + if (ret) + goto out; + + /* + * This code mirrors kvm_write_wall_clock() except that it writes + * directly through the pfn cache and doesn't mark the page dirty. + */ + wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm); + + /* It could be invalid again already, so we need to check */ + read_lock_irq(&gpc->lock); + + if (gpc->valid) + break; + + read_unlock_irq(&gpc->lock); + } while (1); /* Paranoia checks on the 32-bit struct layout */ BUILD_BUG_ON(offsetof(struct compat_shared_info, wc) != 0x900); BUILD_BUG_ON(offsetof(struct compat_shared_info, arch.wc_sec_hi) != 0x924); BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0); - /* 32-bit location by default */ - wc_ofs = offsetof(struct compat_shared_info, wc); - sec_hi_ofs = offsetof(struct compat_shared_info, arch.wc_sec_hi); - #ifdef CONFIG_X86_64 /* Paranoia checks on the 64-bit struct layout */ BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00); BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c); - if (kvm->arch.xen.long_mode) { - wc_ofs = offsetof(struct shared_info, wc); - sec_hi_ofs = offsetof(struct shared_info, wc_sec_hi); - } -#endif + if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + struct shared_info *shinfo = gpc->khva; + + wc_sec_hi = &shinfo->wc_sec_hi; + wc = &shinfo->wc; + } else +#endif + { + struct compat_shared_info *shinfo = gpc->khva; + + wc_sec_hi = &shinfo->arch.wc_sec_hi; + wc = &shinfo->wc; + } + + /* Increment and ensure an odd value */ + wc_version = wc->version = (wc->version + 1) | 1; + smp_wmb(); + + wc->nsec = do_div(wall_nsec, 1000000000); + wc->sec = (u32)wall_nsec; + *wc_sec_hi = wall_nsec >> 32; + smp_wmb(); + + wc->version = wc_version + 1; + read_unlock_irq(&gpc->lock); - kvm_write_wall_clock(kvm, gpa + wc_ofs, sec_hi_ofs - wc_ofs); kvm_make_all_cpus_request(kvm, KVM_REQ_MASTERCLOCK_UPDATE); out: @@ -152,9 +192,9 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) state_entry_time = vx->runstate_entry_time; state_entry_time |= XEN_RUNSTATE_UPDATE; - BUILD_BUG_ON(sizeof(((struct vcpu_runstate_info *)0)->state_entry_time) != + BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, state_entry_time) != sizeof(state_entry_time)); - BUILD_BUG_ON(sizeof(((struct compat_vcpu_runstate_info *)0)->state_entry_time) != + BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state_entry_time) != sizeof(state_entry_time)); if (__put_user(state_entry_time, user_times)) @@ -167,9 +207,9 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) */ BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state) != offsetof(struct compat_vcpu_runstate_info, state)); - BUILD_BUG_ON(sizeof(((struct vcpu_runstate_info *)0)->state) != + BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, state) != sizeof(vx->current_runstate)); - BUILD_BUG_ON(sizeof(((struct compat_vcpu_runstate_info *)0)->state) != + BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state) != sizeof(vx->current_runstate)); if (__put_user(vx->current_runstate, user_state)) @@ -183,9 +223,9 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) offsetof(struct vcpu_runstate_info, time) - sizeof(u64)); BUILD_BUG_ON(offsetof(struct compat_vcpu_runstate_info, state_entry_time) != offsetof(struct compat_vcpu_runstate_info, time) - sizeof(u64)); - BUILD_BUG_ON(sizeof(((struct vcpu_runstate_info *)0)->time) != - sizeof(((struct compat_vcpu_runstate_info *)0)->time)); - BUILD_BUG_ON(sizeof(((struct vcpu_runstate_info *)0)->time) != + BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) != + sizeof_field(struct compat_vcpu_runstate_info, time)); + BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) != sizeof(vx->runstate_times)); if (__copy_to_user(user_times + 1, vx->runstate_times, sizeof(vx->runstate_times))) @@ -209,6 +249,8 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) int __kvm_xen_has_interrupt(struct kvm_vcpu *v) { + unsigned long evtchn_pending_sel = READ_ONCE(v->arch.xen.evtchn_pending_sel); + bool atomic = in_atomic() || !task_is_running(current); int err; u8 rc = 0; @@ -218,23 +260,25 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v) */ struct gfn_to_hva_cache *ghc = &v->arch.xen.vcpu_info_cache; struct kvm_memslots *slots = kvm_memslots(v->kvm); + bool ghc_valid = slots->generation == ghc->generation && + !kvm_is_error_hva(ghc->hva) && ghc->memslot; + unsigned int offset = offsetof(struct vcpu_info, evtchn_upcall_pending); /* No need for compat handling here */ BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) != offsetof(struct compat_vcpu_info, evtchn_upcall_pending)); BUILD_BUG_ON(sizeof(rc) != - sizeof(((struct vcpu_info *)0)->evtchn_upcall_pending)); + sizeof_field(struct vcpu_info, evtchn_upcall_pending)); BUILD_BUG_ON(sizeof(rc) != - sizeof(((struct compat_vcpu_info *)0)->evtchn_upcall_pending)); + sizeof_field(struct compat_vcpu_info, evtchn_upcall_pending)); /* * For efficiency, this mirrors the checks for using the valid * cache in kvm_read_guest_offset_cached(), but just uses * __get_user() instead. And falls back to the slow path. */ - if (likely(slots->generation == ghc->generation && - !kvm_is_error_hva(ghc->hva) && ghc->memslot)) { + if (!evtchn_pending_sel && ghc_valid) { /* Fast path */ pagefault_disable(); err = __get_user(rc, (u8 __user *)ghc->hva + offset); @@ -253,11 +297,76 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v) * and we'll end up getting called again from a context where we *can* * fault in the page and wait for it. */ - if (in_atomic() || !task_is_running(current)) + if (atomic) return 1; - kvm_read_guest_offset_cached(v->kvm, ghc, &rc, offset, - sizeof(rc)); + if (!ghc_valid) { + err = kvm_gfn_to_hva_cache_init(v->kvm, ghc, ghc->gpa, ghc->len); + if (err || !ghc->memslot) { + /* + * If this failed, userspace has screwed up the + * vcpu_info mapping. No interrupts for you. + */ + return 0; + } + } + + /* + * Now we have a valid (protected by srcu) userspace HVA in + * ghc->hva which points to the struct vcpu_info. If there + * are any bits in the in-kernel evtchn_pending_sel then + * we need to write those to the guest vcpu_info and set + * its evtchn_upcall_pending flag. If there aren't any bits + * to add, we only want to *check* evtchn_upcall_pending. + */ + if (evtchn_pending_sel) { + bool long_mode = v->kvm->arch.xen.long_mode; + + if (!user_access_begin((void __user *)ghc->hva, sizeof(struct vcpu_info))) + return 0; + + if (IS_ENABLED(CONFIG_64BIT) && long_mode) { + struct vcpu_info __user *vi = (void __user *)ghc->hva; + + /* Attempt to set the evtchn_pending_sel bits in the + * guest, and if that succeeds then clear the same + * bits in the in-kernel version. */ + asm volatile("1:\t" LOCK_PREFIX "orq %0, %1\n" + "\tnotq %0\n" + "\t" LOCK_PREFIX "andq %0, %2\n" + "2:\n" + _ASM_EXTABLE_UA(1b, 2b) + : "=r" (evtchn_pending_sel), + "+m" (vi->evtchn_pending_sel), + "+m" (v->arch.xen.evtchn_pending_sel) + : "0" (evtchn_pending_sel)); + } else { + struct compat_vcpu_info __user *vi = (void __user *)ghc->hva; + u32 evtchn_pending_sel32 = evtchn_pending_sel; + + /* Attempt to set the evtchn_pending_sel bits in the + * guest, and if that succeeds then clear the same + * bits in the in-kernel version. */ + asm volatile("1:\t" LOCK_PREFIX "orl %0, %1\n" + "\tnotl %0\n" + "\t" LOCK_PREFIX "andl %0, %2\n" + "2:\n" + _ASM_EXTABLE_UA(1b, 2b) + : "=r" (evtchn_pending_sel32), + "+m" (vi->evtchn_pending_sel), + "+m" (v->arch.xen.evtchn_pending_sel) + : "0" (evtchn_pending_sel32)); + } + rc = 1; + unsafe_put_user(rc, (u8 __user *)ghc->hva + offset, err); + + err: + user_access_end(); + + mark_page_dirty_in_slot(v->kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT); + } else { + __get_user(rc, (u8 __user *)ghc->hva + offset); + } return rc; } @@ -279,15 +388,9 @@ int kvm_xen_hvm_set_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data) break; case KVM_XEN_ATTR_TYPE_SHARED_INFO: - if (data->u.shared_info.gfn == GPA_INVALID) { - kvm->arch.xen.shinfo_gfn = GPA_INVALID; - r = 0; - break; - } r = kvm_xen_shared_info_init(kvm, data->u.shared_info.gfn); break; - case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR: if (data->u.vector && data->u.vector < 0x10) r = -EINVAL; @@ -318,7 +421,10 @@ int kvm_xen_hvm_get_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data) break; case KVM_XEN_ATTR_TYPE_SHARED_INFO: - data->u.shared_info.gfn = kvm->arch.xen.shinfo_gfn; + if (kvm->arch.xen.shinfo_cache.active) + data->u.shared_info.gfn = gpa_to_gfn(kvm->arch.xen.shinfo_cache.gpa); + else + data->u.shared_info.gfn = GPA_INVALID; r = 0; break; @@ -698,11 +804,12 @@ int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc) void kvm_xen_init_vm(struct kvm *kvm) { - kvm->arch.xen.shinfo_gfn = GPA_INVALID; } void kvm_xen_destroy_vm(struct kvm *kvm) { + kvm_gfn_to_pfn_cache_destroy(kvm, &kvm->arch.xen.shinfo_cache); + if (kvm->arch.xen_hvm_config.msr) static_branch_slow_dec_deferred(&kvm_xen_enabled); } @@ -774,3 +881,179 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) return 0; } + +static inline int max_evtchn_port(struct kvm *kvm) +{ + if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) + return EVTCHN_2L_NR_CHANNELS; + else + return COMPAT_EVTCHN_2L_NR_CHANNELS; +} + +/* + * This follows the kvm_set_irq() API, so it returns: + * < 0 Interrupt was ignored (masked or not delivered for other reasons) + * = 0 Interrupt was coalesced (previous irq is still pending) + * > 0 Number of CPUs interrupt was delivered to + */ +int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm) +{ + struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; + struct kvm_vcpu *vcpu; + unsigned long *pending_bits, *mask_bits; + unsigned long flags; + int port_word_bit; + bool kick_vcpu = false; + int idx; + int rc; + + vcpu = kvm_get_vcpu_by_id(kvm, e->xen_evtchn.vcpu); + if (!vcpu) + return -1; + + if (!vcpu->arch.xen.vcpu_info_set) + return -1; + + if (e->xen_evtchn.port >= max_evtchn_port(kvm)) + return -1; + + rc = -EWOULDBLOCK; + read_lock_irqsave(&gpc->lock, flags); + + idx = srcu_read_lock(&kvm->srcu); + if (!kvm_gfn_to_pfn_cache_check(kvm, gpc, gpc->gpa, PAGE_SIZE)) + goto out_rcu; + + if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + struct shared_info *shinfo = gpc->khva; + pending_bits = (unsigned long *)&shinfo->evtchn_pending; + mask_bits = (unsigned long *)&shinfo->evtchn_mask; + port_word_bit = e->xen_evtchn.port / 64; + } else { + struct compat_shared_info *shinfo = gpc->khva; + pending_bits = (unsigned long *)&shinfo->evtchn_pending; + mask_bits = (unsigned long *)&shinfo->evtchn_mask; + port_word_bit = e->xen_evtchn.port / 32; + } + + /* + * If this port wasn't already set, and if it isn't masked, then + * we try to set the corresponding bit in the in-kernel shadow of + * evtchn_pending_sel for the target vCPU. And if *that* wasn't + * already set, then we kick the vCPU in question to write to the + * *real* evtchn_pending_sel in its own guest vcpu_info struct. + */ + if (test_and_set_bit(e->xen_evtchn.port, pending_bits)) { + rc = 0; /* It was already raised */ + } else if (test_bit(e->xen_evtchn.port, mask_bits)) { + rc = -1; /* Masked */ + } else { + rc = 1; /* Delivered. But was the vCPU waking already? */ + if (!test_and_set_bit(port_word_bit, &vcpu->arch.xen.evtchn_pending_sel)) + kick_vcpu = true; + } + + out_rcu: + srcu_read_unlock(&kvm->srcu, idx); + read_unlock_irqrestore(&gpc->lock, flags); + + if (kick_vcpu) { + kvm_make_request(KVM_REQ_EVENT, vcpu); + kvm_vcpu_kick(vcpu); + } + + return rc; +} + +/* This is the version called from kvm_set_irq() as the .set function */ +static int evtchn_set_fn(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, + int irq_source_id, int level, bool line_status) +{ + bool mm_borrowed = false; + int rc; + + if (!level) + return -1; + + rc = kvm_xen_set_evtchn_fast(e, kvm); + if (rc != -EWOULDBLOCK) + return rc; + + if (current->mm != kvm->mm) { + /* + * If not on a thread which already belongs to this KVM, + * we'd better be in the irqfd workqueue. + */ + if (WARN_ON_ONCE(current->mm)) + return -EINVAL; + + kthread_use_mm(kvm->mm); + mm_borrowed = true; + } + + /* + * For the irqfd workqueue, using the main kvm->lock mutex is + * fine since this function is invoked from kvm_set_irq() with + * no other lock held, no srcu. In future if it will be called + * directly from a vCPU thread (e.g. on hypercall for an IPI) + * then it may need to switch to using a leaf-node mutex for + * serializing the shared_info mapping. + */ + mutex_lock(&kvm->lock); + + /* + * It is theoretically possible for the page to be unmapped + * and the MMU notifier to invalidate the shared_info before + * we even get to use it. In that case, this looks like an + * infinite loop. It was tempting to do it via the userspace + * HVA instead... but that just *hides* the fact that it's + * an infinite loop, because if a fault occurs and it waits + * for the page to come back, it can *still* immediately + * fault and have to wait again, repeatedly. + * + * Conversely, the page could also have been reinstated by + * another thread before we even obtain the mutex above, so + * check again *first* before remapping it. + */ + do { + struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; + int idx; + + rc = kvm_xen_set_evtchn_fast(e, kvm); + if (rc != -EWOULDBLOCK) + break; + + idx = srcu_read_lock(&kvm->srcu); + rc = kvm_gfn_to_pfn_cache_refresh(kvm, gpc, gpc->gpa, + PAGE_SIZE, false); + srcu_read_unlock(&kvm->srcu, idx); + } while(!rc); + + mutex_unlock(&kvm->lock); + + if (mm_borrowed) + kthread_unuse_mm(kvm->mm); + + return rc; +} + +int kvm_xen_setup_evtchn(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue) + +{ + if (ue->u.xen_evtchn.port >= max_evtchn_port(kvm)) + return -EINVAL; + + /* We only support 2 level event channels for now */ + if (ue->u.xen_evtchn.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL) + return -EINVAL; + + e->xen_evtchn.port = ue->u.xen_evtchn.port; + e->xen_evtchn.vcpu = ue->u.xen_evtchn.vcpu; + e->xen_evtchn.priority = ue->u.xen_evtchn.priority; + e->set = evtchn_set_fn; + + return 0; +} diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index cc0cf5f374..adbcc9ed59 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -24,6 +24,12 @@ int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc); void kvm_xen_init_vm(struct kvm *kvm); void kvm_xen_destroy_vm(struct kvm *kvm); +int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm); +int kvm_xen_setup_evtchn(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue); + static inline bool kvm_xen_msr_enabled(struct kvm *kvm) { return static_branch_unlikely(&kvm_xen_enabled.key) && @@ -134,6 +140,9 @@ struct compat_shared_info { struct compat_arch_shared_info arch; }; +#define COMPAT_EVTCHN_2L_NR_CHANNELS (8 * \ + sizeof_field(struct compat_shared_info, \ + evtchn_pending)) struct compat_vcpu_runstate_info { int state; uint64_t state_entry_time; diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index c6506c6a70..f76747862b 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -63,7 +63,6 @@ ifeq ($(CONFIG_X86_32),y) ifneq ($(CONFIG_X86_CMPXCHG64),y) lib-y += cmpxchg8b_emu.o atomic64_386_32.o endif - lib-$(CONFIG_X86_USE_3DNOW) += mmx_32.o else obj-y += iomap_copy_64.o lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o diff --git a/arch/x86/lib/atomic64_386_32.S b/arch/x86/lib/atomic64_386_32.S index 16bc9130e7..e768815e58 100644 --- a/arch/x86/lib/atomic64_386_32.S +++ b/arch/x86/lib/atomic64_386_32.S @@ -9,81 +9,83 @@ #include /* if you want SMP support, implement these with real spinlocks */ -.macro LOCK reg +.macro IRQ_SAVE reg pushfl cli .endm -.macro UNLOCK reg +.macro IRQ_RESTORE reg popfl .endm -#define BEGIN(op) \ +#define BEGIN_IRQ_SAVE(op) \ .macro endp; \ SYM_FUNC_END(atomic64_##op##_386); \ .purgem endp; \ .endm; \ SYM_FUNC_START(atomic64_##op##_386); \ - LOCK v; + IRQ_SAVE v; #define ENDP endp -#define RET \ - UNLOCK v; \ - ret - -#define RET_ENDP \ - RET; \ - ENDP +#define RET_IRQ_RESTORE \ + IRQ_RESTORE v; \ + RET #define v %ecx -BEGIN(read) +BEGIN_IRQ_SAVE(read) movl (v), %eax movl 4(v), %edx -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(set) +BEGIN_IRQ_SAVE(set) movl %ebx, (v) movl %ecx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(xchg) +BEGIN_IRQ_SAVE(xchg) movl (v), %eax movl 4(v), %edx movl %ebx, (v) movl %ecx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(add) +BEGIN_IRQ_SAVE(add) addl %eax, (v) adcl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(add_return) +BEGIN_IRQ_SAVE(add_return) addl (v), %eax adcl 4(v), %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(sub) +BEGIN_IRQ_SAVE(sub) subl %eax, (v) sbbl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(sub_return) +BEGIN_IRQ_SAVE(sub_return) negl %edx negl %eax sbbl $0, %edx @@ -91,47 +93,52 @@ BEGIN(sub_return) adcl 4(v), %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(inc) +BEGIN_IRQ_SAVE(inc) addl $1, (v) adcl $0, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(inc_return) +BEGIN_IRQ_SAVE(inc_return) movl (v), %eax movl 4(v), %edx addl $1, %eax adcl $0, %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(dec) +BEGIN_IRQ_SAVE(dec) subl $1, (v) sbbl $0, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(dec_return) +BEGIN_IRQ_SAVE(dec_return) movl (v), %eax movl 4(v), %edx subl $1, %eax sbbl $0, %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(add_unless) +BEGIN_IRQ_SAVE(add_unless) addl %eax, %ecx adcl %edx, %edi addl (v), %eax @@ -143,7 +150,7 @@ BEGIN(add_unless) movl %edx, 4(v) movl $1, %eax 2: - RET + RET_IRQ_RESTORE 3: cmpl %edx, %edi jne 1b @@ -153,7 +160,7 @@ ENDP #undef v #define v %esi -BEGIN(inc_not_zero) +BEGIN_IRQ_SAVE(inc_not_zero) movl (v), %eax movl 4(v), %edx testl %eax, %eax @@ -165,7 +172,7 @@ BEGIN(inc_not_zero) movl %edx, 4(v) movl $1, %eax 2: - RET + RET_IRQ_RESTORE 3: testl %edx, %edx jne 1b @@ -174,7 +181,7 @@ ENDP #undef v #define v %esi -BEGIN(dec_if_positive) +BEGIN_IRQ_SAVE(dec_if_positive) movl (v), %eax movl 4(v), %edx subl $1, %eax @@ -183,5 +190,6 @@ BEGIN(dec_if_positive) movl %eax, (v) movl %edx, 4(v) 1: -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S index 4304320e51..23318c338d 100644 --- a/arch/x86/lib/checksum_32.S +++ b/arch/x86/lib/checksum_32.S @@ -127,7 +127,7 @@ SYM_FUNC_START(csum_partial) 8: popl %ebx popl %esi - ret + RET SYM_FUNC_END(csum_partial) #else @@ -245,7 +245,7 @@ SYM_FUNC_START(csum_partial) 90: popl %ebx popl %esi - ret + RET SYM_FUNC_END(csum_partial) #endif @@ -260,9 +260,9 @@ unsigned int csum_partial_copy_generic (const char *src, char *dst, * Copy from ds while checksumming, otherwise like csum_partial */ -#define EXC(y...) \ - 9999: y; \ - _ASM_EXTABLE_UA(9999b, 6001f) +#define EXC(y...) \ + 9999: y; \ + _ASM_EXTABLE_TYPE(9999b, 7f, EX_TYPE_UACCESS | EX_FLAG_CLEAR_AX) #ifndef CONFIG_X86_USE_PPRO_CHECKSUM @@ -358,20 +358,11 @@ EXC( movb %cl, (%edi) ) adcl $0, %eax 7: -# Exception handler: -.section .fixup, "ax" - -6001: - xorl %eax, %eax - jmp 7b - -.previous - popl %ebx popl %esi popl %edi popl %ecx # equivalent to addl $4,%esp - ret + RET SYM_FUNC_END(csum_partial_copy_generic) #else @@ -439,15 +430,11 @@ EXC( movb %dl, (%edi) ) 6: addl %edx, %eax adcl $0, %eax 7: -.section .fixup, "ax" -6001: xorl %eax, %eax - jmp 7b -.previous popl %esi popl %edi popl %ebx - ret + RET SYM_FUNC_END(csum_partial_copy_generic) #undef ROUND diff --git a/arch/x86/lib/copy_mc_64.S b/arch/x86/lib/copy_mc_64.S index e5f77e2930..c859a8a098 100644 --- a/arch/x86/lib/copy_mc_64.S +++ b/arch/x86/lib/copy_mc_64.S @@ -77,10 +77,8 @@ SYM_FUNC_START(copy_mc_fragile) .L_done_memcpy_trap: xorl %eax, %eax .L_done: - ret -SYM_FUNC_END(copy_mc_fragile) + RET - .section .fixup, "ax" /* * Return number of bytes not copied for any failure. Note that * there is no "tail" handling since the source buffer is 8-byte @@ -105,14 +103,14 @@ SYM_FUNC_END(copy_mc_fragile) movl %ecx, %edx jmp copy_mc_fragile_handle_tail - .previous - - _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes) - _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words) - _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes) + _ASM_EXTABLE_TYPE(.L_read_leading_bytes, .E_leading_bytes, EX_TYPE_DEFAULT_MCE_SAFE) + _ASM_EXTABLE_TYPE(.L_read_words, .E_read_words, EX_TYPE_DEFAULT_MCE_SAFE) + _ASM_EXTABLE_TYPE(.L_read_trailing_bytes, .E_trailing_bytes, EX_TYPE_DEFAULT_MCE_SAFE) _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes) _ASM_EXTABLE(.L_write_words, .E_write_words) _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes) + +SYM_FUNC_END(copy_mc_fragile) #endif /* CONFIG_X86_MCE */ /* @@ -132,10 +130,8 @@ SYM_FUNC_START(copy_mc_enhanced_fast_string) rep movsb /* Copy successful. Return zero */ xorl %eax, %eax - ret -SYM_FUNC_END(copy_mc_enhanced_fast_string) + RET - .section .fixup, "ax" .E_copy: /* * On fault %rcx is updated such that the copy instruction could @@ -145,9 +141,9 @@ SYM_FUNC_END(copy_mc_enhanced_fast_string) * user-copy routines. */ movq %rcx, %rax - ret + RET - .previous + _ASM_EXTABLE_TYPE(.L_copy, .E_copy, EX_TYPE_DEFAULT_MCE_SAFE) - _ASM_EXTABLE_FAULT(.L_copy, .E_copy) +SYM_FUNC_END(copy_mc_enhanced_fast_string) #endif /* !CONFIG_UML */ diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S index 57b79c5774..8ca5ecf16d 100644 --- a/arch/x86/lib/copy_user_64.S +++ b/arch/x86/lib/copy_user_64.S @@ -32,14 +32,10 @@ decl %ecx jnz 100b 102: - .section .fixup,"ax" -103: addl %ecx,%edx /* ecx is zerorest also */ - jmp .Lcopy_user_handle_tail - .previous - _ASM_EXTABLE_CPY(100b, 103b) - _ASM_EXTABLE_CPY(101b, 103b) - .endm + _ASM_EXTABLE_CPY(100b, .Lcopy_user_handle_align) + _ASM_EXTABLE_CPY(101b, .Lcopy_user_handle_align) +.endm /* * copy_user_generic_unrolled - memory copy with exception handling. @@ -105,9 +101,8 @@ SYM_FUNC_START(copy_user_generic_unrolled) jnz 21b 23: xor %eax,%eax ASM_CLAC - ret + RET - .section .fixup,"ax" 30: shll $6,%ecx addl %ecx,%edx jmp 60f @@ -115,7 +110,6 @@ SYM_FUNC_START(copy_user_generic_unrolled) jmp 60f 50: movl %ecx,%edx 60: jmp .Lcopy_user_handle_tail /* ecx is zerorest also */ - .previous _ASM_EXTABLE_CPY(1b, 30b) _ASM_EXTABLE_CPY(2b, 30b) @@ -166,20 +160,16 @@ SYM_FUNC_START(copy_user_generic_string) movl %edx,%ecx shrl $3,%ecx andl $7,%edx -1: rep - movsq +1: rep movsq 2: movl %edx,%ecx -3: rep - movsb +3: rep movsb xorl %eax,%eax ASM_CLAC - ret + RET - .section .fixup,"ax" 11: leal (%rdx,%rcx,8),%ecx 12: movl %ecx,%edx /* ecx is zerorest also */ jmp .Lcopy_user_handle_tail - .previous _ASM_EXTABLE_CPY(1b, 11b) _ASM_EXTABLE_CPY(3b, 12b) @@ -200,19 +190,16 @@ EXPORT_SYMBOL(copy_user_generic_string) */ SYM_FUNC_START(copy_user_enhanced_fast_string) ASM_STAC - cmpl $64,%edx - jb .L_copy_short_string /* less then 64 bytes, avoid the costly 'rep' */ + /* CPUs without FSRM should avoid rep movsb for short copies */ + ALTERNATIVE "cmpl $64, %edx; jb .L_copy_short_string", "", X86_FEATURE_FSRM movl %edx,%ecx -1: rep - movsb +1: rep movsb xorl %eax,%eax ASM_CLAC - ret + RET - .section .fixup,"ax" 12: movl %ecx,%edx /* ecx is zerorest also */ jmp .Lcopy_user_handle_tail - .previous _ASM_EXTABLE_CPY(1b, 12b) SYM_FUNC_END(copy_user_enhanced_fast_string) @@ -225,6 +212,7 @@ EXPORT_SYMBOL(copy_user_enhanced_fast_string) * Don't try to copy the tail if machine check happened * * Input: + * eax trap number written by ex_handler_copy() * rdi destination * rsi source * rdx count @@ -233,26 +221,26 @@ EXPORT_SYMBOL(copy_user_enhanced_fast_string) * eax uncopied bytes or 0 if successful. */ SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail) - movl %edx,%ecx - cmp $X86_TRAP_MC,%eax /* check if X86_TRAP_MC */ + cmp $X86_TRAP_MC,%eax je 3f + + movl %edx,%ecx 1: rep movsb 2: mov %ecx,%eax ASM_CLAC - ret + RET - /* - * Return zero to pretend that this copy succeeded. This - * is counter-intuitive, but needed to prevent the code - * in lib/iov_iter.c from retrying and running back into - * the poison cache line again. The machine check handler - * will ensure that a SIGBUS is sent to the task. - */ -3: xorl %eax,%eax +3: + movl %edx,%eax ASM_CLAC - ret + RET _ASM_EXTABLE_CPY(1b, 2b) + +.Lcopy_user_handle_align: + addl %ecx,%edx /* ecx is zerorest also */ + jmp .Lcopy_user_handle_tail + SYM_CODE_END(.Lcopy_user_handle_tail) /* @@ -361,9 +349,8 @@ SYM_FUNC_START(__copy_user_nocache) xorl %eax,%eax ASM_CLAC sfence - ret + RET - .section .fixup,"ax" .L_fixup_4x8b_copy: shll $6,%ecx addl %ecx,%edx @@ -379,7 +366,6 @@ SYM_FUNC_START(__copy_user_nocache) .L_fixup_handle_tail: sfence jmp .Lcopy_user_handle_tail - .previous _ASM_EXTABLE_CPY(1b, .L_fixup_4x8b_copy) _ASM_EXTABLE_CPY(2b, .L_fixup_4x8b_copy) diff --git a/arch/x86/lib/csum-partial_64.c b/arch/x86/lib/csum-partial_64.c index e7925d668b..1f8a8f8951 100644 --- a/arch/x86/lib/csum-partial_64.c +++ b/arch/x86/lib/csum-partial_64.c @@ -9,6 +9,7 @@ #include #include #include +#include static inline unsigned short from32to16(unsigned a) { @@ -21,120 +22,119 @@ static inline unsigned short from32to16(unsigned a) } /* - * Do a 64-bit checksum on an arbitrary memory area. + * Do a checksum on an arbitrary memory area. * Returns a 32bit checksum. * * This isn't as time critical as it used to be because many NICs * do hardware checksumming these days. - * - * Things tried and found to not make it faster: - * Manual Prefetching - * Unrolling to an 128 bytes inner loop. - * Using interleaving with more registers to break the carry chains. - */ -static unsigned do_csum(const unsigned char *buff, unsigned len) -{ - unsigned odd, count; - unsigned long result = 0; - - if (unlikely(len == 0)) - return result; - odd = 1 & (unsigned long) buff; - if (unlikely(odd)) { - result = *buff << 8; - len--; - buff++; - } - count = len >> 1; /* nr of 16-bit words.. */ - if (count) { - if (2 & (unsigned long) buff) { - result += *(unsigned short *)buff; - count--; - len -= 2; - buff += 2; - } - count >>= 1; /* nr of 32-bit words.. */ - if (count) { - unsigned long zero; - unsigned count64; - if (4 & (unsigned long) buff) { - result += *(unsigned int *) buff; - count--; - len -= 4; - buff += 4; - } - count >>= 1; /* nr of 64-bit words.. */ - - /* main loop using 64byte blocks */ - zero = 0; - count64 = count >> 3; - while (count64) { - asm("addq 0*8(%[src]),%[res]\n\t" - "adcq 1*8(%[src]),%[res]\n\t" - "adcq 2*8(%[src]),%[res]\n\t" - "adcq 3*8(%[src]),%[res]\n\t" - "adcq 4*8(%[src]),%[res]\n\t" - "adcq 5*8(%[src]),%[res]\n\t" - "adcq 6*8(%[src]),%[res]\n\t" - "adcq 7*8(%[src]),%[res]\n\t" - "adcq %[zero],%[res]" - : [res] "=r" (result) - : [src] "r" (buff), [zero] "r" (zero), - "[res]" (result)); - buff += 64; - count64--; - } - - /* last up to 7 8byte blocks */ - count %= 8; - while (count) { - asm("addq %1,%0\n\t" - "adcq %2,%0\n" - : "=r" (result) - : "m" (*(unsigned long *)buff), - "r" (zero), "0" (result)); - --count; - buff += 8; - } - result = add32_with_carry(result>>32, - result&0xffffffff); - - if (len & 4) { - result += *(unsigned int *) buff; - buff += 4; - } - } - if (len & 2) { - result += *(unsigned short *) buff; - buff += 2; - } - } - if (len & 1) - result += *buff; - result = add32_with_carry(result>>32, result & 0xffffffff); - if (unlikely(odd)) { - result = from32to16(result); - result = ((result >> 8) & 0xff) | ((result & 0xff) << 8); - } - return result; -} - -/* - * computes the checksum of a memory block at buff, length len, - * and adds in "sum" (32-bit) - * - * returns a 32-bit number suitable for feeding into itself - * or csum_tcpudp_magic - * - * this function must be called with even lengths, except - * for the last fragment, which may be odd * + * Still, with CHECKSUM_COMPLETE this is called to compute + * checksums on IPv6 headers (40 bytes) and other small parts. * it's best to have buff aligned on a 64-bit boundary */ __wsum csum_partial(const void *buff, int len, __wsum sum) { - return (__force __wsum)add32_with_carry(do_csum(buff, len), - (__force u32)sum); + u64 temp64 = (__force u64)sum; + unsigned odd, result; + + odd = 1 & (unsigned long) buff; + if (unlikely(odd)) { + if (unlikely(len == 0)) + return sum; + temp64 = ror32((__force u32)sum, 8); + temp64 += (*(unsigned char *)buff << 8); + len--; + buff++; + } + + while (unlikely(len >= 64)) { + asm("addq 0*8(%[src]),%[res]\n\t" + "adcq 1*8(%[src]),%[res]\n\t" + "adcq 2*8(%[src]),%[res]\n\t" + "adcq 3*8(%[src]),%[res]\n\t" + "adcq 4*8(%[src]),%[res]\n\t" + "adcq 5*8(%[src]),%[res]\n\t" + "adcq 6*8(%[src]),%[res]\n\t" + "adcq 7*8(%[src]),%[res]\n\t" + "adcq $0,%[res]" + : [res] "+r" (temp64) + : [src] "r" (buff) + : "memory"); + buff += 64; + len -= 64; + } + + if (len & 32) { + asm("addq 0*8(%[src]),%[res]\n\t" + "adcq 1*8(%[src]),%[res]\n\t" + "adcq 2*8(%[src]),%[res]\n\t" + "adcq 3*8(%[src]),%[res]\n\t" + "adcq $0,%[res]" + : [res] "+r" (temp64) + : [src] "r" (buff) + : "memory"); + buff += 32; + } + if (len & 16) { + asm("addq 0*8(%[src]),%[res]\n\t" + "adcq 1*8(%[src]),%[res]\n\t" + "adcq $0,%[res]" + : [res] "+r" (temp64) + : [src] "r" (buff) + : "memory"); + buff += 16; + } + if (len & 8) { + asm("addq 0*8(%[src]),%[res]\n\t" + "adcq $0,%[res]" + : [res] "+r" (temp64) + : [src] "r" (buff) + : "memory"); + buff += 8; + } + if (len & 7) { +#ifdef CONFIG_DCACHE_WORD_ACCESS + unsigned int shift = (8 - (len & 7)) * 8; + unsigned long trail; + + trail = (load_unaligned_zeropad(buff) << shift) >> shift; + + asm("addq %[trail],%[res]\n\t" + "adcq $0,%[res]" + : [res] "+r" (temp64) + : [trail] "r" (trail)); +#else + if (len & 4) { + asm("addq %[val],%[res]\n\t" + "adcq $0,%[res]" + : [res] "+r" (temp64) + : [val] "r" ((u64)*(u32 *)buff) + : "memory"); + buff += 4; + } + if (len & 2) { + asm("addq %[val],%[res]\n\t" + "adcq $0,%[res]" + : [res] "+r" (temp64) + : [val] "r" ((u64)*(u16 *)buff) + : "memory"); + buff += 2; + } + if (len & 1) { + asm("addq %[val],%[res]\n\t" + "adcq $0,%[res]" + : [res] "+r" (temp64) + : [val] "r" ((u64)*(u8 *)buff) + : "memory"); + } +#endif + } + result = add32_with_carry(temp64 >> 32, temp64 & 0xffffffff); + if (unlikely(odd)) { + result = from32to16(result); + result = ((result >> 8) & 0xff) | ((result & 0xff) << 8); + } + return (__force __wsum)result; } EXPORT_SYMBOL(csum_partial); @@ -147,4 +147,3 @@ __sum16 ip_compute_csum(const void *buff, int len) return csum_fold(csum_partial(buff,len,0)); } EXPORT_SYMBOL(ip_compute_csum); - diff --git a/arch/x86/lib/error-inject.c b/arch/x86/lib/error-inject.c index be5b5fb159..520897061e 100644 --- a/arch/x86/lib/error-inject.c +++ b/arch/x86/lib/error-inject.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include @@ -10,7 +11,7 @@ asm( ".type just_return_func, @function\n" ".globl just_return_func\n" "just_return_func:\n" - " ret\n" + ASM_RET ".size just_return_func, .-just_return_func\n" ); diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index eb3ccffb9b..b781d32421 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -37,8 +37,6 @@ enum reg_type { */ static bool is_string_insn(struct insn *insn) { - insn_get_opcode(insn); - /* All string instructions have a 1-byte opcode. */ if (insn->opcode.nbytes != 1) return false; @@ -412,32 +410,44 @@ static short get_segment_selector(struct pt_regs *regs, int seg_reg_idx) #endif /* CONFIG_X86_64 */ } -static int get_reg_offset(struct insn *insn, struct pt_regs *regs, - enum reg_type type) +static const int pt_regoff[] = { + offsetof(struct pt_regs, ax), + offsetof(struct pt_regs, cx), + offsetof(struct pt_regs, dx), + offsetof(struct pt_regs, bx), + offsetof(struct pt_regs, sp), + offsetof(struct pt_regs, bp), + offsetof(struct pt_regs, si), + offsetof(struct pt_regs, di), +#ifdef CONFIG_X86_64 + offsetof(struct pt_regs, r8), + offsetof(struct pt_regs, r9), + offsetof(struct pt_regs, r10), + offsetof(struct pt_regs, r11), + offsetof(struct pt_regs, r12), + offsetof(struct pt_regs, r13), + offsetof(struct pt_regs, r14), + offsetof(struct pt_regs, r15), +#else + offsetof(struct pt_regs, ds), + offsetof(struct pt_regs, es), + offsetof(struct pt_regs, fs), + offsetof(struct pt_regs, gs), +#endif +}; + +int pt_regs_offset(struct pt_regs *regs, int regno) { + if ((unsigned)regno < ARRAY_SIZE(pt_regoff)) + return pt_regoff[regno]; + return -EDOM; +} + +static int get_regno(struct insn *insn, enum reg_type type) +{ + int nr_registers = ARRAY_SIZE(pt_regoff); int regno = 0; - static const int regoff[] = { - offsetof(struct pt_regs, ax), - offsetof(struct pt_regs, cx), - offsetof(struct pt_regs, dx), - offsetof(struct pt_regs, bx), - offsetof(struct pt_regs, sp), - offsetof(struct pt_regs, bp), - offsetof(struct pt_regs, si), - offsetof(struct pt_regs, di), -#ifdef CONFIG_X86_64 - offsetof(struct pt_regs, r8), - offsetof(struct pt_regs, r9), - offsetof(struct pt_regs, r10), - offsetof(struct pt_regs, r11), - offsetof(struct pt_regs, r12), - offsetof(struct pt_regs, r13), - offsetof(struct pt_regs, r14), - offsetof(struct pt_regs, r15), -#endif - }; - int nr_registers = ARRAY_SIZE(regoff); /* * Don't possibly decode a 32-bit instructions as * reading a 64-bit-only register. @@ -505,7 +515,18 @@ static int get_reg_offset(struct insn *insn, struct pt_regs *regs, WARN_ONCE(1, "decoded an instruction with an invalid register"); return -EINVAL; } - return regoff[regno]; + return regno; +} + +static int get_reg_offset(struct insn *insn, struct pt_regs *regs, + enum reg_type type) +{ + int regno = get_regno(insn, type); + + if (regno < 0) + return regno; + + return pt_regs_offset(regs, regno); } /** @@ -850,6 +871,26 @@ int insn_get_modrm_reg_off(struct insn *insn, struct pt_regs *regs) return get_reg_offset(insn, regs, REG_TYPE_REG); } +/** + * insn_get_modrm_reg_ptr() - Obtain register pointer based on ModRM byte + * @insn: Instruction containing the ModRM byte + * @regs: Register values as seen when entering kernel mode + * + * Returns: + * + * The register indicated by the reg part of the ModRM byte. + * The register is obtained as a pointer within pt_regs. + */ +unsigned long *insn_get_modrm_reg_ptr(struct insn *insn, struct pt_regs *regs) +{ + int offset; + + offset = insn_get_modrm_reg_off(insn, regs); + if (offset < 0) + return NULL; + return (void *)regs + offset; +} + /** * get_seg_base_limit() - obtain base address and limit of a segment * @insn: Instruction. Must be valid. @@ -1405,6 +1446,9 @@ void __user *insn_get_addr_ref(struct insn *insn, struct pt_regs *regs) if (!insn || !regs) return (void __user *)-1L; + if (insn_get_opcode(insn)) + return (void __user *)-1L; + switch (insn->addr_bytes) { case 2: return get_addr_ref_16(insn, regs); @@ -1539,3 +1583,87 @@ bool insn_decode_from_regs(struct insn *insn, struct pt_regs *regs, return true; } + +/** + * insn_decode_mmio() - Decode a MMIO instruction + * @insn: Structure to store decoded instruction + * @bytes: Returns size of memory operand + * + * Decodes instruction that used for Memory-mapped I/O. + * + * Returns: + * + * Type of the instruction. Size of the memory operand is stored in + * @bytes. If decode failed, MMIO_DECODE_FAILED returned. + */ +enum mmio_type insn_decode_mmio(struct insn *insn, int *bytes) +{ + enum mmio_type type = MMIO_DECODE_FAILED; + + *bytes = 0; + + if (insn_get_opcode(insn)) + return MMIO_DECODE_FAILED; + + switch (insn->opcode.bytes[0]) { + case 0x88: /* MOV m8,r8 */ + *bytes = 1; + fallthrough; + case 0x89: /* MOV m16/m32/m64, r16/m32/m64 */ + if (!*bytes) + *bytes = insn->opnd_bytes; + type = MMIO_WRITE; + break; + + case 0xc6: /* MOV m8, imm8 */ + *bytes = 1; + fallthrough; + case 0xc7: /* MOV m16/m32/m64, imm16/imm32/imm64 */ + if (!*bytes) + *bytes = insn->opnd_bytes; + type = MMIO_WRITE_IMM; + break; + + case 0x8a: /* MOV r8, m8 */ + *bytes = 1; + fallthrough; + case 0x8b: /* MOV r16/r32/r64, m16/m32/m64 */ + if (!*bytes) + *bytes = insn->opnd_bytes; + type = MMIO_READ; + break; + + case 0xa4: /* MOVS m8, m8 */ + *bytes = 1; + fallthrough; + case 0xa5: /* MOVS m16/m32/m64, m16/m32/m64 */ + if (!*bytes) + *bytes = insn->opnd_bytes; + type = MMIO_MOVS; + break; + + case 0x0f: /* Two-byte instruction */ + switch (insn->opcode.bytes[1]) { + case 0xb6: /* MOVZX r16/r32/r64, m8 */ + *bytes = 1; + fallthrough; + case 0xb7: /* MOVZX r32/r64, m16 */ + if (!*bytes) + *bytes = 2; + type = MMIO_READ_ZERO_EXTEND; + break; + + case 0xbe: /* MOVSX r16/r32/r64, m8 */ + *bytes = 1; + fallthrough; + case 0xbf: /* MOVSX r32/r64, m16 */ + if (!*bytes) + *bytes = 2; + type = MMIO_READ_SIGN_EXTEND; + break; + } + break; + } + + return type; +} diff --git a/arch/x86/lib/kaslr.c b/arch/x86/lib/kaslr.c index a536651164..2b3eb8c948 100644 --- a/arch/x86/lib/kaslr.c +++ b/arch/x86/lib/kaslr.c @@ -56,11 +56,14 @@ unsigned long kaslr_get_random_long(const char *purpose) unsigned long raw, random = get_boot_seed(); bool use_i8254 = true; - debug_putstr(purpose); - debug_putstr(" KASLR using"); + if (purpose) { + debug_putstr(purpose); + debug_putstr(" KASLR using"); + } if (has_cpuflag(X86_FEATURE_RDRAND)) { - debug_putstr(" RDRAND"); + if (purpose) + debug_putstr(" RDRAND"); if (rdrand_long(&raw)) { random ^= raw; use_i8254 = false; @@ -68,7 +71,8 @@ unsigned long kaslr_get_random_long(const char *purpose) } if (has_cpuflag(X86_FEATURE_TSC)) { - debug_putstr(" RDTSC"); + if (purpose) + debug_putstr(" RDTSC"); raw = rdtsc(); random ^= raw; @@ -76,7 +80,8 @@ unsigned long kaslr_get_random_long(const char *purpose) } if (use_i8254) { - debug_putstr(" i8254"); + if (purpose) + debug_putstr(" i8254"); random ^= i8254(); } @@ -86,7 +91,8 @@ unsigned long kaslr_get_random_long(const char *purpose) : "a" (random), "rm" (mix_const)); random += raw; - debug_putstr("...\n"); + if (purpose) + debug_putstr("...\n"); return random; } diff --git a/arch/x86/lib/memcpy_32.c b/arch/x86/lib/memcpy_32.c index e565d1c901..3a6e6cfe8c 100644 --- a/arch/x86/lib/memcpy_32.c +++ b/arch/x86/lib/memcpy_32.c @@ -7,11 +7,7 @@ __visible void *memcpy(void *to, const void *from, size_t n) { -#if defined(CONFIG_X86_USE_3DNOW) && !defined(CONFIG_FORTIFY_SOURCE) - return __memcpy3d(to, from, n); -#else return __memcpy(to, from, n); -#endif } EXPORT_SYMBOL(memcpy); diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S index 1cc9da6e29..59cf2343f3 100644 --- a/arch/x86/lib/memcpy_64.S +++ b/arch/x86/lib/memcpy_64.S @@ -39,7 +39,7 @@ SYM_FUNC_START_WEAK(memcpy) rep movsq movl %edx, %ecx rep movsb - ret + RET SYM_FUNC_END(memcpy) SYM_FUNC_END_ALIAS(__memcpy) EXPORT_SYMBOL(memcpy) @@ -53,7 +53,7 @@ SYM_FUNC_START_LOCAL(memcpy_erms) movq %rdi, %rax movq %rdx, %rcx rep movsb - ret + RET SYM_FUNC_END(memcpy_erms) SYM_FUNC_START_LOCAL(memcpy_orig) @@ -137,7 +137,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movq %r9, 1*8(%rdi) movq %r10, -2*8(%rdi, %rdx) movq %r11, -1*8(%rdi, %rdx) - retq + RET .p2align 4 .Lless_16bytes: cmpl $8, %edx @@ -149,7 +149,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movq -1*8(%rsi, %rdx), %r9 movq %r8, 0*8(%rdi) movq %r9, -1*8(%rdi, %rdx) - retq + RET .p2align 4 .Lless_8bytes: cmpl $4, %edx @@ -162,7 +162,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movl -4(%rsi, %rdx), %r8d movl %ecx, (%rdi) movl %r8d, -4(%rdi, %rdx) - retq + RET .p2align 4 .Lless_3bytes: subl $1, %edx @@ -180,7 +180,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movb %cl, (%rdi) .Lend: - retq + RET SYM_FUNC_END(memcpy_orig) .popsection diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S index 64801010d3..50ea390df7 100644 --- a/arch/x86/lib/memmove_64.S +++ b/arch/x86/lib/memmove_64.S @@ -40,7 +40,7 @@ SYM_FUNC_START(__memmove) /* FSRM implies ERMS => no length checks, do the copy directly */ .Lmemmove_begin_forward: ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM - ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; retq", X86_FEATURE_ERMS + ALTERNATIVE "", __stringify(movq %rdx, %rcx; rep movsb; RET), X86_FEATURE_ERMS /* * movsq instruction have many startup latency @@ -205,7 +205,7 @@ SYM_FUNC_START(__memmove) movb (%rsi), %r11b movb %r11b, (%rdi) 13: - retq + RET SYM_FUNC_END(__memmove) SYM_FUNC_END_ALIAS(memmove) EXPORT_SYMBOL(__memmove) diff --git a/arch/x86/lib/mmx_32.c b/arch/x86/lib/mmx_32.c index cc5f4ea943..e69de29bb2 100644 --- a/arch/x86/lib/mmx_32.c +++ b/arch/x86/lib/mmx_32.c @@ -1,388 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * MMX 3DNow! library helper functions - * - * To do: - * We can use MMX just for prefetch in IRQ's. This may be a win. - * (reported so on K6-III) - * We should use a better code neutral filler for the short jump - * leal ebx. [ebx] is apparently best for K6-2, but Cyrix ?? - * We also want to clobber the filler register so we don't get any - * register forwarding stalls on the filler. - * - * Add *user handling. Checksums are not a win with MMX on any CPU - * tested so far for any MMX solution figured. - * - * 22/09/2000 - Arjan van de Ven - * Improved for non-engineering-sample Athlons - * - */ -#include -#include -#include -#include -#include - -#include -#include - -/* - * Use KFPU_387. MMX instructions are not affected by MXCSR, - * but both AMD and Intel documentation states that even integer MMX - * operations will result in #MF if an exception is pending in FCW. - * - * EMMS is not needed afterwards because, after calling kernel_fpu_end(), - * any subsequent user of the 387 stack will reinitialize it using - * KFPU_387. - */ - -void *_mmx_memcpy(void *to, const void *from, size_t len) -{ - void *p; - int i; - - if (unlikely(in_interrupt())) - return __memcpy(to, from, len); - - p = to; - i = len >> 6; /* len/64 */ - - kernel_fpu_begin_mask(KFPU_387); - - __asm__ __volatile__ ( - "1: prefetch (%0)\n" /* This set is 28 bytes */ - " prefetch 64(%0)\n" - " prefetch 128(%0)\n" - " prefetch 192(%0)\n" - " prefetch 256(%0)\n" - "2: \n" - ".section .fixup, \"ax\"\n" - "3: movw $0x1AEB, 1b\n" /* jmp on 26 bytes */ - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) - : : "r" (from)); - - for ( ; i > 5; i--) { - __asm__ __volatile__ ( - "1: prefetch 320(%0)\n" - "2: movq (%0), %%mm0\n" - " movq 8(%0), %%mm1\n" - " movq 16(%0), %%mm2\n" - " movq 24(%0), %%mm3\n" - " movq %%mm0, (%1)\n" - " movq %%mm1, 8(%1)\n" - " movq %%mm2, 16(%1)\n" - " movq %%mm3, 24(%1)\n" - " movq 32(%0), %%mm0\n" - " movq 40(%0), %%mm1\n" - " movq 48(%0), %%mm2\n" - " movq 56(%0), %%mm3\n" - " movq %%mm0, 32(%1)\n" - " movq %%mm1, 40(%1)\n" - " movq %%mm2, 48(%1)\n" - " movq %%mm3, 56(%1)\n" - ".section .fixup, \"ax\"\n" - "3: movw $0x05EB, 1b\n" /* jmp on 5 bytes */ - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) - : : "r" (from), "r" (to) : "memory"); - - from += 64; - to += 64; - } - - for ( ; i > 0; i--) { - __asm__ __volatile__ ( - " movq (%0), %%mm0\n" - " movq 8(%0), %%mm1\n" - " movq 16(%0), %%mm2\n" - " movq 24(%0), %%mm3\n" - " movq %%mm0, (%1)\n" - " movq %%mm1, 8(%1)\n" - " movq %%mm2, 16(%1)\n" - " movq %%mm3, 24(%1)\n" - " movq 32(%0), %%mm0\n" - " movq 40(%0), %%mm1\n" - " movq 48(%0), %%mm2\n" - " movq 56(%0), %%mm3\n" - " movq %%mm0, 32(%1)\n" - " movq %%mm1, 40(%1)\n" - " movq %%mm2, 48(%1)\n" - " movq %%mm3, 56(%1)\n" - : : "r" (from), "r" (to) : "memory"); - - from += 64; - to += 64; - } - /* - * Now do the tail of the block: - */ - __memcpy(to, from, len & 63); - kernel_fpu_end(); - - return p; -} -EXPORT_SYMBOL(_mmx_memcpy); - -#ifdef CONFIG_MK7 - -/* - * The K7 has streaming cache bypass load/store. The Cyrix III, K6 and - * other MMX using processors do not. - */ - -static void fast_clear_page(void *page) -{ - int i; - - kernel_fpu_begin_mask(KFPU_387); - - __asm__ __volatile__ ( - " pxor %%mm0, %%mm0\n" : : - ); - - for (i = 0; i < 4096/64; i++) { - __asm__ __volatile__ ( - " movntq %%mm0, (%0)\n" - " movntq %%mm0, 8(%0)\n" - " movntq %%mm0, 16(%0)\n" - " movntq %%mm0, 24(%0)\n" - " movntq %%mm0, 32(%0)\n" - " movntq %%mm0, 40(%0)\n" - " movntq %%mm0, 48(%0)\n" - " movntq %%mm0, 56(%0)\n" - : : "r" (page) : "memory"); - page += 64; - } - - /* - * Since movntq is weakly-ordered, a "sfence" is needed to become - * ordered again: - */ - __asm__ __volatile__("sfence\n"::); - - kernel_fpu_end(); -} - -static void fast_copy_page(void *to, void *from) -{ - int i; - - kernel_fpu_begin_mask(KFPU_387); - - /* - * maybe the prefetch stuff can go before the expensive fnsave... - * but that is for later. -AV - */ - __asm__ __volatile__( - "1: prefetch (%0)\n" - " prefetch 64(%0)\n" - " prefetch 128(%0)\n" - " prefetch 192(%0)\n" - " prefetch 256(%0)\n" - "2: \n" - ".section .fixup, \"ax\"\n" - "3: movw $0x1AEB, 1b\n" /* jmp on 26 bytes */ - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) : : "r" (from)); - - for (i = 0; i < (4096-320)/64; i++) { - __asm__ __volatile__ ( - "1: prefetch 320(%0)\n" - "2: movq (%0), %%mm0\n" - " movntq %%mm0, (%1)\n" - " movq 8(%0), %%mm1\n" - " movntq %%mm1, 8(%1)\n" - " movq 16(%0), %%mm2\n" - " movntq %%mm2, 16(%1)\n" - " movq 24(%0), %%mm3\n" - " movntq %%mm3, 24(%1)\n" - " movq 32(%0), %%mm4\n" - " movntq %%mm4, 32(%1)\n" - " movq 40(%0), %%mm5\n" - " movntq %%mm5, 40(%1)\n" - " movq 48(%0), %%mm6\n" - " movntq %%mm6, 48(%1)\n" - " movq 56(%0), %%mm7\n" - " movntq %%mm7, 56(%1)\n" - ".section .fixup, \"ax\"\n" - "3: movw $0x05EB, 1b\n" /* jmp on 5 bytes */ - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) : : "r" (from), "r" (to) : "memory"); - - from += 64; - to += 64; - } - - for (i = (4096-320)/64; i < 4096/64; i++) { - __asm__ __volatile__ ( - "2: movq (%0), %%mm0\n" - " movntq %%mm0, (%1)\n" - " movq 8(%0), %%mm1\n" - " movntq %%mm1, 8(%1)\n" - " movq 16(%0), %%mm2\n" - " movntq %%mm2, 16(%1)\n" - " movq 24(%0), %%mm3\n" - " movntq %%mm3, 24(%1)\n" - " movq 32(%0), %%mm4\n" - " movntq %%mm4, 32(%1)\n" - " movq 40(%0), %%mm5\n" - " movntq %%mm5, 40(%1)\n" - " movq 48(%0), %%mm6\n" - " movntq %%mm6, 48(%1)\n" - " movq 56(%0), %%mm7\n" - " movntq %%mm7, 56(%1)\n" - : : "r" (from), "r" (to) : "memory"); - from += 64; - to += 64; - } - /* - * Since movntq is weakly-ordered, a "sfence" is needed to become - * ordered again: - */ - __asm__ __volatile__("sfence \n"::); - kernel_fpu_end(); -} - -#else /* CONFIG_MK7 */ - -/* - * Generic MMX implementation without K7 specific streaming - */ -static void fast_clear_page(void *page) -{ - int i; - - kernel_fpu_begin_mask(KFPU_387); - - __asm__ __volatile__ ( - " pxor %%mm0, %%mm0\n" : : - ); - - for (i = 0; i < 4096/128; i++) { - __asm__ __volatile__ ( - " movq %%mm0, (%0)\n" - " movq %%mm0, 8(%0)\n" - " movq %%mm0, 16(%0)\n" - " movq %%mm0, 24(%0)\n" - " movq %%mm0, 32(%0)\n" - " movq %%mm0, 40(%0)\n" - " movq %%mm0, 48(%0)\n" - " movq %%mm0, 56(%0)\n" - " movq %%mm0, 64(%0)\n" - " movq %%mm0, 72(%0)\n" - " movq %%mm0, 80(%0)\n" - " movq %%mm0, 88(%0)\n" - " movq %%mm0, 96(%0)\n" - " movq %%mm0, 104(%0)\n" - " movq %%mm0, 112(%0)\n" - " movq %%mm0, 120(%0)\n" - : : "r" (page) : "memory"); - page += 128; - } - - kernel_fpu_end(); -} - -static void fast_copy_page(void *to, void *from) -{ - int i; - - kernel_fpu_begin_mask(KFPU_387); - - __asm__ __volatile__ ( - "1: prefetch (%0)\n" - " prefetch 64(%0)\n" - " prefetch 128(%0)\n" - " prefetch 192(%0)\n" - " prefetch 256(%0)\n" - "2: \n" - ".section .fixup, \"ax\"\n" - "3: movw $0x1AEB, 1b\n" /* jmp on 26 bytes */ - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) : : "r" (from)); - - for (i = 0; i < 4096/64; i++) { - __asm__ __volatile__ ( - "1: prefetch 320(%0)\n" - "2: movq (%0), %%mm0\n" - " movq 8(%0), %%mm1\n" - " movq 16(%0), %%mm2\n" - " movq 24(%0), %%mm3\n" - " movq %%mm0, (%1)\n" - " movq %%mm1, 8(%1)\n" - " movq %%mm2, 16(%1)\n" - " movq %%mm3, 24(%1)\n" - " movq 32(%0), %%mm0\n" - " movq 40(%0), %%mm1\n" - " movq 48(%0), %%mm2\n" - " movq 56(%0), %%mm3\n" - " movq %%mm0, 32(%1)\n" - " movq %%mm1, 40(%1)\n" - " movq %%mm2, 48(%1)\n" - " movq %%mm3, 56(%1)\n" - ".section .fixup, \"ax\"\n" - "3: movw $0x05EB, 1b\n" /* jmp on 5 bytes */ - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE(1b, 3b) - : : "r" (from), "r" (to) : "memory"); - - from += 64; - to += 64; - } - kernel_fpu_end(); -} - -#endif /* !CONFIG_MK7 */ - -/* - * Favour MMX for page clear and copy: - */ -static void slow_zero_page(void *page) -{ - int d0, d1; - - __asm__ __volatile__( - "cld\n\t" - "rep ; stosl" - - : "=&c" (d0), "=&D" (d1) - :"a" (0), "1" (page), "0" (1024) - :"memory"); -} - -void mmx_clear_page(void *page) -{ - if (unlikely(in_interrupt())) - slow_zero_page(page); - else - fast_clear_page(page); -} -EXPORT_SYMBOL(mmx_clear_page); - -static void slow_copy_page(void *to, void *from) -{ - int d0, d1, d2; - - __asm__ __volatile__( - "cld\n\t" - "rep ; movsl" - : "=&c" (d0), "=&D" (d1), "=&S" (d2) - : "0" (1024), "1" ((long) to), "2" ((long) from) - : "memory"); -} - -void mmx_copy_page(void *to, void *from) -{ - if (unlikely(in_interrupt())) - slow_copy_page(to, from); - else - fast_copy_page(to, from); -} -EXPORT_SYMBOL(mmx_copy_page); diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index 5385d26af6..afbdda539b 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -23,50 +23,18 @@ .Ldo_rop_\@: mov %\reg, (%_ASM_SP) UNWIND_HINT_FUNC - ret + RET .endm .macro THUNK reg - .align 32 - -SYM_FUNC_START(__x86_indirect_thunk_\reg) + .align RETPOLINE_THUNK_SIZE +SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) + UNWIND_HINT_EMPTY ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE - -SYM_FUNC_END(__x86_indirect_thunk_\reg) - -.endm - -/* - * This generates .altinstr_replacement symbols for use by objtool. They, - * however, must not actually live in .altinstr_replacement since that will be - * discarded after init, but module alternatives will also reference these - * symbols. - * - * Their names matches the "__x86_indirect_" prefix to mark them as retpolines. - */ -.macro ALT_THUNK reg - - .align 1 - -SYM_FUNC_START_NOALIGN(__x86_indirect_alt_call_\reg) - ANNOTATE_RETPOLINE_SAFE -1: call *%\reg -2: .skip 5-(2b-1b), 0x90 -SYM_FUNC_END(__x86_indirect_alt_call_\reg) - -STACK_FRAME_NON_STANDARD(__x86_indirect_alt_call_\reg) - -SYM_FUNC_START_NOALIGN(__x86_indirect_alt_jmp_\reg) - ANNOTATE_RETPOLINE_SAFE -1: jmp *%\reg -2: .skip 5-(2b-1b), 0x90 -SYM_FUNC_END(__x86_indirect_alt_jmp_\reg) - -STACK_FRAME_NON_STANDARD(__x86_indirect_alt_jmp_\reg) + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE .endm @@ -85,22 +53,16 @@ STACK_FRAME_NON_STANDARD(__x86_indirect_alt_jmp_\reg) #define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym) #define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg) -#undef GEN + .align RETPOLINE_THUNK_SIZE +SYM_CODE_START(__x86_indirect_thunk_array) + #define GEN(reg) THUNK reg #include - #undef GEN + + .align RETPOLINE_THUNK_SIZE +SYM_CODE_END(__x86_indirect_thunk_array) + #define GEN(reg) EXPORT_THUNK(reg) #include - #undef GEN -#define GEN(reg) ALT_THUNK reg -#include - -#undef GEN -#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_call_ ## reg) -#include - -#undef GEN -#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_jmp_ ## reg) -#include diff --git a/arch/x86/lib/string_32.c b/arch/x86/lib/string_32.c index d15fdae965..53b3f20226 100644 --- a/arch/x86/lib/string_32.c +++ b/arch/x86/lib/string_32.c @@ -11,6 +11,7 @@ * strings. */ +#define __NO_FORTIFY #include #include diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c index 7d29077724..422257c350 100644 --- a/arch/x86/lib/usercopy_32.c +++ b/arch/x86/lib/usercopy_32.c @@ -8,7 +8,6 @@ */ #include #include -#include #include #ifdef CONFIG_X86_INTEL_USERCOPY @@ -43,11 +42,7 @@ do { \ " movl %2,%0\n" \ "1: rep; stosb\n" \ "2: " ASM_CLAC "\n" \ - ".section .fixup,\"ax\"\n" \ - "3: lea 0(%2,%0,4),%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE_UA(0b, 3b) \ + _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %2) \ _ASM_EXTABLE_UA(1b, 2b) \ : "=&c"(size), "=&D" (__d0) \ : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \ @@ -149,10 +144,6 @@ __copy_user_intel(void __user *to, const void *from, unsigned long size) "36: movl %%eax, %0\n" "37: rep; movsb\n" "100:\n" - ".section .fixup,\"ax\"\n" - "101: lea 0(%%eax,%0,4),%0\n" - " jmp 100b\n" - ".previous\n" _ASM_EXTABLE_UA(1b, 100b) _ASM_EXTABLE_UA(2b, 100b) _ASM_EXTABLE_UA(3b, 100b) @@ -190,7 +181,7 @@ __copy_user_intel(void __user *to, const void *from, unsigned long size) _ASM_EXTABLE_UA(35b, 100b) _ASM_EXTABLE_UA(36b, 100b) _ASM_EXTABLE_UA(37b, 100b) - _ASM_EXTABLE_UA(99b, 101b) + _ASM_EXTABLE_TYPE_REG(99b, 100b, EX_TYPE_UCOPY_LEN4, %%eax) : "=&c"(size), "=&D" (d0), "=&S" (d1) : "1"(to), "2"(from), "0"(size) : "eax", "edx", "memory"); @@ -255,30 +246,26 @@ static unsigned long __copy_user_intel_nocache(void *to, " movl %%eax,%0\n" "7: rep; movsb\n" "8:\n" - ".section .fixup,\"ax\"\n" - "9: lea 0(%%eax,%0,4),%0\n" - "16: jmp 8b\n" - ".previous\n" - _ASM_EXTABLE_UA(0b, 16b) - _ASM_EXTABLE_UA(1b, 16b) - _ASM_EXTABLE_UA(2b, 16b) - _ASM_EXTABLE_UA(21b, 16b) - _ASM_EXTABLE_UA(3b, 16b) - _ASM_EXTABLE_UA(31b, 16b) - _ASM_EXTABLE_UA(4b, 16b) - _ASM_EXTABLE_UA(41b, 16b) - _ASM_EXTABLE_UA(10b, 16b) - _ASM_EXTABLE_UA(51b, 16b) - _ASM_EXTABLE_UA(11b, 16b) - _ASM_EXTABLE_UA(61b, 16b) - _ASM_EXTABLE_UA(12b, 16b) - _ASM_EXTABLE_UA(71b, 16b) - _ASM_EXTABLE_UA(13b, 16b) - _ASM_EXTABLE_UA(81b, 16b) - _ASM_EXTABLE_UA(14b, 16b) - _ASM_EXTABLE_UA(91b, 16b) - _ASM_EXTABLE_UA(6b, 9b) - _ASM_EXTABLE_UA(7b, 16b) + _ASM_EXTABLE_UA(0b, 8b) + _ASM_EXTABLE_UA(1b, 8b) + _ASM_EXTABLE_UA(2b, 8b) + _ASM_EXTABLE_UA(21b, 8b) + _ASM_EXTABLE_UA(3b, 8b) + _ASM_EXTABLE_UA(31b, 8b) + _ASM_EXTABLE_UA(4b, 8b) + _ASM_EXTABLE_UA(41b, 8b) + _ASM_EXTABLE_UA(10b, 8b) + _ASM_EXTABLE_UA(51b, 8b) + _ASM_EXTABLE_UA(11b, 8b) + _ASM_EXTABLE_UA(61b, 8b) + _ASM_EXTABLE_UA(12b, 8b) + _ASM_EXTABLE_UA(71b, 8b) + _ASM_EXTABLE_UA(13b, 8b) + _ASM_EXTABLE_UA(81b, 8b) + _ASM_EXTABLE_UA(14b, 8b) + _ASM_EXTABLE_UA(91b, 8b) + _ASM_EXTABLE_TYPE_REG(6b, 8b, EX_TYPE_UCOPY_LEN4, %%eax) + _ASM_EXTABLE_UA(7b, 8b) : "=&c"(size), "=&D" (d0), "=&S" (d1) : "1"(to), "2"(from), "0"(size) : "eax", "edx", "memory"); @@ -315,14 +302,8 @@ do { \ " movl %3,%0\n" \ "1: rep; movsb\n" \ "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "5: addl %3,%0\n" \ - " jmp 2b\n" \ - "3: lea 0(%3,%0,4),%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE_UA(4b, 5b) \ - _ASM_EXTABLE_UA(0b, 3b) \ + _ASM_EXTABLE_TYPE_REG(4b, 2b, EX_TYPE_UCOPY_LEN1, %3) \ + _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %3) \ _ASM_EXTABLE_UA(1b, 2b) \ : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ : "3"(size), "0"(size), "1"(to), "2"(from) \ diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 508c81e97a..0402a749f3 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -35,12 +35,10 @@ unsigned long __clear_user(void __user *addr, unsigned long size) " incq %[dst]\n" " decl %%ecx ; jnz 1b\n" "2:\n" - ".section .fixup,\"ax\"\n" - "3: lea 0(%[size1],%[size8],8),%[size8]\n" - " jmp 2b\n" - ".previous\n" - _ASM_EXTABLE_UA(0b, 3b) + + _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN8, %[size1]) _ASM_EXTABLE_UA(1b, 2b) + : [size8] "=&c"(size), [dst] "=&D" (__d0) : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr)); clac(); diff --git a/arch/x86/math-emu/fpu_aux.c b/arch/x86/math-emu/fpu_aux.c index 0347484594..d62662bdd4 100644 --- a/arch/x86/math-emu/fpu_aux.c +++ b/arch/x86/math-emu/fpu_aux.c @@ -53,7 +53,7 @@ void fpstate_init_soft(struct swregs_state *soft) void finit(void) { - fpstate_init_soft(¤t->thread.fpu.state.soft); + fpstate_init_soft(¤t->thread.fpu.fpstate->regs.soft); } /* diff --git a/arch/x86/math-emu/fpu_entry.c b/arch/x86/math-emu/fpu_entry.c index 8679a9d6c4..7fe56c594a 100644 --- a/arch/x86/math-emu/fpu_entry.c +++ b/arch/x86/math-emu/fpu_entry.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include "fpu_system.h" #include "fpu_emu.h" @@ -640,7 +640,7 @@ int fpregs_soft_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct swregs_state *s387 = &target->thread.fpu.state.soft; + struct swregs_state *s387 = &target->thread.fpu.fpstate->regs.soft; void *space = s387->st_space; int ret; int offset, other, i, tags, regnr, tag, newtop; @@ -691,7 +691,7 @@ int fpregs_soft_get(struct task_struct *target, const struct user_regset *regset, struct membuf to) { - struct swregs_state *s387 = &target->thread.fpu.state.soft; + struct swregs_state *s387 = &target->thread.fpu.fpstate->regs.soft; const void *space = s387->st_space; int offset = (S387->ftop & 7) * 10, other = 80 - offset; diff --git a/arch/x86/math-emu/fpu_system.h b/arch/x86/math-emu/fpu_system.h index 9b41391867..eec3e4805c 100644 --- a/arch/x86/math-emu/fpu_system.h +++ b/arch/x86/math-emu/fpu_system.h @@ -73,7 +73,7 @@ static inline bool seg_writable(struct desc_struct *d) return (d->type & SEG_TYPE_EXECUTE_MASK) == SEG_TYPE_WRITABLE; } -#define I387 (¤t->thread.fpu.state) +#define I387 (¤t->thread.fpu.fpstate->regs) #define FPU_info (I387->soft.info) #define FPU_CS (*(unsigned short *) &(FPU_info->regs->cs)) diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 5864219221..fe3d3061fc 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -2,9 +2,11 @@ # Kernel does not boot with instrumentation of tlb.c and mem_encrypt*.c KCOV_INSTRUMENT_tlb.o := n KCOV_INSTRUMENT_mem_encrypt.o := n +KCOV_INSTRUMENT_mem_encrypt_amd.o := n KCOV_INSTRUMENT_mem_encrypt_identity.o := n KASAN_SANITIZE_mem_encrypt.o := n +KASAN_SANITIZE_mem_encrypt_amd.o := n KASAN_SANITIZE_mem_encrypt_identity.o := n # Disable KCSAN entirely, because otherwise we get warnings that some functions @@ -13,6 +15,7 @@ KCSAN_SANITIZE := n ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_mem_encrypt.o = -pg +CFLAGS_REMOVE_mem_encrypt_amd.o = -pg CFLAGS_REMOVE_mem_encrypt_identity.o = -pg endif @@ -52,6 +55,8 @@ obj-$(CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS) += pkeys.o obj-$(CONFIG_RANDOMIZE_MEMORY) += kaslr.o obj-$(CONFIG_PAGE_TABLE_ISOLATION) += pti.o -obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt.o +obj-$(CONFIG_X86_MEM_ENCRYPT) += mem_encrypt.o +obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_amd.o + obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_identity.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += mem_encrypt_boot.o diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index e1664e9f96..dba2197c05 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -2,48 +2,58 @@ #include #include #include +#include #include -#include +#include #include #include #include +#include +#include -typedef bool (*ex_handler_t)(const struct exception_table_entry *, - struct pt_regs *, int, unsigned long, - unsigned long); +static inline unsigned long *pt_regs_nr(struct pt_regs *regs, int nr) +{ + int reg_offset = pt_regs_offset(regs, nr); + static unsigned long __dummy; + + if (WARN_ON_ONCE(reg_offset < 0)) + return &__dummy; + + return (unsigned long *)((unsigned long)regs + reg_offset); +} static inline unsigned long ex_fixup_addr(const struct exception_table_entry *x) { return (unsigned long)&x->fixup + x->fixup; } -static inline ex_handler_t -ex_fixup_handler(const struct exception_table_entry *x) -{ - return (ex_handler_t)((unsigned long)&x->handler + x->handler); -} -__visible bool ex_handler_default(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) +static bool ex_handler_default(const struct exception_table_entry *e, + struct pt_regs *regs) { - regs->ip = ex_fixup_addr(fixup); + if (e->data & EX_FLAG_CLEAR_AX) + regs->ax = 0; + if (e->data & EX_FLAG_CLEAR_DX) + regs->dx = 0; + + regs->ip = ex_fixup_addr(e); return true; } -EXPORT_SYMBOL(ex_handler_default); -__visible bool ex_handler_fault(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) +static bool ex_handler_fault(const struct exception_table_entry *fixup, + struct pt_regs *regs, int trapnr) { - regs->ip = ex_fixup_addr(fixup); regs->ax = trapnr; - return true; + return ex_handler_default(fixup, regs); +} + +static bool ex_handler_sgx(const struct exception_table_entry *fixup, + struct pt_regs *regs, int trapnr) +{ + regs->ax = trapnr | SGX_ENCLS_FAULT_FLAG; + return ex_handler_default(fixup, regs); } -EXPORT_SYMBOL_GPL(ex_handler_fault); /* * Handler for when we fail to restore a task's FPU state. We should never get @@ -55,111 +65,93 @@ EXPORT_SYMBOL_GPL(ex_handler_fault); * of vulnerability by restoring from the initial state (essentially, zeroing * out all the FPU registers) if we can't restore from the task's FPU state. */ -__visible bool ex_handler_fprestore(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) +static bool ex_handler_fprestore(const struct exception_table_entry *fixup, + struct pt_regs *regs) { regs->ip = ex_fixup_addr(fixup); WARN_ONCE(1, "Bad FPU state detected at %pB, reinitializing FPU registers.", (void *)instruction_pointer(regs)); - __restore_fpregs_from_fpstate(&init_fpstate, xfeatures_mask_fpstate()); + fpu_reset_from_exception_fixup(); return true; } -EXPORT_SYMBOL_GPL(ex_handler_fprestore); -__visible bool ex_handler_uaccess(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) +static bool ex_handler_uaccess(const struct exception_table_entry *fixup, + struct pt_regs *regs, int trapnr) { WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?"); - regs->ip = ex_fixup_addr(fixup); - return true; + return ex_handler_default(fixup, regs); } -EXPORT_SYMBOL(ex_handler_uaccess); -__visible bool ex_handler_copy(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) +static bool ex_handler_copy(const struct exception_table_entry *fixup, + struct pt_regs *regs, int trapnr) { WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?"); - regs->ip = ex_fixup_addr(fixup); - regs->ax = trapnr; - return true; + return ex_handler_fault(fixup, regs, trapnr); } -EXPORT_SYMBOL(ex_handler_copy); -__visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) +static bool ex_handler_msr(const struct exception_table_entry *fixup, + struct pt_regs *regs, bool wrmsr, bool safe, int reg) { - if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", - (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) - show_stack_regs(regs); - - /* Pretend that the read succeeded and returned 0. */ - regs->ip = ex_fixup_addr(fixup); - regs->ax = 0; - regs->dx = 0; - return true; -} -EXPORT_SYMBOL(ex_handler_rdmsr_unsafe); - -__visible bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) -{ - if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", + if (!safe && wrmsr && + pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax, regs->ip, (void *)regs->ip)) show_stack_regs(regs); - /* Pretend that the write succeeded. */ - regs->ip = ex_fixup_addr(fixup); - return true; -} -EXPORT_SYMBOL(ex_handler_wrmsr_unsafe); + if (!safe && !wrmsr && + pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", + (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) + show_stack_regs(regs); -__visible bool ex_handler_clear_fs(const struct exception_table_entry *fixup, - struct pt_regs *regs, int trapnr, - unsigned long error_code, - unsigned long fault_addr) + if (!wrmsr) { + /* Pretend that the read succeeded and returned 0. */ + regs->ax = 0; + regs->dx = 0; + } + + if (safe) + *pt_regs_nr(regs, reg) = -EIO; + + return ex_handler_default(fixup, regs); +} + +static bool ex_handler_clear_fs(const struct exception_table_entry *fixup, + struct pt_regs *regs) { if (static_cpu_has(X86_BUG_NULL_SEG)) asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS)); asm volatile ("mov %0, %%fs" : : "rm" (0)); - return ex_handler_default(fixup, regs, trapnr, error_code, fault_addr); + return ex_handler_default(fixup, regs); } -EXPORT_SYMBOL(ex_handler_clear_fs); -enum handler_type ex_get_fault_handler_type(unsigned long ip) +static bool ex_handler_imm_reg(const struct exception_table_entry *fixup, + struct pt_regs *regs, int reg, int imm) { - const struct exception_table_entry *e; - ex_handler_t handler; + *pt_regs_nr(regs, reg) = (long)imm; + return ex_handler_default(fixup, regs); +} - e = search_exception_tables(ip); - if (!e) - return EX_HANDLER_NONE; - handler = ex_fixup_handler(e); - if (handler == ex_handler_fault) - return EX_HANDLER_FAULT; - else if (handler == ex_handler_uaccess || handler == ex_handler_copy) - return EX_HANDLER_UACCESS; - else - return EX_HANDLER_OTHER; +static bool ex_handler_ucopy_len(const struct exception_table_entry *fixup, + struct pt_regs *regs, int trapnr, int reg, int imm) +{ + regs->cx = imm * regs->cx + *pt_regs_nr(regs, reg); + return ex_handler_uaccess(fixup, regs, trapnr); +} + +int ex_get_fixup_type(unsigned long ip) +{ + const struct exception_table_entry *e = search_exception_tables(ip); + + return e ? FIELD_GET(EX_DATA_TYPE_MASK, e->data) : EX_TYPE_NONE; } int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, unsigned long fault_addr) { const struct exception_table_entry *e; - ex_handler_t handler; + int type, reg, imm; #ifdef CONFIG_PNPBIOS if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) { @@ -179,8 +171,52 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, if (!e) return 0; - handler = ex_fixup_handler(e); - return handler(e, regs, trapnr, error_code, fault_addr); + type = FIELD_GET(EX_DATA_TYPE_MASK, e->data); + reg = FIELD_GET(EX_DATA_REG_MASK, e->data); + imm = FIELD_GET(EX_DATA_IMM_MASK, e->data); + + switch (type) { + case EX_TYPE_DEFAULT: + case EX_TYPE_DEFAULT_MCE_SAFE: + return ex_handler_default(e, regs); + case EX_TYPE_FAULT: + case EX_TYPE_FAULT_MCE_SAFE: + return ex_handler_fault(e, regs, trapnr); + case EX_TYPE_UACCESS: + return ex_handler_uaccess(e, regs, trapnr); + case EX_TYPE_COPY: + return ex_handler_copy(e, regs, trapnr); + case EX_TYPE_CLEAR_FS: + return ex_handler_clear_fs(e, regs); + case EX_TYPE_FPU_RESTORE: + return ex_handler_fprestore(e, regs); + case EX_TYPE_BPF: + return ex_handler_bpf(e, regs); + case EX_TYPE_WRMSR: + return ex_handler_msr(e, regs, true, false, reg); + case EX_TYPE_RDMSR: + return ex_handler_msr(e, regs, false, false, reg); + case EX_TYPE_WRMSR_SAFE: + return ex_handler_msr(e, regs, true, true, reg); + case EX_TYPE_RDMSR_SAFE: + return ex_handler_msr(e, regs, false, true, reg); + case EX_TYPE_WRMSR_IN_MCE: + ex_handler_msr_mce(regs, true); + break; + case EX_TYPE_RDMSR_IN_MCE: + ex_handler_msr_mce(regs, false); + break; + case EX_TYPE_POP_REG: + regs->sp += sizeof(long); + fallthrough; + case EX_TYPE_IMM_REG: + return ex_handler_imm_reg(e, regs, reg, imm); + case EX_TYPE_FAULT_SGX: + return ex_handler_sgx(e, regs, trapnr); + case EX_TYPE_UCOPY_LEN: + return ex_handler_ucopy_len(e, regs, trapnr, reg, imm); + } + BUG(); } extern unsigned int early_recursion_flag; diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 4bfed53e21..d0074c6ed3 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1413,8 +1413,7 @@ void do_user_addr_fault(struct pt_regs *regs, * and if there is a fatal signal pending there is no guarantee * that we made any progress. Handle this case first. */ - if (unlikely((fault & VM_FAULT_RETRY) && - (flags & FAULT_FLAG_ALLOW_RETRY))) { + if (unlikely(fault & VM_FAULT_RETRY)) { flags |= FAULT_FLAG_TRIED; goto retry; } diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 23a14d82e7..4ba024d5b6 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -618,7 +618,7 @@ static void __init memory_map_top_down(unsigned long map_start, */ addr = memblock_phys_alloc_range(PMD_SIZE, PMD_SIZE, map_start, map_end); - memblock_free(addr, PMD_SIZE); + memblock_phys_free(addr, PMD_SIZE); real_end = addr + PMD_SIZE; /* step_size need to be small so pgt_buf from BRK could cover it */ @@ -714,6 +714,11 @@ static void __init memory_map_bottom_up(unsigned long map_start, static void __init init_trampoline(void) { #ifdef CONFIG_X86_64 + /* + * The code below will alias kernel page-tables in the user-range of the + * address space, including the Global bit. So global TLB entries will + * be created when using the trampoline page-table. + */ if (!kaslr_memory_enabled()) trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)]; else diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index bd90b8fe81..d4e2648a1d 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c @@ -238,11 +238,7 @@ page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base) } } -/* - * The already defines is_kernel_text, - * using '__' prefix not to get in conflict. - */ -static inline int __is_kernel_text(unsigned long addr) +static inline int is_x86_32_kernel_text(unsigned long addr) { if (addr >= (unsigned long)_text && addr <= (unsigned long)__init_end) return 1; @@ -333,8 +329,8 @@ kernel_physical_mapping_init(unsigned long start, addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE + PAGE_OFFSET + PAGE_SIZE-1; - if (__is_kernel_text(addr) || - __is_kernel_text(addr2)) + if (is_x86_32_kernel_text(addr) || + is_x86_32_kernel_text(addr2)) prot = PAGE_KERNEL_LARGE_EXEC; pages_2m++; @@ -359,7 +355,7 @@ kernel_physical_mapping_init(unsigned long start, */ pgprot_t init_prot = __pgprot(PTE_IDENT_ATTR); - if (__is_kernel_text(addr)) + if (is_x86_32_kernel_text(addr)) prot = PAGE_KERNEL_EXEC; pages_4k++; @@ -779,37 +775,6 @@ void __init mem_init(void) test_wp_bit(); } -#ifdef CONFIG_MEMORY_HOTPLUG -int arch_add_memory(int nid, u64 start, u64 size, - struct mhp_params *params) -{ - unsigned long start_pfn = start >> PAGE_SHIFT; - unsigned long nr_pages = size >> PAGE_SHIFT; - int ret; - - /* - * The page tables were already mapped at boot so if the caller - * requests a different mapping type then we must change all the - * pages with __set_memory_prot(). - */ - if (params->pgprot.pgprot != PAGE_KERNEL.pgprot) { - ret = __set_memory_prot(start, nr_pages, params->pgprot); - if (ret) - return ret; - } - - return __add_pages(nid, start_pfn, nr_pages, params); -} - -void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) -{ - unsigned long start_pfn = start >> PAGE_SHIFT; - unsigned long nr_pages = size >> PAGE_SHIFT; - - __remove_pages(start_pfn, nr_pages, altmap); -} -#endif - int kernel_set_to_readonly __read_mostly; static void mark_nxdata_nx(void) @@ -820,7 +785,7 @@ static void mark_nxdata_nx(void) */ unsigned long start = PFN_ALIGN(_etext); /* - * This comes from __is_kernel_text upper limit. Also HPAGE where used: + * This comes from is_x86_32_kernel_text upper limit. Also HPAGE where used: */ unsigned long size = (((unsigned long)__init_end + HPAGE_SIZE) & HPAGE_MASK) - start; diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 36098226a9..96d34ebb20 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -981,7 +981,7 @@ static void __meminit free_pagetable(struct page *page, int order) if (PageReserved(page)) { __ClearPageReserved(page); - magic = (unsigned long)page->freelist; + magic = page->index; if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) { while (nr_pages--) put_page_bootmem(page++); diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c index 7ce9b8dd87..17a492c273 100644 --- a/arch/x86/mm/ioremap.c +++ b/arch/x86/mm/ioremap.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include @@ -92,7 +92,7 @@ static unsigned int __ioremap_check_ram(struct resource *res) */ static unsigned int __ioremap_check_encrypted(struct resource *res) { - if (!sev_active()) + if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) return 0; switch (res->desc) { @@ -112,7 +112,7 @@ static unsigned int __ioremap_check_encrypted(struct resource *res) */ static void __ioremap_check_other(resource_size_t addr, struct ioremap_desc *desc) { - if (!sev_active()) + if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) return; if (!IS_ENABLED(CONFIG_EFI)) @@ -508,6 +508,7 @@ void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr) memunmap((void *)((unsigned long)addr & PAGE_MASK)); } +#ifdef CONFIG_AMD_MEM_ENCRYPT /* * Examine the physical address to determine if it is an area of memory * that should be mapped decrypted. If the memory is not part of the @@ -555,7 +556,7 @@ static bool memremap_should_map_decrypted(resource_size_t phys_addr, case E820_TYPE_NVS: case E820_TYPE_UNUSABLE: /* For SEV, these areas are encrypted */ - if (sev_active()) + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) break; fallthrough; @@ -738,7 +739,7 @@ static bool __init early_memremap_is_setup_data(resource_size_t phys_addr, bool arch_memremap_can_ram_remap(resource_size_t phys_addr, unsigned long size, unsigned long flags) { - if (!mem_encrypt_active()) + if (!cc_platform_has(CC_ATTR_MEM_ENCRYPT)) return true; if (flags & MEMREMAP_ENC) @@ -747,7 +748,7 @@ bool arch_memremap_can_ram_remap(resource_size_t phys_addr, unsigned long size, if (flags & MEMREMAP_DEC) return false; - if (sme_active()) { + if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) { if (memremap_is_setup_data(phys_addr, size) || memremap_is_efi_data(phys_addr, size)) return false; @@ -768,12 +769,12 @@ pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr, { bool encrypted_prot; - if (!mem_encrypt_active()) + if (!cc_platform_has(CC_ATTR_MEM_ENCRYPT)) return prot; encrypted_prot = true; - if (sme_active()) { + if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) { if (early_memremap_is_setup_data(phys_addr, size) || memremap_is_efi_data(phys_addr, size)) encrypted_prot = false; @@ -791,7 +792,6 @@ bool phys_mem_access_encrypted(unsigned long phys_addr, unsigned long size) return arch_memremap_can_ram_remap(phys_addr, size, 0); } -#ifdef CONFIG_AMD_MEM_ENCRYPT /* Remap memory with encryption */ void __init *early_memremap_encrypted(resource_size_t phys_addr, unsigned long size) diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c index ef88537071..e7b9b464a8 100644 --- a/arch/x86/mm/kasan_init_64.c +++ b/arch/x86/mm/kasan_init_64.c @@ -49,7 +49,7 @@ static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr, p = early_alloc(PMD_SIZE, nid, false); if (p && pmd_set_huge(pmd, __pa(p), PAGE_KERNEL)) return; - memblock_free_ptr(p, PMD_SIZE); + memblock_free(p, PMD_SIZE); } p = early_alloc(PAGE_SIZE, nid, true); @@ -85,7 +85,7 @@ static void __init kasan_populate_pud(pud_t *pud, unsigned long addr, p = early_alloc(PUD_SIZE, nid, false); if (p && pud_set_huge(pud, __pa(p), PAGE_KERNEL)) return; - memblock_free_ptr(p, PUD_SIZE); + memblock_free(p, PUD_SIZE); } p = early_alloc(PAGE_SIZE, nid, true); diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index e29b1418d0..50d209939c 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -1,394 +1,18 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * AMD Memory Encryption Support + * Memory Encryption Support Common Code * * Copyright (C) 2016 Advanced Micro Devices, Inc. * * Author: Tom Lendacky */ -#define DISABLE_BRANCH_PROFILING - -#include -#include -#include #include -#include -#include -#include -#include -#include #include -#include +#include #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "mm_internal.h" - -/* - * Since SME related variables are set early in the boot process they must - * reside in the .data section so as not to be zeroed out when the .bss - * section is later cleared. - */ -u64 sme_me_mask __section(".data") = 0; -u64 sev_status __section(".data") = 0; -u64 sev_check_data __section(".data") = 0; -EXPORT_SYMBOL(sme_me_mask); -DEFINE_STATIC_KEY_FALSE(sev_enable_key); -EXPORT_SYMBOL_GPL(sev_enable_key); - -/* Buffer used for early in-place encryption by BSP, no locking needed */ -static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE); - -/* - * This routine does not change the underlying encryption setting of the - * page(s) that map this memory. It assumes that eventually the memory is - * meant to be accessed as either encrypted or decrypted but the contents - * are currently not in the desired state. - * - * This routine follows the steps outlined in the AMD64 Architecture - * Programmer's Manual Volume 2, Section 7.10.8 Encrypt-in-Place. - */ -static void __init __sme_early_enc_dec(resource_size_t paddr, - unsigned long size, bool enc) -{ - void *src, *dst; - size_t len; - - if (!sme_me_mask) - return; - - wbinvd(); - - /* - * There are limited number of early mapping slots, so map (at most) - * one page at time. - */ - while (size) { - len = min_t(size_t, sizeof(sme_early_buffer), size); - - /* - * Create mappings for the current and desired format of - * the memory. Use a write-protected mapping for the source. - */ - src = enc ? early_memremap_decrypted_wp(paddr, len) : - early_memremap_encrypted_wp(paddr, len); - - dst = enc ? early_memremap_encrypted(paddr, len) : - early_memremap_decrypted(paddr, len); - - /* - * If a mapping can't be obtained to perform the operation, - * then eventual access of that area in the desired mode - * will cause a crash. - */ - BUG_ON(!src || !dst); - - /* - * Use a temporary buffer, of cache-line multiple size, to - * avoid data corruption as documented in the APM. - */ - memcpy(sme_early_buffer, src, len); - memcpy(dst, sme_early_buffer, len); - - early_memunmap(dst, len); - early_memunmap(src, len); - - paddr += len; - size -= len; - } -} - -void __init sme_early_encrypt(resource_size_t paddr, unsigned long size) -{ - __sme_early_enc_dec(paddr, size, true); -} - -void __init sme_early_decrypt(resource_size_t paddr, unsigned long size) -{ - __sme_early_enc_dec(paddr, size, false); -} - -static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size, - bool map) -{ - unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET; - pmdval_t pmd_flags, pmd; - - /* Use early_pmd_flags but remove the encryption mask */ - pmd_flags = __sme_clr(early_pmd_flags); - - do { - pmd = map ? (paddr & PMD_MASK) + pmd_flags : 0; - __early_make_pgtable((unsigned long)vaddr, pmd); - - vaddr += PMD_SIZE; - paddr += PMD_SIZE; - size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE; - } while (size); - - flush_tlb_local(); -} - -void __init sme_unmap_bootdata(char *real_mode_data) -{ - struct boot_params *boot_data; - unsigned long cmdline_paddr; - - if (!sme_active()) - return; - - /* Get the command line address before unmapping the real_mode_data */ - boot_data = (struct boot_params *)real_mode_data; - cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32); - - __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), false); - - if (!cmdline_paddr) - return; - - __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, false); -} - -void __init sme_map_bootdata(char *real_mode_data) -{ - struct boot_params *boot_data; - unsigned long cmdline_paddr; - - if (!sme_active()) - return; - - __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), true); - - /* Get the command line address after mapping the real_mode_data */ - boot_data = (struct boot_params *)real_mode_data; - cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32); - - if (!cmdline_paddr) - return; - - __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true); -} - -void __init sme_early_init(void) -{ - unsigned int i; - - if (!sme_me_mask) - return; - - early_pmd_flags = __sme_set(early_pmd_flags); - - __supported_pte_mask = __sme_set(__supported_pte_mask); - - /* Update the protection map with memory encryption mask */ - for (i = 0; i < ARRAY_SIZE(protection_map); i++) - protection_map[i] = pgprot_encrypted(protection_map[i]); - - if (sev_active()) - swiotlb_force = SWIOTLB_FORCE; -} - -void __init sev_setup_arch(void) -{ - phys_addr_t total_mem = memblock_phys_mem_size(); - unsigned long size; - - if (!sev_active()) - return; - - /* - * For SEV, all DMA has to occur via shared/unencrypted pages. - * SEV uses SWIOTLB to make this happen without changing device - * drivers. However, depending on the workload being run, the - * default 64MB of SWIOTLB may not be enough and SWIOTLB may - * run out of buffers for DMA, resulting in I/O errors and/or - * performance degradation especially with high I/O workloads. - * - * Adjust the default size of SWIOTLB for SEV guests using - * a percentage of guest memory for SWIOTLB buffers. - * Also, as the SWIOTLB bounce buffer memory is allocated - * from low memory, ensure that the adjusted size is within - * the limits of low available memory. - * - * The percentage of guest memory used here for SWIOTLB buffers - * is more of an approximation of the static adjustment which - * 64MB for <1G, and ~128M to 256M for 1G-to-4G, i.e., the 6% - */ - size = total_mem * 6 / 100; - size = clamp_val(size, IO_TLB_DEFAULT_SIZE, SZ_1G); - swiotlb_adjust_size(size); -} - -static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc) -{ - pgprot_t old_prot, new_prot; - unsigned long pfn, pa, size; - pte_t new_pte; - - switch (level) { - case PG_LEVEL_4K: - pfn = pte_pfn(*kpte); - old_prot = pte_pgprot(*kpte); - break; - case PG_LEVEL_2M: - pfn = pmd_pfn(*(pmd_t *)kpte); - old_prot = pmd_pgprot(*(pmd_t *)kpte); - break; - case PG_LEVEL_1G: - pfn = pud_pfn(*(pud_t *)kpte); - old_prot = pud_pgprot(*(pud_t *)kpte); - break; - default: - return; - } - - new_prot = old_prot; - if (enc) - pgprot_val(new_prot) |= _PAGE_ENC; - else - pgprot_val(new_prot) &= ~_PAGE_ENC; - - /* If prot is same then do nothing. */ - if (pgprot_val(old_prot) == pgprot_val(new_prot)) - return; - - pa = pfn << PAGE_SHIFT; - size = page_level_size(level); - - /* - * We are going to perform in-place en-/decryption and change the - * physical page attribute from C=1 to C=0 or vice versa. Flush the - * caches to ensure that data gets accessed with the correct C-bit. - */ - clflush_cache_range(__va(pa), size); - - /* Encrypt/decrypt the contents in-place */ - if (enc) - sme_early_encrypt(pa, size); - else - sme_early_decrypt(pa, size); - - /* Change the page encryption mask. */ - new_pte = pfn_pte(pfn, new_prot); - set_pte_atomic(kpte, new_pte); -} - -static int __init early_set_memory_enc_dec(unsigned long vaddr, - unsigned long size, bool enc) -{ - unsigned long vaddr_end, vaddr_next; - unsigned long psize, pmask; - int split_page_size_mask; - int level, ret; - pte_t *kpte; - - vaddr_next = vaddr; - vaddr_end = vaddr + size; - - for (; vaddr < vaddr_end; vaddr = vaddr_next) { - kpte = lookup_address(vaddr, &level); - if (!kpte || pte_none(*kpte)) { - ret = 1; - goto out; - } - - if (level == PG_LEVEL_4K) { - __set_clr_pte_enc(kpte, level, enc); - vaddr_next = (vaddr & PAGE_MASK) + PAGE_SIZE; - continue; - } - - psize = page_level_size(level); - pmask = page_level_mask(level); - - /* - * Check whether we can change the large page in one go. - * We request a split when the address is not aligned and - * the number of pages to set/clear encryption bit is smaller - * than the number of pages in the large page. - */ - if (vaddr == (vaddr & pmask) && - ((vaddr_end - vaddr) >= psize)) { - __set_clr_pte_enc(kpte, level, enc); - vaddr_next = (vaddr & pmask) + psize; - continue; - } - - /* - * The virtual address is part of a larger page, create the next - * level page table mapping (4K or 2M). If it is part of a 2M - * page then we request a split of the large page into 4K - * chunks. A 1GB large page is split into 2M pages, resp. - */ - if (level == PG_LEVEL_2M) - split_page_size_mask = 0; - else - split_page_size_mask = 1 << PG_LEVEL_2M; - - /* - * kernel_physical_mapping_change() does not flush the TLBs, so - * a TLB flush is required after we exit from the for loop. - */ - kernel_physical_mapping_change(__pa(vaddr & pmask), - __pa((vaddr_end & pmask) + psize), - split_page_size_mask); - } - - ret = 0; - -out: - __flush_tlb_all(); - return ret; -} - -int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size) -{ - return early_set_memory_enc_dec(vaddr, size, false); -} - -int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size) -{ - return early_set_memory_enc_dec(vaddr, size, true); -} - -/* - * SME and SEV are very similar but they are not the same, so there are - * times that the kernel will need to distinguish between SME and SEV. The - * sme_active() and sev_active() functions are used for this. When a - * distinction isn't needed, the mem_encrypt_active() function can be used. - * - * The trampoline code is a good example for this requirement. Before - * paging is activated, SME will access all memory as decrypted, but SEV - * will access all memory as encrypted. So, when APs are being brought - * up under SME the trampoline area cannot be encrypted, whereas under SEV - * the trampoline area must be encrypted. - */ -bool sev_active(void) -{ - return sev_status & MSR_AMD64_SEV_ENABLED; -} - -bool sme_active(void) -{ - return sme_me_mask && !sev_active(); -} -EXPORT_SYMBOL_GPL(sev_active); - -/* Needs to be called from non-instrumentable code */ -bool noinstr sev_es_active(void) -{ - return sev_status & MSR_AMD64_SEV_ES_ENABLED; -} +#include +#include /* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */ bool force_dma_unencrypted(struct device *dev) @@ -396,7 +20,7 @@ bool force_dma_unencrypted(struct device *dev) /* * For SEV, all DMA must be to unencrypted addresses. */ - if (sev_active()) + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) return true; /* @@ -404,7 +28,7 @@ bool force_dma_unencrypted(struct device *dev) * device does not support DMA to addresses that include the * encryption mask. */ - if (sme_active()) { + if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) { u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask)); u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); @@ -416,36 +40,12 @@ bool force_dma_unencrypted(struct device *dev) return false; } -void __init mem_encrypt_free_decrypted_mem(void) -{ - unsigned long vaddr, vaddr_end, npages; - int r; - - vaddr = (unsigned long)__start_bss_decrypted_unused; - vaddr_end = (unsigned long)__end_bss_decrypted; - npages = (vaddr_end - vaddr) >> PAGE_SHIFT; - - /* - * The unused memory range was mapped decrypted, change the encryption - * attribute from decrypted to encrypted before freeing it. - */ - if (mem_encrypt_active()) { - r = set_memory_encrypted(vaddr, npages); - if (r) { - pr_warn("failed to free unused decrypted pages\n"); - return; - } - } - - free_init_pages("unused decrypted", vaddr, vaddr_end); -} - static void print_mem_encrypt_feature_info(void) { pr_info("AMD Memory Encryption Features active:"); /* Secure Memory Encryption */ - if (sme_active()) { + if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) { /* * SME is mutually exclusive with any of the SEV * features below. @@ -455,11 +55,11 @@ static void print_mem_encrypt_feature_info(void) } /* Secure Encrypted Virtualization */ - if (sev_active()) + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) pr_cont(" SEV"); /* Encrypted Register State */ - if (sev_es_active()) + if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) pr_cont(" SEV-ES"); pr_cont("\n"); @@ -468,24 +68,17 @@ static void print_mem_encrypt_feature_info(void) /* Architecture __weak replacement functions */ void __init mem_encrypt_init(void) { - if (!sme_me_mask) + if (!cc_platform_has(CC_ATTR_MEM_ENCRYPT)) return; /* Call into SWIOTLB to update the SWIOTLB DMA buffers */ swiotlb_update_mem_attributes(); - /* - * With SEV, we need to unroll the rep string I/O instructions, - * but SEV-ES supports them through the #VC handler. - */ - if (sev_active() && !sev_es_active()) - static_branch_enable(&sev_enable_key); - print_mem_encrypt_feature_info(); } int arch_has_restricted_virtio_memory_access(void) { - return sev_active(); + return cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT); } EXPORT_SYMBOL_GPL(arch_has_restricted_virtio_memory_access); diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c index 700ce8fdea..3f0abb4033 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -296,7 +297,13 @@ void __init sme_encrypt_kernel(struct boot_params *bp) unsigned long pgtable_area_len; unsigned long decrypted_base; - if (!sme_active()) + /* + * This is early code, use an open coded check for SME instead of + * using cc_platform_has(). This eliminates worries about removing + * instrumentation or checking boot_cpu_data in the cc_platform_has() + * function. + */ + if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED) return; /* diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 1e9b93b088..c6b1213086 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -355,7 +355,7 @@ void __init numa_reset_distance(void) /* numa_distance could be 1LU marking allocation failure, test cnt */ if (numa_distance_cnt) - memblock_free_ptr(numa_distance, size); + memblock_free(numa_distance, size); numa_distance_cnt = 0; numa_distance = NULL; /* enable table creation */ } diff --git a/arch/x86/mm/numa_emulation.c b/arch/x86/mm/numa_emulation.c index e801e30089..1a02b791d2 100644 --- a/arch/x86/mm/numa_emulation.c +++ b/arch/x86/mm/numa_emulation.c @@ -517,7 +517,7 @@ void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt) } /* free the copied physical distance table */ - memblock_free_ptr(phys_dist, phys_size); + memblock_free(phys_dist, phys_size); return; no_emu: diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index ad8a5c586a..b4072115c8 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,8 @@ #include #include #include +#include +#include #include "../mm_internal.h" @@ -1980,15 +1983,15 @@ int set_memory_global(unsigned long addr, int numpages) __pgprot(_PAGE_GLOBAL), 0); } -static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc) +/* + * __set_memory_enc_pgtable() is used for the hypervisors that get + * informed about "encryption" status via page tables. + */ +static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc) { struct cpa_data cpa; int ret; - /* Nothing to do if memory encryption is not active */ - if (!mem_encrypt_active()) - return 0; - /* Should not be working on unaligned addresses */ if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr)) addr &= PAGE_MASK; @@ -2020,9 +2023,26 @@ static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc) */ cpa_flush(&cpa, 0); + /* + * Notify hypervisor that a given memory range is mapped encrypted + * or decrypted. + */ + notify_range_enc_status_changed(addr, numpages, enc); + return ret; } +static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc) +{ + if (hv_is_isolation_supported()) + return hv_set_mem_host_visibility(addr, numpages, !enc); + + if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) + return __set_memory_enc_pgtable(addr, numpages, enc); + + return 0; +} + int set_memory_encrypted(unsigned long addr, int numpages) { return __set_memory_enc_dec(addr, numpages, true); diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 59ba2968af..a6cf56a149 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -361,7 +361,7 @@ static void l1d_flush_evaluate(unsigned long prev_mm, unsigned long next_mm, static unsigned long mm_mangle_tif_spec_bits(struct task_struct *next) { - unsigned long next_tif = task_thread_info(next)->flags; + unsigned long next_tif = read_task_thread_flags(next); unsigned long spec_bits = (next_tif >> TIF_SPEC_IB) & LAST_USER_MM_SPEC_MASK; /* @@ -1148,7 +1148,7 @@ void flush_tlb_one_user(unsigned long addr) */ STATIC_NOPV void native_flush_tlb_global(void) { - unsigned long cr4, flags; + unsigned long flags; if (static_cpu_has(X86_FEATURE_INVPCID)) { /* @@ -1168,11 +1168,7 @@ STATIC_NOPV void native_flush_tlb_global(void) */ raw_local_irq_save(flags); - cr4 = this_cpu_read(cpu_tlbstate.cr4); - /* toggle PGE */ - native_write_cr4(cr4 ^ X86_CR4_PGE); - /* write old PGE again and flush TLBs */ - native_write_cr4(cr4); + __native_tlb_flush_global(this_cpu_read(cpu_tlbstate.cr4)); raw_local_irq_restore(flags); } diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d99434dc21..0ecb140864 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * bpf_jit_comp.c: BPF JIT compiler + * BPF JIT compiler * * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com) - * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com + * Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com */ #include #include @@ -15,7 +15,6 @@ #include #include #include -#include static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) { @@ -225,6 +224,14 @@ static void jit_fill_hole(void *area, unsigned int size) struct jit_context { int cleanup_addr; /* Epilogue code offset */ + + /* + * Program specific offsets of labels in the code; these rely on the + * JIT doing at least 2 passes, recording the position on the first + * pass, only to generate the correct offset on the second pass. + */ + int tail_call_direct_label; + int tail_call_indirect_label; }; /* Maximum number of bytes emitted while JITing one eBPF insn */ @@ -380,20 +387,23 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, return __bpf_arch_text_poke(ip, t, old_addr, new_addr, true); } -static int get_pop_bytes(bool *callee_regs_used) +#define EMIT_LFENCE() EMIT3(0x0F, 0xAE, 0xE8) + +static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) { - int bytes = 0; + u8 *prog = *pprog; - if (callee_regs_used[3]) - bytes += 2; - if (callee_regs_used[2]) - bytes += 2; - if (callee_regs_used[1]) - bytes += 2; - if (callee_regs_used[0]) - bytes += 1; +#ifdef CONFIG_RETPOLINE + if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { + EMIT_LFENCE(); + EMIT2(0xFF, 0xE0 + reg); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) { + emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); + } else +#endif + EMIT2(0xFF, 0xE0 + reg); - return bytes; + *pprog = prog; } /* @@ -402,7 +412,7 @@ static int get_pop_bytes(bool *callee_regs_used) * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ... * if (index >= array->map.max_entries) * goto out; - * if (++tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; * prog = array->ptrs[index]; * if (prog == NULL) @@ -411,29 +421,12 @@ static int get_pop_bytes(bool *callee_regs_used) * out: */ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, - u32 stack_depth) + u32 stack_depth, u8 *ip, + struct jit_context *ctx) { int tcc_off = -4 - round_up(stack_depth, 8); - u8 *prog = *pprog; - int pop_bytes = 0; - int off1 = 42; - int off2 = 31; - int off3 = 9; - - /* count the additional bytes used for popping callee regs from stack - * that need to be taken into account for each of the offsets that - * are used for bailing out of the tail call - */ - pop_bytes = get_pop_bytes(callee_regs_used); - off1 += pop_bytes; - off2 += pop_bytes; - off3 += pop_bytes; - - if (stack_depth) { - off1 += 7; - off2 += 7; - off3 += 7; - } + u8 *prog = *pprog, *start = *pprog; + int offset; /* * rdi - pointer to ctx @@ -448,17 +441,19 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, EMIT2(0x89, 0xD2); /* mov edx, edx */ EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ offsetof(struct bpf_array, map.max_entries)); -#define OFFSET1 (off1 + RETPOLINE_RCX_BPF_JIT_SIZE) /* Number of bytes to jump */ - EMIT2(X86_JBE, OFFSET1); /* jbe out */ + + offset = ctx->tail_call_indirect_label - (prog + 2 - start); + EMIT2(X86_JBE, offset); /* jbe out */ /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; */ EMIT2_off32(0x8B, 0x85, tcc_off); /* mov eax, dword ptr [rbp - tcc_off] */ EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ -#define OFFSET2 (off2 + RETPOLINE_RCX_BPF_JIT_SIZE) - EMIT2(X86_JA, OFFSET2); /* ja out */ + + offset = ctx->tail_call_indirect_label - (prog + 2 - start); + EMIT2(X86_JAE, offset); /* jae out */ EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ EMIT2_off32(0x89, 0x85, tcc_off); /* mov dword ptr [rbp - tcc_off], eax */ @@ -471,12 +466,11 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, * goto out; */ EMIT3(0x48, 0x85, 0xC9); /* test rcx,rcx */ -#define OFFSET3 (off3 + RETPOLINE_RCX_BPF_JIT_SIZE) - EMIT2(X86_JE, OFFSET3); /* je out */ - *pprog = prog; - pop_callee_regs(pprog, callee_regs_used); - prog = *pprog; + offset = ctx->tail_call_indirect_label - (prog + 2 - start); + EMIT2(X86_JE, offset); /* je out */ + + pop_callee_regs(&prog, callee_regs_used); EMIT1(0x58); /* pop rax */ if (stack_depth) @@ -493,70 +487,52 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, * rdi == ctx (1st arg) * rcx == prog->bpf_func + X86_TAIL_CALL_OFFSET */ - RETPOLINE_RCX_BPF_JIT(); + emit_indirect_jump(&prog, 1 /* rcx */, ip + (prog - start)); /* out: */ + ctx->tail_call_indirect_label = prog - start; *pprog = prog; } static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, - u8 **pprog, int addr, u8 *image, - bool *callee_regs_used, u32 stack_depth) + u8 **pprog, u8 *ip, + bool *callee_regs_used, u32 stack_depth, + struct jit_context *ctx) { int tcc_off = -4 - round_up(stack_depth, 8); - u8 *prog = *pprog; - int pop_bytes = 0; - int off1 = 20; - int poke_off; - - /* count the additional bytes used for popping callee regs to stack - * that need to be taken into account for jump offset that is used for - * bailing out from of the tail call when limit is reached - */ - pop_bytes = get_pop_bytes(callee_regs_used); - off1 += pop_bytes; + u8 *prog = *pprog, *start = *pprog; + int offset; /* - * total bytes for: - * - nop5/ jmpq $off - * - pop callee regs - * - sub rsp, $val if depth > 0 - * - pop rax - */ - poke_off = X86_PATCH_SIZE + pop_bytes + 1; - if (stack_depth) { - poke_off += 7; - off1 += 7; - } - - /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; */ EMIT2_off32(0x8B, 0x85, tcc_off); /* mov eax, dword ptr [rbp - tcc_off] */ EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ - EMIT2(X86_JA, off1); /* ja out */ + + offset = ctx->tail_call_direct_label - (prog + 2 - start); + EMIT2(X86_JAE, offset); /* jae out */ EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ EMIT2_off32(0x89, 0x85, tcc_off); /* mov dword ptr [rbp - tcc_off], eax */ - poke->tailcall_bypass = image + (addr - poke_off - X86_PATCH_SIZE); + poke->tailcall_bypass = ip + (prog - start); poke->adj_off = X86_TAIL_CALL_OFFSET; - poke->tailcall_target = image + (addr - X86_PATCH_SIZE); + poke->tailcall_target = ip + ctx->tail_call_direct_label - X86_PATCH_SIZE; poke->bypass_addr = (u8 *)poke->tailcall_target + X86_PATCH_SIZE; emit_jump(&prog, (u8 *)poke->tailcall_target + X86_PATCH_SIZE, poke->tailcall_bypass); - *pprog = prog; - pop_callee_regs(pprog, callee_regs_used); - prog = *pprog; + pop_callee_regs(&prog, callee_regs_used); EMIT1(0x58); /* pop rax */ if (stack_depth) EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog += X86_PATCH_SIZE; + /* out: */ + ctx->tail_call_direct_label = prog - start; *pprog = prog; } @@ -841,9 +817,7 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, return 0; } -static bool ex_handler_bpf(const struct exception_table_entry *x, - struct pt_regs *regs, int trapnr, - unsigned long error_code, unsigned long fault_addr) +bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) { u32 reg = x->fixup >> 8; @@ -1038,52 +1012,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, case BPF_ALU64 | BPF_MOD | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_K: - case BPF_ALU64 | BPF_DIV | BPF_K: - EMIT1(0x50); /* push rax */ - EMIT1(0x52); /* push rdx */ - - if (BPF_SRC(insn->code) == BPF_X) - /* mov r11, src_reg */ - EMIT_mov(AUX_REG, src_reg); - else - /* mov r11, imm32 */ - EMIT3_off32(0x49, 0xC7, 0xC3, imm32); - - /* mov rax, dst_reg */ - EMIT_mov(BPF_REG_0, dst_reg); - - /* - * xor edx, edx - * equivalent to 'xor rdx, rdx', but one byte less - */ - EMIT2(0x31, 0xd2); - - if (BPF_CLASS(insn->code) == BPF_ALU64) - /* div r11 */ - EMIT3(0x49, 0xF7, 0xF3); - else - /* div r11d */ - EMIT3(0x41, 0xF7, 0xF3); - - if (BPF_OP(insn->code) == BPF_MOD) - /* mov r11, rdx */ - EMIT3(0x49, 0x89, 0xD3); - else - /* mov r11, rax */ - EMIT3(0x49, 0x89, 0xC3); - - EMIT1(0x5A); /* pop rdx */ - EMIT1(0x58); /* pop rax */ - - /* mov dst_reg, r11 */ - EMIT_mov(dst_reg, AUX_REG); - break; - - case BPF_ALU | BPF_MUL | BPF_K: - case BPF_ALU | BPF_MUL | BPF_X: - case BPF_ALU64 | BPF_MUL | BPF_K: - case BPF_ALU64 | BPF_MUL | BPF_X: - { + case BPF_ALU64 | BPF_DIV | BPF_K: { bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; if (dst_reg != BPF_REG_0) @@ -1091,30 +1020,74 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, if (dst_reg != BPF_REG_3) EMIT1(0x52); /* push rdx */ - /* mov r11, dst_reg */ - EMIT_mov(AUX_REG, dst_reg); + if (BPF_SRC(insn->code) == BPF_X) { + if (src_reg == BPF_REG_0 || + src_reg == BPF_REG_3) { + /* mov r11, src_reg */ + EMIT_mov(AUX_REG, src_reg); + src_reg = AUX_REG; + } + } else { + /* mov r11, imm32 */ + EMIT3_off32(0x49, 0xC7, 0xC3, imm32); + src_reg = AUX_REG; + } - if (BPF_SRC(insn->code) == BPF_X) - emit_mov_reg(&prog, is64, BPF_REG_0, src_reg); - else - emit_mov_imm32(&prog, is64, BPF_REG_0, imm32); + if (dst_reg != BPF_REG_0) + /* mov rax, dst_reg */ + emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg); - if (is64) - EMIT1(add_1mod(0x48, AUX_REG)); - else if (is_ereg(AUX_REG)) - EMIT1(add_1mod(0x40, AUX_REG)); - /* mul(q) r11 */ - EMIT2(0xF7, add_1reg(0xE0, AUX_REG)); + /* + * xor edx, edx + * equivalent to 'xor rdx, rdx', but one byte less + */ + EMIT2(0x31, 0xd2); + + /* div src_reg */ + maybe_emit_1mod(&prog, src_reg, is64); + EMIT2(0xF7, add_1reg(0xF0, src_reg)); + + if (BPF_OP(insn->code) == BPF_MOD && + dst_reg != BPF_REG_3) + /* mov dst_reg, rdx */ + emit_mov_reg(&prog, is64, dst_reg, BPF_REG_3); + else if (BPF_OP(insn->code) == BPF_DIV && + dst_reg != BPF_REG_0) + /* mov dst_reg, rax */ + emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0); if (dst_reg != BPF_REG_3) EMIT1(0x5A); /* pop rdx */ - if (dst_reg != BPF_REG_0) { - /* mov dst_reg, rax */ - EMIT_mov(dst_reg, BPF_REG_0); + if (dst_reg != BPF_REG_0) EMIT1(0x58); /* pop rax */ - } break; } + + case BPF_ALU | BPF_MUL | BPF_K: + case BPF_ALU64 | BPF_MUL | BPF_K: + maybe_emit_mod(&prog, dst_reg, dst_reg, + BPF_CLASS(insn->code) == BPF_ALU64); + + if (is_imm8(imm32)) + /* imul dst_reg, dst_reg, imm8 */ + EMIT3(0x6B, add_2reg(0xC0, dst_reg, dst_reg), + imm32); + else + /* imul dst_reg, dst_reg, imm32 */ + EMIT2_off32(0x69, + add_2reg(0xC0, dst_reg, dst_reg), + imm32); + break; + + case BPF_ALU | BPF_MUL | BPF_X: + case BPF_ALU64 | BPF_MUL | BPF_X: + maybe_emit_mod(&prog, src_reg, dst_reg, + BPF_CLASS(insn->code) == BPF_ALU64); + + /* imul dst_reg, src_reg */ + EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg)); + break; + /* Shifts */ case BPF_ALU | BPF_LSH | BPF_K: case BPF_ALU | BPF_RSH | BPF_K: @@ -1228,8 +1201,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, /* speculation barrier */ case BPF_ST | BPF_NOSPEC: if (boot_cpu_has(X86_FEATURE_XMM2)) - /* Emit 'lfence' */ - EMIT3(0x0F, 0xAE, 0xE8); + EMIT_LFENCE(); break; /* ST: *(u8*)(dst_reg + off) = imm */ @@ -1354,12 +1326,7 @@ st: if (is_imm8(insn->off)) } ex->insn = delta; - delta = (u8 *)ex_handler_bpf - (u8 *)&ex->handler; - if (!is_simm32(delta)) { - pr_err("extable->handler doesn't fit into 32-bit\n"); - return -EFAULT; - } - ex->handler = delta; + ex->data = EX_TYPE_BPF; if (dst_reg > BPF_REG_9) { pr_err("verifier error\n"); @@ -1453,13 +1420,16 @@ st: if (is_imm8(insn->off)) case BPF_JMP | BPF_TAIL_CALL: if (imm32) emit_bpf_tail_call_direct(&bpf_prog->aux->poke_tab[imm32 - 1], - &prog, addrs[i], image, + &prog, image + addrs[i - 1], callee_regs_used, - bpf_prog->aux->stack_depth); + bpf_prog->aux->stack_depth, + ctx); else emit_bpf_tail_call_indirect(&prog, callee_regs_used, - bpf_prog->aux->stack_depth); + bpf_prog->aux->stack_depth, + image + addrs[i - 1], + ctx); break; /* cond jump */ @@ -2006,7 +1976,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i void *orig_call) { int ret, i, nr_args = m->nr_args; - int stack_size = nr_args * 8; + int regs_off, ip_off, args_off, stack_size = nr_args * 8; struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY]; struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT]; struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN]; @@ -2021,14 +1991,39 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (!is_valid_bpf_tramp_flags(flags)) return -EINVAL; + /* Generated trampoline stack layout: + * + * RBP + 8 [ return address ] + * RBP + 0 [ RBP ] + * + * RBP - 8 [ return value ] BPF_TRAMP_F_CALL_ORIG or + * BPF_TRAMP_F_RET_FENTRY_RET flags + * + * [ reg_argN ] always + * [ ... ] + * RBP - regs_off [ reg_arg1 ] program's ctx pointer + * + * RBP - args_off [ args count ] always + * + * RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag + */ + /* room for return value of orig_call or fentry prog */ save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); if (save_ret) stack_size += 8; + regs_off = stack_size; + + /* args count */ + stack_size += 8; + args_off = stack_size; + if (flags & BPF_TRAMP_F_IP_ARG) stack_size += 8; /* room for IP address argument */ + ip_off = stack_size; + if (flags & BPF_TRAMP_F_SKIP_FRAME) /* skip patched call instruction and point orig_call to actual * body of the kernel function. @@ -2042,23 +2037,25 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */ EMIT1(0x53); /* push rbx */ + /* Store number of arguments of the traced function: + * mov rax, nr_args + * mov QWORD PTR [rbp - args_off], rax + */ + emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args); + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -args_off); + if (flags & BPF_TRAMP_F_IP_ARG) { /* Store IP address of the traced function: * mov rax, QWORD PTR [rbp + 8] * sub rax, X86_PATCH_SIZE - * mov QWORD PTR [rbp - stack_size], rax + * mov QWORD PTR [rbp - ip_off], rax */ emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8); EMIT4(0x48, 0x83, 0xe8, X86_PATCH_SIZE); - emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -stack_size); - - /* Continue with stack_size for regs storage, stack will - * be correctly restored with 'leave' instruction. - */ - stack_size -= 8; + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off); } - save_regs(m, &prog, nr_args, stack_size); + save_regs(m, &prog, nr_args, regs_off); if (flags & BPF_TRAMP_F_CALL_ORIG) { /* arg1: mov rdi, im */ @@ -2070,7 +2067,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } if (fentry->nr_progs) - if (invoke_bpf(m, &prog, fentry, stack_size, + if (invoke_bpf(m, &prog, fentry, regs_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) return -EINVAL; @@ -2080,7 +2077,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (!branches) return -ENOMEM; - if (invoke_bpf_mod_ret(m, &prog, fmod_ret, stack_size, + if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off, branches)) { ret = -EINVAL; goto cleanup; @@ -2088,7 +2085,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } if (flags & BPF_TRAMP_F_CALL_ORIG) { - restore_regs(m, &prog, nr_args, stack_size); + restore_regs(m, &prog, nr_args, regs_off); /* call original function */ if (emit_call(&prog, orig_call, prog)) { @@ -2118,13 +2115,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i } if (fexit->nr_progs) - if (invoke_bpf(m, &prog, fexit, stack_size, false)) { + if (invoke_bpf(m, &prog, fexit, regs_off, false)) { ret = -EINVAL; goto cleanup; } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_regs(m, &prog, nr_args, stack_size); + restore_regs(m, &prog, nr_args, regs_off); /* This needs to be done regardless. If there were fmod_ret programs, * the return value is only updated on the stack and still needs to be @@ -2161,24 +2158,6 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i return ret; } -static int emit_fallback_jump(u8 **pprog) -{ - u8 *prog = *pprog; - int err = 0; - -#ifdef CONFIG_RETPOLINE - /* Note that this assumes the the compiler uses external - * thunks for indirect calls. Both clang and GCC use the same - * naming convention for external thunks. - */ - err = emit_jump(&prog, __x86_indirect_thunk_rdx, prog); -#else - EMIT2(0xFF, 0xE2); /* jmp rdx */ -#endif - *pprog = prog; - return err; -} - static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs) { u8 *jg_reloc, *prog = *pprog; @@ -2200,9 +2179,7 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs) if (err) return err; - err = emit_fallback_jump(&prog); /* jmp thunk/indirect */ - if (err) - return err; + emit_indirect_jump(&prog, 2 /* rdx */, prog); *pprog = prog; return 0; diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index 3bfda5f502..429a89c546 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -15,6 +15,7 @@ #include #include #include +#include #include /* @@ -1267,6 +1268,21 @@ static void emit_epilogue(u8 **pprog, u32 stack_depth) *pprog = prog; } +static int emit_jmp_edx(u8 **pprog, u8 *ip) +{ + u8 *prog = *pprog; + int cnt = 0; + +#ifdef CONFIG_RETPOLINE + EMIT1_off32(0xE9, (u8 *)__x86_indirect_thunk_edx - (ip + 5)); +#else + EMIT2(0xFF, 0xE2); +#endif + *pprog = prog; + + return cnt; +} + /* * Generate the following code: * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ... @@ -1280,7 +1296,7 @@ static void emit_epilogue(u8 **pprog, u32 stack_depth) * goto *(prog->bpf_func + prologue_size); * out: */ -static void emit_bpf_tail_call(u8 **pprog) +static void emit_bpf_tail_call(u8 **pprog, u8 *ip) { u8 *prog = *pprog; int cnt = 0; @@ -1307,7 +1323,7 @@ static void emit_bpf_tail_call(u8 **pprog) EMIT2(IA32_JBE, jmp_label(jmp_label1, 2)); /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; */ lo = (u32)MAX_TAIL_CALL_CNT; @@ -1321,7 +1337,7 @@ static void emit_bpf_tail_call(u8 **pprog) /* cmp ecx,lo */ EMIT3(0x83, add_1reg(0xF8, IA32_ECX), lo); - /* ja out */ + /* jae out */ EMIT2(IA32_JAE, jmp_label(jmp_label1, 2)); /* add eax,0x1 */ @@ -1362,7 +1378,7 @@ static void emit_bpf_tail_call(u8 **pprog) * eax == ctx (1st arg) * edx == prog->bpf_func + prologue_size */ - RETPOLINE_EDX_BPF_JIT(); + cnt += emit_jmp_edx(&prog, ip + cnt); if (jmp_label1 == -1) jmp_label1 = cnt; @@ -2122,7 +2138,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, break; } case BPF_JMP | BPF_TAIL_CALL: - emit_bpf_tail_call(&prog); + emit_bpf_tail_call(&prog, image + addrs[i - 1]); break; /* cond jump */ diff --git a/arch/x86/pci/acpi.c b/arch/x86/pci/acpi.c index 948656069c..052f1d78a5 100644 --- a/arch/x86/pci/acpi.c +++ b/arch/x86/pci/acpi.c @@ -20,7 +20,7 @@ struct pci_root_info { }; static bool pci_use_crs = true; -static bool pci_ignore_seg = false; +static bool pci_ignore_seg; static int __init set_use_crs(const struct dmi_system_id *id) { diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 5debe4ac6f..9bb1e29411 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -183,7 +184,7 @@ static int xen_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (ret) goto error; i = 0; - for_each_pci_msi_entry(msidesc, dev) { + msi_for_each_desc(msidesc, &dev->dev, MSI_DESC_NOTASSOCIATED) { irq = xen_bind_pirq_msi_to_irq(dev, msidesc, v[i], (type == PCI_CAP_ID_MSI) ? nvec : 1, (type == PCI_CAP_ID_MSIX) ? @@ -234,7 +235,7 @@ static int xen_hvm_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (type == PCI_CAP_ID_MSI && nvec > 1) return 1; - for_each_pci_msi_entry(msidesc, dev) { + msi_for_each_desc(msidesc, &dev->dev, MSI_DESC_NOTASSOCIATED) { pirq = xen_allocate_pirq_msi(dev, msidesc); if (pirq < 0) { irq = -ENODEV; @@ -269,7 +270,7 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) int ret = 0; struct msi_desc *msidesc; - for_each_pci_msi_entry(msidesc, dev) { + msi_for_each_desc(msidesc, &dev->dev, MSI_DESC_NOTASSOCIATED) { struct physdev_map_pirq map_irq; domid_t domid; @@ -305,7 +306,7 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) return -EINVAL; map_irq.table_base = pci_resource_start(dev, bir); - map_irq.entry_nr = msidesc->msi_attrib.entry_nr; + map_irq.entry_nr = msidesc->msi_index; } ret = -EINVAL; @@ -350,10 +351,13 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) return ret; } -static void xen_initdom_restore_msi_irqs(struct pci_dev *dev) +bool xen_initdom_restore_msi(struct pci_dev *dev) { int ret = 0; + if (!xen_initial_domain()) + return true; + if (pci_seg_supported) { struct physdev_pci_device restore_ext; @@ -374,10 +378,10 @@ static void xen_initdom_restore_msi_irqs(struct pci_dev *dev) ret = HYPERVISOR_physdev_op(PHYSDEVOP_restore_msi, &restore); WARN(ret && ret != -ENOSYS, "restore_msi -> %d\n", ret); } + return false; } #else /* CONFIG_XEN_PV_DOM0 */ #define xen_initdom_setup_msi_irqs NULL -#define xen_initdom_restore_msi_irqs NULL #endif /* !CONFIG_XEN_PV_DOM0 */ static void xen_teardown_msi_irqs(struct pci_dev *dev) @@ -385,19 +389,15 @@ static void xen_teardown_msi_irqs(struct pci_dev *dev) struct msi_desc *msidesc; int i; - for_each_pci_msi_entry(msidesc, dev) { - if (msidesc->irq) { - for (i = 0; i < msidesc->nvec_used; i++) - xen_destroy_irq(msidesc->irq + i); - } + msi_for_each_desc(msidesc, &dev->dev, MSI_DESC_ASSOCIATED) { + for (i = 0; i < msidesc->nvec_used; i++) + xen_destroy_irq(msidesc->irq + i); } } static void xen_pv_teardown_msi_irqs(struct pci_dev *dev) { - struct msi_desc *msidesc = first_pci_msi_entry(dev); - - if (msidesc->msi_attrib.is_msix) + if (dev->msix_enabled) xen_pci_frontend_disable_msix(dev); else xen_pci_frontend_disable_msi(dev); @@ -413,10 +413,7 @@ static int xen_msi_domain_alloc_irqs(struct irq_domain *domain, if (WARN_ON_ONCE(!dev_is_pci(dev))) return -EINVAL; - if (first_msi_entry(dev)->msi_attrib.is_msix) - type = PCI_CAP_ID_MSIX; - else - type = PCI_CAP_ID_MSI; + type = to_pci_dev(dev)->msix_enabled ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; return xen_msi_ops.setup_msi_irqs(to_pci_dev(dev), nvec, type); } @@ -465,12 +462,10 @@ static __init struct irq_domain *xen_create_pci_msi_domain(void) static __init void xen_setup_pci_msi(void) { if (xen_pv_domain()) { - if (xen_initial_domain()) { + if (xen_initial_domain()) xen_msi_ops.setup_msi_irqs = xen_initdom_setup_msi_irqs; - x86_msi.restore_msi_irqs = xen_initdom_restore_msi_irqs; - } else { + else xen_msi_ops.setup_msi_irqs = xen_setup_msi_irqs; - } xen_msi_ops.teardown_msi_irqs = xen_pv_teardown_msi_irqs; pci_msi_ignore_mask = 1; } else if (xen_hvm_domain()) { @@ -585,78 +580,3 @@ int __init pci_xen_initial_domain(void) } #endif -#ifdef CONFIG_XEN_DOM0 - -struct xen_device_domain_owner { - domid_t domain; - struct pci_dev *dev; - struct list_head list; -}; - -static DEFINE_SPINLOCK(dev_domain_list_spinlock); -static struct list_head dev_domain_list = LIST_HEAD_INIT(dev_domain_list); - -static struct xen_device_domain_owner *find_device(struct pci_dev *dev) -{ - struct xen_device_domain_owner *owner; - - list_for_each_entry(owner, &dev_domain_list, list) { - if (owner->dev == dev) - return owner; - } - return NULL; -} - -int xen_find_device_domain_owner(struct pci_dev *dev) -{ - struct xen_device_domain_owner *owner; - int domain = -ENODEV; - - spin_lock(&dev_domain_list_spinlock); - owner = find_device(dev); - if (owner) - domain = owner->domain; - spin_unlock(&dev_domain_list_spinlock); - return domain; -} -EXPORT_SYMBOL_GPL(xen_find_device_domain_owner); - -int xen_register_device_domain_owner(struct pci_dev *dev, uint16_t domain) -{ - struct xen_device_domain_owner *owner; - - owner = kzalloc(sizeof(struct xen_device_domain_owner), GFP_KERNEL); - if (!owner) - return -ENODEV; - - spin_lock(&dev_domain_list_spinlock); - if (find_device(dev)) { - spin_unlock(&dev_domain_list_spinlock); - kfree(owner); - return -EEXIST; - } - owner->domain = domain; - owner->dev = dev; - list_add_tail(&owner->list, &dev_domain_list); - spin_unlock(&dev_domain_list_spinlock); - return 0; -} -EXPORT_SYMBOL_GPL(xen_register_device_domain_owner); - -int xen_unregister_device_domain_owner(struct pci_dev *dev) -{ - struct xen_device_domain_owner *owner; - - spin_lock(&dev_domain_list_spinlock); - owner = find_device(dev); - if (!owner) { - spin_unlock(&dev_domain_list_spinlock); - return -ENODEV; - } - list_del(&owner->list); - spin_unlock(&dev_domain_list_spinlock); - kfree(owner); - return 0; -} -EXPORT_SYMBOL_GPL(xen_unregister_device_domain_owner); -#endif /* CONFIG_XEN_DOM0 */ diff --git a/arch/x86/platform/ce4100/falconfalls.dts b/arch/x86/platform/ce4100/falconfalls.dts index 0ac3d43571..65fa3d8662 100644 --- a/arch/x86/platform/ce4100/falconfalls.dts +++ b/arch/x86/platform/ce4100/falconfalls.dts @@ -249,7 +249,7 @@ i2c@1 { gpio@26 { #gpio-cells = <2>; - compatible = "ti,pcf8575"; + compatible = "nxp,pcf8575"; reg = <0x26>; gpio-controller; }; @@ -263,7 +263,7 @@ i2c@2 { gpio@26 { #gpio-cells = <2>; - compatible = "ti,pcf8575"; + compatible = "nxp,pcf8575"; reg = <0x26>; gpio-controller; }; diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 7515e78ef8..1f3675453a 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include #include @@ -284,7 +284,8 @@ static void __init __map_region(efi_memory_desc_t *md, u64 va) if (!(md->attribute & EFI_MEMORY_WB)) flags |= _PAGE_PCD; - if (sev_active() && md->type != EFI_MEMORY_MAPPED_IO) + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) && + md->type != EFI_MEMORY_MAPPED_IO) flags |= _PAGE_ENC; pfn = md->phys_addr >> PAGE_SHIFT; @@ -390,7 +391,7 @@ static int __init efi_update_mem_attr(struct mm_struct *mm, efi_memory_desc_t *m if (!(md->attribute & EFI_MEMORY_RO)) pf |= _PAGE_RW; - if (sev_active()) + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) pf |= _PAGE_ENC; return efi_update_mappings(md, pf); @@ -438,7 +439,7 @@ void __init efi_runtime_update_mappings(void) (md->type != EFI_RUNTIME_SERVICES_CODE)) pf |= _PAGE_RW; - if (sev_active()) + if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) pf |= _PAGE_ENC; efi_update_mappings(md, pf); diff --git a/arch/x86/platform/efi/efi_thunk_64.S b/arch/x86/platform/efi/efi_thunk_64.S index fd3dd1708e..25799d7686 100644 --- a/arch/x86/platform/efi/efi_thunk_64.S +++ b/arch/x86/platform/efi/efi_thunk_64.S @@ -36,6 +36,17 @@ SYM_CODE_START(__efi64_thunk) movq efi_mixed_mode_stack_pa(%rip), %rsp push %rax + /* + * Copy args passed via the stack + */ + subq $0x24, %rsp + movq 0x18(%rax), %rbp + movq 0x20(%rax), %rbx + movq 0x28(%rax), %rax + movl %ebp, 0x18(%rsp) + movl %ebx, 0x1c(%rsp) + movl %eax, 0x20(%rsp) + /* * Calculate the physical address of the kernel text. */ @@ -47,7 +58,6 @@ SYM_CODE_START(__efi64_thunk) subq %rax, %rbp subq %rax, %rbx - subq $28, %rsp movl %ebx, 0x0(%rsp) /* return address */ movl %esi, 0x4(%rsp) movl %edx, 0x8(%rsp) @@ -60,10 +70,10 @@ SYM_CODE_START(__efi64_thunk) pushq %rdi /* EFI runtime service address */ lretq -1: movq 24(%rsp), %rsp +1: movq 0x20(%rsp), %rsp pop %rbx pop %rbp - retq + RET .code32 2: pushl $__KERNEL_CS diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index 6665f88020..9f2b251e83 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile index 95ea17a9d2..ae53d54d79 100644 --- a/arch/x86/purgatory/Makefile +++ b/arch/x86/purgatory/Makefile @@ -16,7 +16,7 @@ CFLAGS_sha256.o := -D__DISABLE_EXPORTS # When linking purgatory.ro with -r unresolved symbols are not checked, # also link a purgatory.chk binary without -r to check for unresolved symbols. -PURGATORY_LDFLAGS := -e purgatory_start -nostdlib -z nodefaultlib +PURGATORY_LDFLAGS := -e purgatory_start -z nodefaultlib LDFLAGS_purgatory.ro := -r $(PURGATORY_LDFLAGS) LDFLAGS_purgatory.chk := $(PURGATORY_LDFLAGS) targets += purgatory.ro purgatory.chk diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c index 1d20ed4b28..c5e29db02a 100644 --- a/arch/x86/realmode/init.c +++ b/arch/x86/realmode/init.c @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include @@ -70,10 +70,10 @@ void __init reserve_real_mode(void) static void sme_sev_setup_real_mode(struct trampoline_header *th) { #ifdef CONFIG_AMD_MEM_ENCRYPT - if (sme_active()) + if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) th->flags |= TH_FLAGS_SME_ACTIVE; - if (sev_es_active()) { + if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) { /* * Skip the call to verify_cpu() in secondary_startup_64 as it * will cause #VC exceptions when the AP can't handle them yet. @@ -108,7 +108,7 @@ static void __init setup_real_mode(void) * decrypted memory in order to bring up other processors * successfully. This is not needed for SEV. */ - if (sme_active()) + if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) set_memory_decrypted((unsigned long)base, size >> PAGE_SHIFT); memcpy(base, real_mode_blob, size); diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c index 27c82207d3..e2c5b29612 100644 --- a/arch/x86/tools/relocs.c +++ b/arch/x86/tools/relocs.c @@ -14,6 +14,10 @@ static Elf_Ehdr ehdr; static unsigned long shnum; static unsigned int shstrndx; +static unsigned int shsymtabndx; +static unsigned int shxsymtabndx; + +static int sym_index(Elf_Sym *sym); struct relocs { uint32_t *offset; @@ -35,6 +39,7 @@ struct section { Elf_Shdr shdr; struct section *link; Elf_Sym *symtab; + Elf32_Word *xsymtab; Elf_Rel *reltab; char *strtab; }; @@ -63,7 +68,9 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = { "(__parainstructions|__alt_instructions)(_end)?|" "(__iommu_table|__apicdrivers|__smp_locks)(_end)?|" "__(start|end)_pci_.*|" +#if CONFIG_FW_LOADER "__(start|end)_builtin_fw|" +#endif "__(start|stop)___ksymtab(_gpl)?|" "__(start|stop)___kcrctab(_gpl)?|" "__(start|stop)___param|" @@ -268,7 +275,7 @@ static const char *sym_name(const char *sym_strtab, Elf_Sym *sym) name = sym_strtab + sym->st_name; } else { - name = sec_name(sym->st_shndx); + name = sec_name(sym_index(sym)); } return name; } @@ -338,6 +345,23 @@ static uint64_t elf64_to_cpu(uint64_t val) #define elf_xword_to_cpu(x) elf32_to_cpu(x) #endif +static int sym_index(Elf_Sym *sym) +{ + Elf_Sym *symtab = secs[shsymtabndx].symtab; + Elf32_Word *xsymtab = secs[shxsymtabndx].xsymtab; + unsigned long offset; + int index; + + if (sym->st_shndx != SHN_XINDEX) + return sym->st_shndx; + + /* calculate offset of sym from head of table. */ + offset = (unsigned long)sym - (unsigned long)symtab; + index = offset / sizeof(*sym); + + return elf32_to_cpu(xsymtab[index]); +} + static void read_ehdr(FILE *fp) { if (fread(&ehdr, sizeof(ehdr), 1, fp) != 1) { @@ -471,31 +495,60 @@ static void read_strtabs(FILE *fp) static void read_symtabs(FILE *fp) { int i,j; + for (i = 0; i < shnum; i++) { struct section *sec = &secs[i]; - if (sec->shdr.sh_type != SHT_SYMTAB) { + int num_syms; + + switch (sec->shdr.sh_type) { + case SHT_SYMTAB_SHNDX: + sec->xsymtab = malloc(sec->shdr.sh_size); + if (!sec->xsymtab) { + die("malloc of %" FMT " bytes for xsymtab failed\n", + sec->shdr.sh_size); + } + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) { + die("Seek to %" FMT " failed: %s\n", + sec->shdr.sh_offset, strerror(errno)); + } + if (fread(sec->xsymtab, 1, sec->shdr.sh_size, fp) + != sec->shdr.sh_size) { + die("Cannot read extended symbol table: %s\n", + strerror(errno)); + } + shxsymtabndx = i; + continue; + + case SHT_SYMTAB: + num_syms = sec->shdr.sh_size / sizeof(Elf_Sym); + + sec->symtab = malloc(sec->shdr.sh_size); + if (!sec->symtab) { + die("malloc of %" FMT " bytes for symtab failed\n", + sec->shdr.sh_size); + } + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) { + die("Seek to %" FMT " failed: %s\n", + sec->shdr.sh_offset, strerror(errno)); + } + if (fread(sec->symtab, 1, sec->shdr.sh_size, fp) + != sec->shdr.sh_size) { + die("Cannot read symbol table: %s\n", + strerror(errno)); + } + for (j = 0; j < num_syms; j++) { + Elf_Sym *sym = &sec->symtab[j]; + + sym->st_name = elf_word_to_cpu(sym->st_name); + sym->st_value = elf_addr_to_cpu(sym->st_value); + sym->st_size = elf_xword_to_cpu(sym->st_size); + sym->st_shndx = elf_half_to_cpu(sym->st_shndx); + } + shsymtabndx = i; + continue; + + default: continue; - } - sec->symtab = malloc(sec->shdr.sh_size); - if (!sec->symtab) { - die("malloc of %" FMT " bytes for symtab failed\n", - sec->shdr.sh_size); - } - if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) { - die("Seek to %" FMT " failed: %s\n", - sec->shdr.sh_offset, strerror(errno)); - } - if (fread(sec->symtab, 1, sec->shdr.sh_size, fp) - != sec->shdr.sh_size) { - die("Cannot read symbol table: %s\n", - strerror(errno)); - } - for (j = 0; j < sec->shdr.sh_size/sizeof(Elf_Sym); j++) { - Elf_Sym *sym = &sec->symtab[j]; - sym->st_name = elf_word_to_cpu(sym->st_name); - sym->st_value = elf_addr_to_cpu(sym->st_value); - sym->st_size = elf_xword_to_cpu(sym->st_size); - sym->st_shndx = elf_half_to_cpu(sym->st_shndx); } } } @@ -762,7 +815,9 @@ static void percpu_init(void) */ static int is_percpu_sym(ElfW(Sym) *sym, const char *symname) { - return (sym->st_shndx == per_cpu_shndx) && + int shndx = sym_index(sym); + + return (shndx == per_cpu_shndx) && strcmp(symname, "__init_begin") && strcmp(symname, "__per_cpu_load") && strncmp(symname, "init_per_cpu_", 13); @@ -1095,7 +1150,7 @@ static int do_reloc_info(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym, sec_name(sec->shdr.sh_info), rel_type(ELF_R_TYPE(rel->r_info)), symname, - sec_name(sym->st_shndx)); + sec_name(sym_index(sym))); return 0; } diff --git a/arch/x86/um/Kconfig b/arch/x86/um/Kconfig index 95d26a6908..40d6a06e41 100644 --- a/arch/x86/um/Kconfig +++ b/arch/x86/um/Kconfig @@ -8,7 +8,6 @@ endmenu config UML_X86 def_bool y - select GENERIC_FIND_FIRST_BIT config 64BIT bool "64-bit kernel" if "$(SUBARCH)" = "x86" diff --git a/arch/x86/um/Makefile b/arch/x86/um/Makefile index 5ccb18290d..ba5789c358 100644 --- a/arch/x86/um/Makefile +++ b/arch/x86/um/Makefile @@ -40,7 +40,7 @@ $(obj)/user-offsets.s: c_flags = -Wp,-MD,$(depfile) $(USER_CFLAGS) \ -Iarch/x86/include/generated targets += user-offsets.s -include/generated/user_constants.h: $(obj)/user-offsets.s +include/generated/user_constants.h: $(obj)/user-offsets.s FORCE $(call filechk,offsets,__USER_CONSTANT_H__) UNPROFILE_OBJS := stub_segv.o diff --git a/arch/x86/um/asm/barrier.h b/arch/x86/um/asm/barrier.h index 165be7f9a9..4da3369656 100644 --- a/arch/x86/um/asm/barrier.h +++ b/arch/x86/um/asm/barrier.h @@ -2,6 +2,7 @@ #ifndef _ASM_UM_BARRIER_H_ #define _ASM_UM_BARRIER_H_ +#include #include /* diff --git a/arch/x86/um/asm/segment.h b/arch/x86/um/asm/segment.h index 453db37715..2ef507bc69 100644 --- a/arch/x86/um/asm/segment.h +++ b/arch/x86/um/asm/segment.h @@ -8,12 +8,4 @@ extern int host_gdt_entry_tls_min; #define GDT_ENTRY_TLS_MIN host_gdt_entry_tls_min #define GDT_ENTRY_TLS_MAX (GDT_ENTRY_TLS_MIN + GDT_ENTRY_TLS_ENTRIES - 1) -typedef struct { - unsigned long seg; -} mm_segment_t; - -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) -#define KERNEL_DS MAKE_MM_SEG(~0UL) -#define USER_DS MAKE_MM_SEG(TASK_SIZE) - #endif diff --git a/arch/x86/um/os-Linux/registers.c b/arch/x86/um/os-Linux/registers.c index 3c423dfcd7..df8f4b4bf9 100644 --- a/arch/x86/um/os-Linux/registers.c +++ b/arch/x86/um/os-Linux/registers.c @@ -15,6 +15,7 @@ #include #include #include +#include int have_xstate_support; diff --git a/arch/x86/um/ptrace_32.c b/arch/x86/um/ptrace_32.c index 2497bac560..0bc4b73a9c 100644 --- a/arch/x86/um/ptrace_32.c +++ b/arch/x86/um/ptrace_32.c @@ -7,6 +7,7 @@ #include #include #include +#include #include extern int arch_switch_tls(struct task_struct *to); diff --git a/arch/x86/um/ptrace_64.c b/arch/x86/um/ptrace_64.c index 1401899dee..289d0159b0 100644 --- a/arch/x86/um/ptrace_64.c +++ b/arch/x86/um/ptrace_64.c @@ -11,6 +11,7 @@ #define __FRAME_OFFSETS #include #include +#include #include /* diff --git a/arch/x86/um/shared/sysdep/syscalls_64.h b/arch/x86/um/shared/sysdep/syscalls_64.h index 8a7d5e1da9..48d6cd12f8 100644 --- a/arch/x86/um/shared/sysdep/syscalls_64.h +++ b/arch/x86/um/shared/sysdep/syscalls_64.h @@ -23,9 +23,6 @@ extern syscall_handler_t *sys_call_table[]; UPT_SYSCALL_ARG5(®s->regs), \ UPT_SYSCALL_ARG6(®s->regs))) -extern long old_mmap(unsigned long addr, unsigned long len, - unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff); extern syscall_handler_t sys_modify_ldt; extern syscall_handler_t sys_arch_prctl; diff --git a/arch/x86/um/signal.c b/arch/x86/um/signal.c index 7c11c9e5d7..263e1d08f2 100644 --- a/arch/x86/um/signal.c +++ b/arch/x86/um/signal.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #ifdef CONFIG_X86_32 diff --git a/arch/x86/um/sys_call_table_32.c b/arch/x86/um/sys_call_table_32.c index 0575decb5e..89df5d89d6 100644 --- a/arch/x86/um/sys_call_table_32.c +++ b/arch/x86/um/sys_call_table_32.c @@ -9,8 +9,6 @@ #include #include -#define __NO_STUBS - /* * Below you can see, in terms of #define's, the differences between the x86-64 * and the UML syscall table. @@ -23,8 +21,6 @@ #define sys_vm86old sys_ni_syscall #define sys_vm86 sys_ni_syscall -#define old_mmap sys_old_mmap - #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) #define __SYSCALL(nr, sym) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); diff --git a/arch/x86/um/sys_call_table_64.c b/arch/x86/um/sys_call_table_64.c index 95725b5a41..b0b4cfd230 100644 --- a/arch/x86/um/sys_call_table_64.c +++ b/arch/x86/um/sys_call_table_64.c @@ -9,8 +9,6 @@ #include #include -#define __NO_STUBS - /* * Below you can see, in terms of #define's, the differences between the x86-64 * and the UML syscall table. @@ -20,21 +18,6 @@ #define sys_iopl sys_ni_syscall #define sys_ioperm sys_ni_syscall -/* - * The UML TLS problem. Note that x86_64 does not implement this, so the below - * is needed only for the ia32 compatibility. - */ - -/* On UML we call it this way ("old" means it's not mmap2) */ -#define sys_mmap old_mmap - -#define stub_clone sys_clone -#define stub_fork sys_fork -#define stub_vfork sys_vfork -#define stub_execve sys_execve -#define stub_execveat sys_execveat -#define stub_rt_sigreturn sys_rt_sigreturn - #define __SYSCALL(nr, sym) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); #include diff --git a/arch/x86/um/syscalls_64.c b/arch/x86/um/syscalls_64.c index 8249685b40..fe5323f0c4 100644 --- a/arch/x86/um/syscalls_64.c +++ b/arch/x86/um/syscalls_64.c @@ -10,6 +10,7 @@ #include #include #include /* XXX This should get the constants from libc */ +#include #include #include @@ -88,3 +89,13 @@ void arch_switch_to(struct task_struct *to) arch_prctl(to, ARCH_SET_FS, (void __user *) to->thread.arch.fs); } + +SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, + unsigned long, fd, unsigned long, off) +{ + if (off & ~PAGE_MASK) + return -EINVAL; + + return ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); +} diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig index 6bcd3d8ca6..85246dd9fa 100644 --- a/arch/x86/xen/Kconfig +++ b/arch/x86/xen/Kconfig @@ -23,6 +23,7 @@ config XEN_PV select PARAVIRT_XXL select XEN_HAVE_PVMMU select XEN_HAVE_VPMU + select GUEST_PERF_EVENTS help Support running as a Xen PV guest. diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 95d970359e..30c6e986a6 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -31,25 +31,10 @@ EXPORT_SYMBOL_GPL(hypercall_page); * Pointer to the xen_vcpu_info structure or * &HYPERVISOR_shared_info->vcpu_info[cpu]. See xen_hvm_init_shared_info * and xen_vcpu_setup for details. By default it points to share_info->vcpu_info - * but if the hypervisor supports VCPUOP_register_vcpu_info then it can point - * to xen_vcpu_info. The pointer is used in __xen_evtchn_do_upcall to - * acknowledge pending events. - * Also more subtly it is used by the patched version of irq enable/disable - * e.g. xen_irq_enable_direct and xen_iret in PV mode. - * - * The desire to be able to do those mask/unmask operations as a single - * instruction by using the per-cpu offset held in %gs is the real reason - * vcpu info is in a per-cpu pointer and the original reason for this - * hypercall. - * + * but during boot it is switched to point to xen_vcpu_info. + * The pointer is used in __xen_evtchn_do_upcall to acknowledge pending events. */ DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu); - -/* - * Per CPU pages used if hypervisor supports VCPUOP_register_vcpu_info - * hypercall. This can be used both in PV and PVHVM mode. The structure - * overrides the default per_cpu(xen_vcpu, cpu) value. - */ DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info); /* Linux <-> Xen vCPU id mapping */ @@ -84,21 +69,6 @@ EXPORT_SYMBOL(xen_start_flags); */ struct shared_info *HYPERVISOR_shared_info = &xen_dummy_shared_info; -/* - * Flag to determine whether vcpu info placement is available on all - * VCPUs. We assume it is to start with, and then set it to zero on - * the first failure. This is because it can succeed on some VCPUs - * and not others, since it can involve hypervisor memory allocation, - * or because the guest failed to guarantee all the appropriate - * constraints on all VCPUs (ie buffer can't cross a page boundary). - * - * Note that any particular CPU may be using a placed vcpu structure, - * but we can only optimise if the all are. - * - * 0: not available, 1: available - */ -int xen_have_vcpu_info_placement = 1; - static int xen_cpu_up_online(unsigned int cpu) { xen_init_lock_cpu(cpu); @@ -124,10 +94,8 @@ int xen_cpuhp_setup(int (*cpu_up_prepare_cb)(unsigned int), return rc >= 0 ? 0 : rc; } -static int xen_vcpu_setup_restore(int cpu) +static void xen_vcpu_setup_restore(int cpu) { - int rc = 0; - /* Any per_cpu(xen_vcpu) is stale, so reset it */ xen_vcpu_info_reset(cpu); @@ -136,11 +104,8 @@ static int xen_vcpu_setup_restore(int cpu) * be handled by hotplug. */ if (xen_pv_domain() || - (xen_hvm_domain() && cpu_online(cpu))) { - rc = xen_vcpu_setup(cpu); - } - - return rc; + (xen_hvm_domain() && cpu_online(cpu))) + xen_vcpu_setup(cpu); } /* @@ -150,7 +115,7 @@ static int xen_vcpu_setup_restore(int cpu) */ void xen_vcpu_restore(void) { - int cpu, rc; + int cpu; for_each_possible_cpu(cpu) { bool other_cpu = (cpu != smp_processor_id()); @@ -170,20 +135,9 @@ void xen_vcpu_restore(void) if (xen_pv_domain() || xen_feature(XENFEAT_hvm_safe_pvclock)) xen_setup_runstate_info(cpu); - rc = xen_vcpu_setup_restore(cpu); - if (rc) - pr_emerg_once("vcpu restore failed for cpu=%d err=%d. " - "System will hang.\n", cpu, rc); - /* - * In case xen_vcpu_setup_restore() fails, do not bring up the - * VCPU. This helps us avoid the resulting OOPS when the VCPU - * accesses pvclock_vcpu_time via xen_vcpu (which is NULL.) - * Note that this does not improve the situation much -- now the - * VM hangs instead of OOPSing -- with the VCPUs that did not - * fail, spinning in stop_machine(), waiting for the failed - * VCPUs to come up. - */ - if (other_cpu && is_up && (rc == 0) && + xen_vcpu_setup_restore(cpu); + + if (other_cpu && is_up && HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL)) BUG(); } @@ -200,7 +154,7 @@ void xen_vcpu_info_reset(int cpu) } } -int xen_vcpu_setup(int cpu) +void xen_vcpu_setup(int cpu) { struct vcpu_register_vcpu_info info; int err; @@ -221,44 +175,26 @@ int xen_vcpu_setup(int cpu) */ if (xen_hvm_domain()) { if (per_cpu(xen_vcpu, cpu) == &per_cpu(xen_vcpu_info, cpu)) - return 0; + return; } - if (xen_have_vcpu_info_placement) { - vcpup = &per_cpu(xen_vcpu_info, cpu); - info.mfn = arbitrary_virt_to_mfn(vcpup); - info.offset = offset_in_page(vcpup); + vcpup = &per_cpu(xen_vcpu_info, cpu); + info.mfn = arbitrary_virt_to_mfn(vcpup); + info.offset = offset_in_page(vcpup); - /* - * Check to see if the hypervisor will put the vcpu_info - * structure where we want it, which allows direct access via - * a percpu-variable. - * N.B. This hypercall can _only_ be called once per CPU. - * Subsequent calls will error out with -EINVAL. This is due to - * the fact that hypervisor has no unregister variant and this - * hypercall does not allow to over-write info.mfn and - * info.offset. - */ - err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, - xen_vcpu_nr(cpu), &info); + /* + * N.B. This hypercall can _only_ be called once per CPU. + * Subsequent calls will error out with -EINVAL. This is due to + * the fact that hypervisor has no unregister variant and this + * hypercall does not allow to over-write info.mfn and + * info.offset. + */ + err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, xen_vcpu_nr(cpu), + &info); + if (err) + panic("register_vcpu_info failed: cpu=%d err=%d\n", cpu, err); - if (err) { - pr_warn_once("register_vcpu_info failed: cpu=%d err=%d\n", - cpu, err); - xen_have_vcpu_info_placement = 0; - } else { - /* - * This cpu is using the registered vcpu info, even if - * later ones fail to. - */ - per_cpu(xen_vcpu, cpu) = vcpup; - } - } - - if (!xen_have_vcpu_info_placement) - xen_vcpu_info_reset(cpu); - - return ((per_cpu(xen_vcpu, cpu) == NULL) ? -ENODEV : 0); + per_cpu(xen_vcpu, cpu) = vcpup; } void __init xen_banner(void) diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index e68ea5f4ad..517a9d8d8f 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -163,9 +164,9 @@ static int xen_cpu_up_prepare_hvm(unsigned int cpu) per_cpu(xen_vcpu_id, cpu) = cpu_acpi_id(cpu); else per_cpu(xen_vcpu_id, cpu) = cpu; - rc = xen_vcpu_setup(cpu); - if (rc || !xen_have_vector_callback) - return rc; + xen_vcpu_setup(cpu); + if (!xen_have_vector_callback) + return 0; if (xen_feature(XENFEAT_hvm_safe_pvclock)) xen_setup_timer(cpu); @@ -184,8 +185,7 @@ static int xen_cpu_dead_hvm(unsigned int cpu) if (xen_have_vector_callback && xen_feature(XENFEAT_hvm_safe_pvclock)) xen_teardown_timer(cpu); - - return 0; + return 0; } static bool no_vector_callback __initdata; @@ -242,15 +242,14 @@ static __init int xen_parse_no_vector_callback(char *arg) } early_param("xen_no_vector_callback", xen_parse_no_vector_callback); -bool __init xen_hvm_need_lapic(void) +static __init bool xen_x2apic_available(void) { - if (xen_pv_domain()) - return false; - if (!xen_hvm_domain()) - return false; - if (xen_feature(XENFEAT_hvm_pirqs) && xen_have_vector_callback) - return false; - return true; + return x2apic_supported(); +} + +static bool __init msi_ext_dest_id(void) +{ + return cpuid_eax(xen_cpuid_base() + 4) & XEN_HVM_CPUID_EXT_DEST_ID; } static __init void xen_hvm_guest_late_init(void) @@ -312,9 +311,10 @@ struct hypervisor_x86 x86_hyper_xen_hvm __initdata = { .detect = xen_platform_hvm, .type = X86_HYPER_XEN_HVM, .init.init_platform = xen_hvm_guest_init, - .init.x2apic_available = xen_x2apic_para_available, + .init.x2apic_available = xen_x2apic_available, .init.init_mem_mapping = xen_hvm_init_mem_mapping, .init.guest_late_init = xen_hvm_guest_late_init, + .init.msi_ext_dest_id = msi_ext_dest_id, .runtime.pin_vcpu = xen_pin_vcpu, .ignore_nopv = true, }; diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 133ef31639..d47c3d176a 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -283,12 +282,12 @@ static void __init xen_init_capabilities(void) } } -static void xen_set_debugreg(int reg, unsigned long val) +static noinstr void xen_set_debugreg(int reg, unsigned long val) { HYPERVISOR_set_debugreg(reg, val); } -static unsigned long xen_get_debugreg(int reg) +static noinstr unsigned long xen_get_debugreg(int reg) { return HYPERVISOR_get_debugreg(reg); } @@ -993,31 +992,13 @@ void __init xen_setup_vcpu_info_placement(void) for_each_possible_cpu(cpu) { /* Set up direct vCPU id mapping for PV guests. */ per_cpu(xen_vcpu_id, cpu) = cpu; - - /* - * xen_vcpu_setup(cpu) can fail -- in which case it - * falls back to the shared_info version for cpus - * where xen_vcpu_nr(cpu) < MAX_VIRT_CPUS. - * - * xen_cpu_up_prepare_pv() handles the rest by failing - * them in hotplug. - */ - (void) xen_vcpu_setup(cpu); + xen_vcpu_setup(cpu); } - /* - * xen_vcpu_setup managed to place the vcpu_info within the - * percpu area for all cpus, so make use of it. - */ - if (xen_have_vcpu_info_placement) { - pv_ops.irq.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct); - pv_ops.irq.irq_disable = - __PV_IS_CALLEE_SAVE(xen_irq_disable_direct); - pv_ops.irq.irq_enable = - __PV_IS_CALLEE_SAVE(xen_irq_enable_direct); - pv_ops.mmu.read_cr2 = - __PV_IS_CALLEE_SAVE(xen_read_cr2_direct); - } + pv_ops.irq.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct); + pv_ops.irq.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct); + pv_ops.irq.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct); + pv_ops.mmu.read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2_direct); } static const struct pv_info xen_info __initconst = { @@ -1025,52 +1006,54 @@ static const struct pv_info xen_info __initconst = { .name = "Xen", }; -static const struct pv_cpu_ops xen_cpu_ops __initconst = { - .cpuid = xen_cpuid, +static const typeof(pv_ops) xen_cpu_ops __initconst = { + .cpu = { + .cpuid = xen_cpuid, - .set_debugreg = xen_set_debugreg, - .get_debugreg = xen_get_debugreg, + .set_debugreg = xen_set_debugreg, + .get_debugreg = xen_get_debugreg, - .read_cr0 = xen_read_cr0, - .write_cr0 = xen_write_cr0, + .read_cr0 = xen_read_cr0, + .write_cr0 = xen_write_cr0, - .write_cr4 = xen_write_cr4, + .write_cr4 = xen_write_cr4, - .wbinvd = native_wbinvd, + .wbinvd = native_wbinvd, - .read_msr = xen_read_msr, - .write_msr = xen_write_msr, + .read_msr = xen_read_msr, + .write_msr = xen_write_msr, - .read_msr_safe = xen_read_msr_safe, - .write_msr_safe = xen_write_msr_safe, + .read_msr_safe = xen_read_msr_safe, + .write_msr_safe = xen_write_msr_safe, - .read_pmc = xen_read_pmc, + .read_pmc = xen_read_pmc, - .load_tr_desc = paravirt_nop, - .set_ldt = xen_set_ldt, - .load_gdt = xen_load_gdt, - .load_idt = xen_load_idt, - .load_tls = xen_load_tls, - .load_gs_index = xen_load_gs_index, + .load_tr_desc = paravirt_nop, + .set_ldt = xen_set_ldt, + .load_gdt = xen_load_gdt, + .load_idt = xen_load_idt, + .load_tls = xen_load_tls, + .load_gs_index = xen_load_gs_index, - .alloc_ldt = xen_alloc_ldt, - .free_ldt = xen_free_ldt, + .alloc_ldt = xen_alloc_ldt, + .free_ldt = xen_free_ldt, - .store_tr = xen_store_tr, + .store_tr = xen_store_tr, - .write_ldt_entry = xen_write_ldt_entry, - .write_gdt_entry = xen_write_gdt_entry, - .write_idt_entry = xen_write_idt_entry, - .load_sp0 = xen_load_sp0, + .write_ldt_entry = xen_write_ldt_entry, + .write_gdt_entry = xen_write_gdt_entry, + .write_idt_entry = xen_write_idt_entry, + .load_sp0 = xen_load_sp0, #ifdef CONFIG_X86_IOPL_IOPERM - .invalidate_io_bitmap = xen_invalidate_io_bitmap, - .update_io_bitmap = xen_update_io_bitmap, + .invalidate_io_bitmap = xen_invalidate_io_bitmap, + .update_io_bitmap = xen_update_io_bitmap, #endif - .io_delay = xen_io_delay, + .io_delay = xen_io_delay, - .start_context_switch = paravirt_start_context_switch, - .end_context_switch = xen_end_context_switch, + .start_context_switch = paravirt_start_context_switch, + .end_context_switch = xen_end_context_switch, + }, }; static void xen_restart(char *msg) @@ -1211,7 +1194,7 @@ asmlinkage __visible void __init xen_start_kernel(void) /* Install Xen paravirt ops */ pv_info = xen_info; - pv_ops.cpu = xen_cpu_ops; + pv_ops.cpu = xen_cpu_ops.cpu; paravirt_iret = xen_iret; xen_init_irq_ops(); @@ -1245,12 +1228,6 @@ asmlinkage __visible void __init xen_start_kernel(void) __supported_pte_mask &= ~_PAGE_GLOBAL; __default_kernel_pte_mask &= ~_PAGE_GLOBAL; - /* - * Prevent page tables from being allocated in highmem, even - * if CONFIG_HIGHPTE is enabled. - */ - __userpte_alloc_gfp &= ~__GFP_HIGHMEM; - /* Get mfn list */ xen_build_dynamic_phys_to_machine(); diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c index dfa091d79c..06c3c2fb4b 100644 --- a/arch/x86/xen/irq.c +++ b/arch/x86/xen/irq.c @@ -19,65 +19,11 @@ * callback mask. We do this in a very simple manner, by making a call * down into Xen. The pending flag will be checked by Xen on return. */ -void xen_force_evtchn_callback(void) +noinstr void xen_force_evtchn_callback(void) { (void)HYPERVISOR_xen_version(0, NULL); } -asmlinkage __visible unsigned long xen_save_fl(void) -{ - struct vcpu_info *vcpu; - unsigned long flags; - - vcpu = this_cpu_read(xen_vcpu); - - /* flag has opposite sense of mask */ - flags = !vcpu->evtchn_upcall_mask; - - /* convert to IF type flag - -0 -> 0x00000000 - -1 -> 0xffffffff - */ - return (-flags) & X86_EFLAGS_IF; -} -PV_CALLEE_SAVE_REGS_THUNK(xen_save_fl); - -asmlinkage __visible void xen_irq_disable(void) -{ - /* There's a one instruction preempt window here. We need to - make sure we're don't switch CPUs between getting the vcpu - pointer and updating the mask. */ - preempt_disable(); - this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1; - preempt_enable_no_resched(); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable); - -asmlinkage __visible void xen_irq_enable(void) -{ - struct vcpu_info *vcpu; - - /* - * We may be preempted as soon as vcpu->evtchn_upcall_mask is - * cleared, so disable preemption to ensure we check for - * events on the VCPU we are still running on. - */ - preempt_disable(); - - vcpu = this_cpu_read(xen_vcpu); - vcpu->evtchn_upcall_mask = 0; - - /* Doesn't matter if we get preempted here, because any - pending event will get dealt with anyway. */ - - barrier(); /* unmask then check (avoid races) */ - if (unlikely(vcpu->evtchn_upcall_pending)) - xen_force_evtchn_callback(); - - preempt_enable(); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_irq_enable); - static void xen_safe_halt(void) { /* Blocking includes an implicit local_irq_enable(). */ @@ -94,17 +40,20 @@ static void xen_halt(void) xen_safe_halt(); } -static const struct pv_irq_ops xen_irq_ops __initconst = { - .save_fl = PV_CALLEE_SAVE(xen_save_fl), - .irq_disable = PV_CALLEE_SAVE(xen_irq_disable), - .irq_enable = PV_CALLEE_SAVE(xen_irq_enable), +static const typeof(pv_ops) xen_irq_ops __initconst = { + .irq = { + /* Initial interrupt flag handling only called while interrupts off. */ + .save_fl = __PV_IS_CALLEE_SAVE(paravirt_ret0), + .irq_disable = __PV_IS_CALLEE_SAVE(paravirt_nop), + .irq_enable = __PV_IS_CALLEE_SAVE(paravirt_BUG), - .safe_halt = xen_safe_halt, - .halt = xen_halt, + .safe_halt = xen_safe_halt, + .halt = xen_halt, + }, }; void __init xen_init_irq_ops(void) { - pv_ops.irq = xen_irq_ops; + pv_ops.irq = xen_irq_ops.irq; x86_init.irqs.intr_init = xen_init_IRQ; } diff --git a/arch/x86/xen/mmu_hvm.c b/arch/x86/xen/mmu_hvm.c index 5740937375..509bdee3ab 100644 --- a/arch/x86/xen/mmu_hvm.c +++ b/arch/x86/xen/mmu_hvm.c @@ -9,39 +9,28 @@ #ifdef CONFIG_PROC_VMCORE /* - * This function is used in two contexts: - * - the kdump kernel has to check whether a pfn of the crashed kernel - * was a ballooned page. vmcore is using this function to decide - * whether to access a pfn of the crashed kernel. - * - the kexec kernel has to check whether a pfn was ballooned by the - * previous kernel. If the pfn is ballooned, handle it properly. - * Returns 0 if the pfn is not backed by a RAM page, the caller may + * The kdump kernel has to check whether a pfn of the crashed kernel + * was a ballooned page. vmcore is using this function to decide + * whether to access a pfn of the crashed kernel. + * Returns "false" if the pfn is not backed by a RAM page, the caller may * handle the pfn special in this case. */ -static int xen_oldmem_pfn_is_ram(unsigned long pfn) +static bool xen_vmcore_pfn_is_ram(struct vmcore_cb *cb, unsigned long pfn) { struct xen_hvm_get_mem_type a = { .domid = DOMID_SELF, .pfn = pfn, }; - int ram; - if (HYPERVISOR_hvm_op(HVMOP_get_mem_type, &a)) - return -ENXIO; - - switch (a.mem_type) { - case HVMMEM_mmio_dm: - ram = 0; - break; - case HVMMEM_ram_rw: - case HVMMEM_ram_ro: - default: - ram = 1; - break; + if (HYPERVISOR_hvm_op(HVMOP_get_mem_type, &a)) { + pr_warn_once("Unexpected HVMOP_get_mem_type failure\n"); + return true; } - - return ram; + return a.mem_type != HVMMEM_mmio_dm; } +static struct vmcore_cb xen_vmcore_cb = { + .pfn_is_ram = xen_vmcore_pfn_is_ram, +}; #endif static void xen_hvm_exit_mmap(struct mm_struct *mm) @@ -75,6 +64,6 @@ void __init xen_hvm_init_mmu_ops(void) if (is_pagetable_dying_supported()) pv_ops.mmu.exit_mmap = xen_hvm_exit_mmap; #ifdef CONFIG_PROC_VMCORE - WARN_ON(register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram)); + register_vmcore_cb(&xen_vmcore_cb); #endif } diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index 3359c23573..0035486692 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -41,7 +41,6 @@ * Jeremy Fitzhardinge , XenSource Inc, 2007 */ #include -#include #include #include #include @@ -86,8 +85,10 @@ #include "mmu.h" #include "debugfs.h" +#ifdef CONFIG_X86_VSYSCALL_EMULATION /* l3 pud for userspace vsyscall mapping */ static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss; +#endif /* * Protects atomic reservation decrease/increase against concurrent increases. @@ -241,9 +242,11 @@ static void xen_set_pmd(pmd_t *ptr, pmd_t val) * Associate a virtual page frame with a given physical page frame * and protection flags for that frame. */ -void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags) +void __init set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags) { - set_pte_vaddr(vaddr, mfn_pte(mfn, flags)); + if (HYPERVISOR_update_va_mapping(vaddr, mfn_pte(mfn, flags), + UVMF_INVLPG)) + BUG(); } static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval) @@ -789,7 +792,9 @@ static void __init xen_mark_pinned(struct mm_struct *mm, struct page *page, static void __init xen_after_bootmem(void) { static_branch_enable(&xen_struct_pages_ready); +#ifdef CONFIG_X86_VSYSCALL_EMULATION SetPagePinned(virt_to_page(level3_user_vsyscall)); +#endif xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP); } @@ -1025,7 +1030,7 @@ static void __init xen_free_ro_pages(unsigned long paddr, unsigned long size) for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) make_lowmem_page_readwrite(vaddr); - memblock_free(paddr, size); + memblock_phys_free(paddr, size); } static void __init xen_cleanmfnmap_free_pgtbl(void *pgtbl, bool unpin) @@ -1151,7 +1156,7 @@ static void __init xen_pagetable_p2m_free(void) xen_cleanhighmap(addr, addr + size); size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); - memblock_free(__pa(addr), size); + memblock_free((void *)addr, size); } else { xen_cleanmfnmap(addr); } @@ -1192,6 +1197,13 @@ static void __init xen_pagetable_p2m_setup(void) static void __init xen_pagetable_init(void) { + /* + * The majority of further PTE writes is to pagetables already + * announced as such to Xen. Hence it is more efficient to use + * hypercalls for these updates. + */ + pv_ops.mmu.set_pte = __xen_set_pte; + paging_init(); xen_post_allocator_init(); @@ -1204,7 +1216,8 @@ static void __init xen_pagetable_init(void) xen_remap_memory(); xen_setup_mfn_list_list(); } -static void xen_write_cr2(unsigned long cr2) + +static noinstr void xen_write_cr2(unsigned long cr2) { this_cpu_read(xen_vcpu)->arch.cr2 = cr2; } @@ -1420,10 +1433,18 @@ static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd) * * Many of these PTE updates are done on unpinned and writable pages * and doing a hypercall for these is unnecessary and expensive. At - * this point it is not possible to tell if a page is pinned or not, - * so always write the PTE directly and rely on Xen trapping and + * this point it is rarely possible to tell if a page is pinned, so + * mostly write the PTE directly and rely on Xen trapping and * emulating any updates as necessary. */ +static void __init xen_set_pte_init(pte_t *ptep, pte_t pte) +{ + if (unlikely(is_early_ioremap_ptep(ptep))) + __xen_set_pte(ptep, pte); + else + native_set_pte(ptep, pte); +} + __visible pte_t xen_make_pte_init(pteval_t pte) { unsigned long pfn; @@ -1445,11 +1466,6 @@ __visible pte_t xen_make_pte_init(pteval_t pte) } PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte_init); -static void __init xen_set_pte_init(pte_t *ptep, pte_t pte) -{ - __xen_set_pte(ptep, pte); -} - /* Early in boot, while setting up the initial pagetable, assume everything is pinned. */ static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn) @@ -1749,7 +1765,6 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) set_page_prot(init_top_pgt, PAGE_KERNEL_RO); set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO); set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO); - set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO); set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO); set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO); set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO); @@ -1766,6 +1781,13 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) /* Unpin Xen-provided one */ pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); +#ifdef CONFIG_X86_VSYSCALL_EMULATION + /* Pin user vsyscall L3 */ + set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO); + pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, + PFN_DOWN(__pa_symbol(level3_user_vsyscall))); +#endif + /* * At this stage there can be no user pgd, and no page structure to * attach it to, so make sure we just set kernel pgd. @@ -1955,7 +1977,7 @@ void __init xen_relocate_p2m(void) pfn_end = p2m_pfn_end; } - memblock_free(PFN_PHYS(pfn), PAGE_SIZE * (pfn_end - pfn)); + memblock_phys_free(PFN_PHYS(pfn), PAGE_SIZE * (pfn_end - pfn)); while (pfn < pfn_end) { if (pfn == p2m_pfn) { pfn = p2m_pfn_end; @@ -1998,6 +2020,7 @@ static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss; static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) { pte_t pte; + unsigned long vaddr; phys >>= PAGE_SHIFT; @@ -2038,15 +2061,15 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) break; } - __native_set_fixmap(idx, pte); + vaddr = __fix_to_virt(idx); + if (HYPERVISOR_update_va_mapping(vaddr, pte, UVMF_INVLPG)) + BUG(); #ifdef CONFIG_X86_VSYSCALL_EMULATION /* Replicate changes to map the vsyscall page into the user pagetable vsyscall mapping. */ - if (idx == VSYSCALL_PAGE) { - unsigned long vaddr = __fix_to_virt(idx); + if (idx == VSYSCALL_PAGE) set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte); - } #endif } @@ -2078,67 +2101,69 @@ static void xen_leave_lazy_mmu(void) preempt_enable(); } -static const struct pv_mmu_ops xen_mmu_ops __initconst = { - .read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2), - .write_cr2 = xen_write_cr2, +static const typeof(pv_ops) xen_mmu_ops __initconst = { + .mmu = { + .read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2), + .write_cr2 = xen_write_cr2, - .read_cr3 = xen_read_cr3, - .write_cr3 = xen_write_cr3_init, + .read_cr3 = xen_read_cr3, + .write_cr3 = xen_write_cr3_init, - .flush_tlb_user = xen_flush_tlb, - .flush_tlb_kernel = xen_flush_tlb, - .flush_tlb_one_user = xen_flush_tlb_one_user, - .flush_tlb_multi = xen_flush_tlb_multi, - .tlb_remove_table = tlb_remove_table, + .flush_tlb_user = xen_flush_tlb, + .flush_tlb_kernel = xen_flush_tlb, + .flush_tlb_one_user = xen_flush_tlb_one_user, + .flush_tlb_multi = xen_flush_tlb_multi, + .tlb_remove_table = tlb_remove_table, - .pgd_alloc = xen_pgd_alloc, - .pgd_free = xen_pgd_free, + .pgd_alloc = xen_pgd_alloc, + .pgd_free = xen_pgd_free, - .alloc_pte = xen_alloc_pte_init, - .release_pte = xen_release_pte_init, - .alloc_pmd = xen_alloc_pmd_init, - .release_pmd = xen_release_pmd_init, + .alloc_pte = xen_alloc_pte_init, + .release_pte = xen_release_pte_init, + .alloc_pmd = xen_alloc_pmd_init, + .release_pmd = xen_release_pmd_init, - .set_pte = xen_set_pte_init, - .set_pmd = xen_set_pmd_hyper, + .set_pte = xen_set_pte_init, + .set_pmd = xen_set_pmd_hyper, - .ptep_modify_prot_start = xen_ptep_modify_prot_start, - .ptep_modify_prot_commit = xen_ptep_modify_prot_commit, + .ptep_modify_prot_start = xen_ptep_modify_prot_start, + .ptep_modify_prot_commit = xen_ptep_modify_prot_commit, - .pte_val = PV_CALLEE_SAVE(xen_pte_val), - .pgd_val = PV_CALLEE_SAVE(xen_pgd_val), + .pte_val = PV_CALLEE_SAVE(xen_pte_val), + .pgd_val = PV_CALLEE_SAVE(xen_pgd_val), - .make_pte = PV_CALLEE_SAVE(xen_make_pte_init), - .make_pgd = PV_CALLEE_SAVE(xen_make_pgd), + .make_pte = PV_CALLEE_SAVE(xen_make_pte_init), + .make_pgd = PV_CALLEE_SAVE(xen_make_pgd), - .set_pud = xen_set_pud_hyper, + .set_pud = xen_set_pud_hyper, - .make_pmd = PV_CALLEE_SAVE(xen_make_pmd), - .pmd_val = PV_CALLEE_SAVE(xen_pmd_val), + .make_pmd = PV_CALLEE_SAVE(xen_make_pmd), + .pmd_val = PV_CALLEE_SAVE(xen_pmd_val), - .pud_val = PV_CALLEE_SAVE(xen_pud_val), - .make_pud = PV_CALLEE_SAVE(xen_make_pud), - .set_p4d = xen_set_p4d_hyper, + .pud_val = PV_CALLEE_SAVE(xen_pud_val), + .make_pud = PV_CALLEE_SAVE(xen_make_pud), + .set_p4d = xen_set_p4d_hyper, - .alloc_pud = xen_alloc_pmd_init, - .release_pud = xen_release_pmd_init, + .alloc_pud = xen_alloc_pmd_init, + .release_pud = xen_release_pmd_init, #if CONFIG_PGTABLE_LEVELS >= 5 - .p4d_val = PV_CALLEE_SAVE(xen_p4d_val), - .make_p4d = PV_CALLEE_SAVE(xen_make_p4d), + .p4d_val = PV_CALLEE_SAVE(xen_p4d_val), + .make_p4d = PV_CALLEE_SAVE(xen_make_p4d), #endif - .activate_mm = xen_activate_mm, - .dup_mmap = xen_dup_mmap, - .exit_mmap = xen_exit_mmap, + .activate_mm = xen_activate_mm, + .dup_mmap = xen_dup_mmap, + .exit_mmap = xen_exit_mmap, - .lazy_mode = { - .enter = paravirt_enter_lazy_mmu, - .leave = xen_leave_lazy_mmu, - .flush = paravirt_flush_lazy_mmu, + .lazy_mode = { + .enter = paravirt_enter_lazy_mmu, + .leave = xen_leave_lazy_mmu, + .flush = paravirt_flush_lazy_mmu, + }, + + .set_fixmap = xen_set_fixmap, }, - - .set_fixmap = xen_set_fixmap, }; void __init xen_init_mmu_ops(void) @@ -2146,7 +2171,7 @@ void __init xen_init_mmu_ops(void) x86_init.paging.pagetable_init = xen_pagetable_init; x86_init.hyper.init_after_bootmem = xen_after_bootmem; - pv_ops.mmu = xen_mmu_ops; + pv_ops.mmu = xen_mmu_ops.mmu; memset(dummy_mapping, 0xff, PAGE_SIZE); } diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index 5e6e236977..58db86f7b3 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c @@ -197,7 +197,7 @@ static void * __ref alloc_p2m_page(void) static void __ref free_p2m_page(void *p) { if (unlikely(!slab_is_available())) { - memblock_free((unsigned long)p, PAGE_SIZE); + memblock_free(p, PAGE_SIZE); return; } diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index e13b0b49fc..89dd6b1708 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -413,34 +413,29 @@ int pmu_apic_update(uint32_t val) } /* perf callbacks */ -static int xen_is_in_guest(void) +static unsigned int xen_guest_state(void) { const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + unsigned int state = 0; if (!xenpmu_data) { pr_warn_once("%s: pmudata not initialized\n", __func__); - return 0; + return state; } if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF)) - return 0; + return state; - return 1; -} + state |= PERF_GUEST_ACTIVE; -static int xen_is_user_mode(void) -{ - const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); - - if (!xenpmu_data) { - pr_warn_once("%s: pmudata not initialized\n", __func__); - return 0; + if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) { + if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER) + state |= PERF_GUEST_USER; + } else if (xenpmu_data->pmu.r.regs.cpl & 3) { + state |= PERF_GUEST_USER; } - if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) - return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER); - else - return !!(xenpmu_data->pmu.r.regs.cpl & 3); + return state; } static unsigned long xen_get_guest_ip(void) @@ -456,9 +451,8 @@ static unsigned long xen_get_guest_ip(void) } static struct perf_guest_info_callbacks xen_guest_cbs = { - .is_in_guest = xen_is_in_guest, - .is_user_mode = xen_is_user_mode, - .get_guest_ip = xen_get_guest_ip, + .state = xen_guest_state, + .get_ip = xen_get_guest_ip, }; /* Convert registers from Xen's format to Linux' */ diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index 8bfc103301..af216feb63 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c @@ -153,7 +153,7 @@ static void __init xen_del_extra_mem(unsigned long start_pfn, break; } } - memblock_free(PFN_PHYS(start_pfn), PFN_PHYS(n_pfns)); + memblock_phys_free(PFN_PHYS(start_pfn), PFN_PHYS(n_pfns)); } /* @@ -306,10 +306,6 @@ static void __init xen_update_mem_tables(unsigned long pfn, unsigned long mfn) BUG(); } - /* Update kernel mapping, but not for highmem. */ - if (pfn >= PFN_UP(__pa(high_memory - 1))) - return; - if (HYPERVISOR_update_va_mapping((unsigned long)__va(pfn << PAGE_SHIFT), mfn_pte(mfn, PAGE_KERNEL), 0)) { WARN(1, "Failed to update kernel mapping for mfn=%ld pfn=%ld\n", @@ -429,13 +425,13 @@ static unsigned long __init xen_set_identity_and_remap_chunk( } /* - * If the PFNs are currently mapped, the VA mapping also needs - * to be updated to be 1:1. + * If the PFNs are currently mapped, their VA mappings need to be + * zapped. */ for (pfn = start_pfn; pfn <= max_pfn_mapped && pfn < end_pfn; pfn++) (void)HYPERVISOR_update_va_mapping( (unsigned long)__va(pfn << PAGE_SHIFT), - mfn_pte(pfn, PAGE_KERNEL_IO), 0); + native_make_pte(0), 0); return remap_pfn; } @@ -719,7 +715,7 @@ static void __init xen_reserve_xen_mfnlist(void) return; xen_relocate_p2m(); - memblock_free(start, size); + memblock_phys_free(start, size); } /** @@ -885,7 +881,7 @@ char * __init xen_memory_setup(void) xen_phys_memcpy(new_area, start, size); pr_info("initrd moved from [mem %#010llx-%#010llx] to [mem %#010llx-%#010llx]\n", start, start + size, new_area, new_area + size); - memblock_free(start, size); + memblock_phys_free(start, size); boot_params.hdr.ramdisk_image = new_area; boot_params.ext_ramdisk_image = new_area >> 32; } diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index c1b2f764b2..c3e1f9a7d4 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -121,34 +121,10 @@ int xen_smp_intr_init(unsigned int cpu) void __init xen_smp_cpus_done(unsigned int max_cpus) { - int cpu, rc, count = 0; - if (xen_hvm_domain()) native_smp_cpus_done(max_cpus); else calculate_max_logical_packages(); - - if (xen_have_vcpu_info_placement) - return; - - for_each_online_cpu(cpu) { - if (xen_vcpu_nr(cpu) < MAX_VIRT_CPUS) - continue; - - rc = remove_cpu(cpu); - - if (rc == 0) { - /* - * Reset vcpu_info so this cpu cannot be onlined again. - */ - xen_vcpu_info_reset(cpu); - count++; - } else { - pr_warn("%s: failed to bring CPU %d down, error %d\n", - __func__, cpu, rc); - } - } - WARN(count, "%s: brought %d CPUs offline\n", __func__, count); } void xen_smp_send_reschedule(int cpu) @@ -268,20 +244,16 @@ void xen_send_IPI_allbutself(int vector) static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) { - irq_enter(); generic_smp_call_function_interrupt(); inc_irq_stat(irq_call_count); - irq_exit(); return IRQ_HANDLED; } static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id) { - irq_enter(); generic_smp_call_function_single_interrupt(); inc_irq_stat(irq_call_count); - irq_exit(); return IRQ_HANDLED; } diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c index 477c484eb2..4a6019238e 100644 --- a/arch/x86/xen/smp_pv.c +++ b/arch/x86/xen/smp_pv.c @@ -208,7 +208,6 @@ static void __init xen_pv_smp_prepare_boot_cpu(void) static void __init xen_pv_smp_prepare_cpus(unsigned int max_cpus) { unsigned cpu; - unsigned int i; if (skip_ioapic_setup) { char *m = (max_cpus == 0) ? @@ -221,16 +220,9 @@ static void __init xen_pv_smp_prepare_cpus(unsigned int max_cpus) } xen_init_lock_cpu(0); - smp_store_boot_cpu_info(); - cpu_data(0).x86_max_cores = 1; + smp_prepare_cpus_common(); - for_each_possible_cpu(i) { - zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL); - zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL); - zalloc_cpumask_var(&per_cpu(cpu_die_map, i), GFP_KERNEL); - zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL); - } - set_cpu_sibling_map(0); + cpu_data(0).x86_max_cores = 1; speculative_store_bypass_ht_init(); @@ -441,10 +433,8 @@ static void xen_pv_stop_other_cpus(int wait) static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id) { - irq_enter(); irq_work_run(); inc_irq_stat(apic_irq_work_irqs); - irq_exit(); return IRQ_HANDLED; } diff --git a/arch/x86/xen/vga.c b/arch/x86/xen/vga.c index e336f223f7..14ea32e734 100644 --- a/arch/x86/xen/vga.c +++ b/arch/x86/xen/vga.c @@ -57,15 +57,19 @@ void __init xen_init_vga(const struct dom0_vga_console_info *info, size_t size) screen_info->rsvd_size = info->u.vesa_lfb.rsvd_size; screen_info->rsvd_pos = info->u.vesa_lfb.rsvd_pos; + if (size >= offsetof(struct dom0_vga_console_info, + u.vesa_lfb.ext_lfb_base) + + sizeof(info->u.vesa_lfb.ext_lfb_base) + && info->u.vesa_lfb.ext_lfb_base) { + screen_info->ext_lfb_base = info->u.vesa_lfb.ext_lfb_base; + screen_info->capabilities |= VIDEO_CAPABILITY_64BIT_BASE; + } + if (info->video_type == XEN_VGATYPE_EFI_LFB) { screen_info->orig_video_isVGA = VIDEO_TYPE_EFI; break; } - if (size >= offsetof(struct dom0_vga_console_info, - u.vesa_lfb.gbl_caps) - + sizeof(info->u.vesa_lfb.gbl_caps)) - screen_info->capabilities = info->u.vesa_lfb.gbl_caps; if (size >= offsetof(struct dom0_vga_console_info, u.vesa_lfb.mode_attrs) + sizeof(info->u.vesa_lfb.mode_attrs)) diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S index 3bebf66569..e730e6200e 100644 --- a/arch/x86/xen/xen-asm.S +++ b/arch/x86/xen/xen-asm.S @@ -22,58 +22,16 @@ #include #include <../entry/calling.h> -/* - * Enable events. This clears the event mask and tests the pending - * event status with one and operation. If there are pending events, - * then enter the hypervisor to get them handled. - */ -SYM_FUNC_START(xen_irq_enable_direct) - FRAME_BEGIN - /* Unmask events */ - movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask - - /* - * Preempt here doesn't matter because that will deal with any - * pending interrupts. The pending check may end up being run - * on the wrong CPU, but that doesn't hurt. - */ - - /* Test for pending */ - testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending - jz 1f - - call check_events -1: - FRAME_END - ret -SYM_FUNC_END(xen_irq_enable_direct) - - +.pushsection .noinstr.text, "ax" /* * Disabling events is simply a matter of making the event mask * non-zero. */ SYM_FUNC_START(xen_irq_disable_direct) movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask - ret + RET SYM_FUNC_END(xen_irq_disable_direct) -/* - * (xen_)save_fl is used to get the current interrupt enable status. - * Callers expect the status to be in X86_EFLAGS_IF, and other bits - * may be set in the return value. We take advantage of this by - * making sure that X86_EFLAGS_IF has the right value (and other bits - * in that byte are 0), but other bits in the return value are - * undefined. We need to toggle the state of the bit, because Xen and - * x86 use opposite senses (mask vs enable). - */ -SYM_FUNC_START(xen_save_fl_direct) - testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask - setz %ah - addb %ah, %ah - ret -SYM_FUNC_END(xen_save_fl_direct) - /* * Force an event check by making a hypercall, but preserve regs * before making the call. @@ -100,23 +58,66 @@ SYM_FUNC_START(check_events) pop %rcx pop %rax FRAME_END - ret + RET SYM_FUNC_END(check_events) +/* + * Enable events. This clears the event mask and tests the pending + * event status with one and operation. If there are pending events, + * then enter the hypervisor to get them handled. + */ +SYM_FUNC_START(xen_irq_enable_direct) + FRAME_BEGIN + /* Unmask events */ + movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask + + /* + * Preempt here doesn't matter because that will deal with any + * pending interrupts. The pending check may end up being run + * on the wrong CPU, but that doesn't hurt. + */ + + /* Test for pending */ + testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending + jz 1f + + call check_events +1: + FRAME_END + RET +SYM_FUNC_END(xen_irq_enable_direct) + +/* + * (xen_)save_fl is used to get the current interrupt enable status. + * Callers expect the status to be in X86_EFLAGS_IF, and other bits + * may be set in the return value. We take advantage of this by + * making sure that X86_EFLAGS_IF has the right value (and other bits + * in that byte are 0), but other bits in the return value are + * undefined. We need to toggle the state of the bit, because Xen and + * x86 use opposite senses (mask vs enable). + */ +SYM_FUNC_START(xen_save_fl_direct) + testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask + setz %ah + addb %ah, %ah + RET +SYM_FUNC_END(xen_save_fl_direct) + SYM_FUNC_START(xen_read_cr2) FRAME_BEGIN _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX FRAME_END - ret + RET SYM_FUNC_END(xen_read_cr2); SYM_FUNC_START(xen_read_cr2_direct) FRAME_BEGIN _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX FRAME_END - ret + RET SYM_FUNC_END(xen_read_cr2_direct); +.popsection .macro xen_pv_trap name SYM_CODE_START(xen_\name) diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S index cb6538ae2f..11d286529f 100644 --- a/arch/x86/xen/xen-head.S +++ b/arch/x86/xen/xen-head.S @@ -20,6 +20,23 @@ #include #include +.pushsection .noinstr.text, "ax" + .balign PAGE_SIZE +SYM_CODE_START(hypercall_page) + .rept (PAGE_SIZE / 32) + UNWIND_HINT_FUNC + .skip 31, 0x90 + RET + .endr + +#define HYPERCALL(n) \ + .equ xen_hypercall_##n, hypercall_page + __HYPERVISOR_##n * 32; \ + .type xen_hypercall_##n, @function; .size xen_hypercall_##n, 32 +#include +#undef HYPERCALL +SYM_CODE_END(hypercall_page) +.popsection + #ifdef CONFIG_XEN_PV __INIT SYM_CODE_START(startup_xen) @@ -28,13 +45,13 @@ SYM_CODE_START(startup_xen) /* Clear .bss */ xor %eax,%eax - mov $__bss_start, %_ASM_DI - mov $__bss_stop, %_ASM_CX - sub %_ASM_DI, %_ASM_CX - shr $__ASM_SEL(2, 3), %_ASM_CX - rep __ASM_SIZE(stos) + mov $__bss_start, %rdi + mov $__bss_stop, %rcx + sub %rdi, %rcx + shr $3, %rcx + rep stosq - mov %_ASM_SI, xen_start_info + mov %rsi, xen_start_info mov initial_stack(%rip), %rsp /* Set up %gs. @@ -64,23 +81,6 @@ SYM_CODE_END(asm_cpu_bringup_and_idle) #endif #endif -.pushsection .text - .balign PAGE_SIZE -SYM_CODE_START(hypercall_page) - .rept (PAGE_SIZE / 32) - UNWIND_HINT_FUNC - .skip 31, 0x90 - ret - .endr - -#define HYPERCALL(n) \ - .equ xen_hypercall_##n, hypercall_page + __HYPERVISOR_##n * 32; \ - .type xen_hypercall_##n, @function; .size xen_hypercall_##n, 32 -#include -#undef HYPERCALL -SYM_CODE_END(hypercall_page) -.popsection - ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz "linux") ELFNOTE(Xen, XEN_ELFNOTE_GUEST_VERSION, .asciz "2.6") ELFNOTE(Xen, XEN_ELFNOTE_XEN_VERSION, .asciz "xen-3.0") diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 8bc8b72a20..fd0fec6e92 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -76,9 +76,7 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id); bool xen_vcpu_stolen(int vcpu); -extern int xen_have_vcpu_info_placement; - -int xen_vcpu_setup(int cpu); +void xen_vcpu_setup(int cpu); void xen_vcpu_info_reset(int cpu); void xen_setup_vcpu_info_placement(void);