diff --git a/MAINTAINERS b/MAINTAINERS index 2c57d8f66ab6513fd14ab77fcfc248494a36590b..3973ffb5fbbfc9db9bee590c18e62025954a4f19 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11611,6 +11611,7 @@ L: kvm@vger.kernel.org L: loongarch@lists.linux.dev S: Maintained T: git git://git.kernel.org/pub/scm/virt/kvm/kvm.git +F: Documentation/virt/kvm/loongarch/ F: arch/loongarch/include/asm/kvm* F: arch/loongarch/include/uapi/asm/kvm* F: arch/loongarch/kvm/ diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index c6a5b99f11c5c67dab521c545ea139821e5d2d99..75dffe2d72e801be1f1060201aad4ade9568a231 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -140,7 +140,7 @@ config LOONGARCH select HAVE_LIVEPATCH select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI - select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS + select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS && AS_HAS_THIN_ADD_SUB select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS @@ -269,6 +269,9 @@ config AS_HAS_EXPLICIT_RELOCS config AS_HAS_FCSR_CLASS def_bool $(as-instr,movfcsr2gr \$t0$(comma)\$fcsr0) +config AS_HAS_THIN_ADD_SUB + def_bool $(cc-option,-Wa$(comma)-mthin-add-sub) || AS_IS_LLVM + config AS_HAS_LSX_EXTENSION def_bool $(as-instr,vld \$vr0$(comma)\$a0$(comma)0) diff --git a/arch/loongarch/Kconfig.debug b/arch/loongarch/Kconfig.debug index 98d60630c3d4b4771d7fe5570e15b7f5e0ab2071..8b2ce5b5d43e8f7df0af14019d2c8b28a7da1511 100644 --- a/arch/loongarch/Kconfig.debug +++ b/arch/loongarch/Kconfig.debug @@ -28,6 +28,7 @@ config UNWINDER_PROLOGUE config UNWINDER_ORC bool "ORC unwinder" + depends on HAVE_OBJTOOL select OBJTOOL help This option enables the ORC (Oops Rewind Capability) unwinder for diff --git a/arch/loongarch/configs/openkylin_generic_defconfig b/arch/loongarch/configs/openkylin_generic_defconfig index 69150069e181a0f4ace31018d2d8cee4e0a6c438..1a77591b09d78a6ec7540efa7237d33319536804 100644 --- a/arch/loongarch/configs/openkylin_generic_defconfig +++ b/arch/loongarch/configs/openkylin_generic_defconfig @@ -47,6 +47,7 @@ CONFIG_NUMA=y CONFIG_CPU_HAS_LSX=y CONFIG_CPU_HAS_LASX=y CONFIG_CPU_HAS_LBT=y +CONFIG_LIVEPATCH=y CONFIG_CPU_FREQ=y CONFIG_CPU_FREQ_STAT=y CONFIG_CPU_FREQ_GOV_POWERSAVE=y @@ -2444,6 +2445,7 @@ CONFIG_RCU_TORTURE_TEST=m CONFIG_RCU_REF_SCALE_TEST=m CONFIG_RCU_CPU_STALL_TIMEOUT=60 # CONFIG_RCU_TRACE is not set +CONFIG_FUNCTION_TRACER=y CONFIG_BOOTTIME_TRACING=y CONFIG_STACK_TRACER=y CONFIG_SCHED_TRACER=y @@ -2456,6 +2458,7 @@ CONFIG_RING_BUFFER_BENCHMARK=m CONFIG_KUNIT=m CONFIG_KUNIT_ALL_TESTS=m # CONFIG_TEST_SORT is not set +CONFIG_UNWINDER_ORC=y CONFIG_ATOMIC64_SELFTEST=y CONFIG_ASYNC_RAID6_TEST=m CONFIG_TEST_KSTRTOX=m diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild index c862672ed953fd92dd998def34a2e016e06db364..aa4ab6ccc0c0faac49efb44a142954da895f55ea 100644 --- a/arch/loongarch/include/asm/Kbuild +++ b/arch/loongarch/include/asm/Kbuild @@ -6,7 +6,6 @@ generic-y += mcs_spinlock.h generic-y += parport.h generic-y += early_ioremap.h generic-y += qrwlock.h -generic-y += qspinlock.h generic-y += rwsem.h generic-y += segment.h generic-y += user.h diff --git a/arch/loongarch/include/asm/addrspace.h b/arch/loongarch/include/asm/addrspace.h index 838fa4d47490403c70ac2ac7d99b46f2a487a9ca..8ef3e36009fca279280b62d685614b375078eecd 100644 --- a/arch/loongarch/include/asm/addrspace.h +++ b/arch/loongarch/include/asm/addrspace.h @@ -37,6 +37,10 @@ extern unsigned long vm_map_base; #define UNCACHE_BASE CSR_DMW0_BASE #endif +#ifndef WRITECOMBINE_BASE +#define WRITECOMBINE_BASE CSR_DMW2_BASE +#endif + #define DMW_PABITS 48 #define TO_PHYS_MASK ((1ULL << DMW_PABITS) - 1) diff --git a/arch/loongarch/include/asm/io.h b/arch/loongarch/include/asm/io.h index c2f9979b2979e5e92e791e3f8304975db9e929c9..5e95a60df1808aa294b791f12a8bfe3826059116 100644 --- a/arch/loongarch/include/asm/io.h +++ b/arch/loongarch/include/asm/io.h @@ -25,10 +25,16 @@ extern void __init early_iounmap(void __iomem *addr, unsigned long size); static inline void __iomem *ioremap_prot(phys_addr_t offset, unsigned long size, unsigned long prot_val) { - if (prot_val & _CACHE_CC) + switch (prot_val & _CACHE_MASK) { + case _CACHE_CC: return (void __iomem *)(unsigned long)(CACHE_BASE + offset); - else + case _CACHE_SUC: return (void __iomem *)(unsigned long)(UNCACHE_BASE + offset); + case _CACHE_WUC: + return (void __iomem *)(unsigned long)(WRITECOMBINE_BASE + offset); + default: + return NULL; + } } #define ioremap(offset, size) \ diff --git a/arch/loongarch/include/asm/kvm_csr.h b/arch/loongarch/include/asm/kvm_csr.h index 0a52f115a87eb37e00c363e4969a1f64c75266ac..4a76ce796f1f401eba8ce246fe8fb5892aee07c4 100644 --- a/arch/loongarch/include/asm/kvm_csr.h +++ b/arch/loongarch/include/asm/kvm_csr.h @@ -181,6 +181,7 @@ __BUILD_GCSR_OP(tlbidx) #define kvm_save_hw_gcsr(csr, gid) (csr->csrs[gid] = gcsr_read(gid)) #define kvm_restore_hw_gcsr(csr, gid) (gcsr_write(csr->csrs[gid], gid)) + #define kvm_read_clear_hw_gcsr(csr, gid) (csr->csrs[gid] = gcsr_write(0, gid)) int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu); @@ -210,9 +211,7 @@ static __always_inline void kvm_change_sw_gcsr(struct loongarch_csrs *csr, csr->csrs[gid] |= val & _mask; } -#define KVM_PMU_EVENT_ENABLED (CSR_PERFCTRL_PLV0 | \ - CSR_PERFCTRL_PLV1 | \ - CSR_PERFCTRL_PLV2 | \ - CSR_PERFCTRL_PLV3) +#define KVM_PMU_EVENT_ENABLED (CSR_PERFCTRL_PLV0 | CSR_PERFCTRL_PLV1 | \ + CSR_PERFCTRL_PLV2 | CSR_PERFCTRL_PLV3) #endif /* __ASM_LOONGARCH_KVM_CSR_H__ */ diff --git a/arch/loongarch/include/asm/kvm_extioi.h b/arch/loongarch/include/asm/kvm_extioi.h new file mode 100644 index 0000000000000000000000000000000000000000..c2bd295d0edcb6d0b750ba89e55fef2a4d6a1966 --- /dev/null +++ b/arch/loongarch/include/asm/kvm_extioi.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#ifndef LOONGARCH_EXTIOI_H +#define LOONGARCH_EXTIOI_H + +#include + +#define EXTIOI_IRQS 256 +#define EXTIOI_ROUTE_MAX_VCPUS 256 +#define EXTIOI_IRQS_U8_NUMS (EXTIOI_IRQS / 8) +#define EXTIOI_IRQS_U32_NUMS (EXTIOI_IRQS_U8_NUMS / 4) +#define EXTIOI_IRQS_U64_NUMS (EXTIOI_IRQS_U32_NUMS / 2) +/* map to ipnum per 32 irqs */ +#define EXTIOI_IRQS_NODETYPE_COUNT 16 + +#define EXTIOI_BASE 0x1400 +#define EXTIOI_SIZE 0x900 + +#define EXTIOI_NODETYPE_START 0xa0 +#define EXTIOI_NODETYPE_END 0xbf +#define EXTIOI_IPMAP_START 0xc0 +#define EXTIOI_IPMAP_END 0xc7 +#define EXTIOI_ENABLE_START 0x200 +#define EXTIOI_ENABLE_END 0x21f +#define EXTIOI_BOUNCE_START 0x280 +#define EXTIOI_BOUNCE_END 0x29f +#define EXTIOI_ISR_START 0x300 +#define EXTIOI_ISR_END 0x31f +#define EXTIOI_COREISR_START 0x400 +#define EXTIOI_COREISR_END 0x71f +#define EXTIOI_COREMAP_START 0x800 +#define EXTIOI_COREMAP_END 0x8ff + +#define LS3A_INTC_IP 8 + +#define EXTIOI_SW_COREMAP_FLAG (1 << 0) + +struct loongarch_extioi { + spinlock_t lock; + struct kvm *kvm; + struct kvm_io_device device; + /* hardware state */ + union nodetype { + u64 reg_u64[EXTIOI_IRQS_NODETYPE_COUNT / 4]; + u32 reg_u32[EXTIOI_IRQS_NODETYPE_COUNT / 2]; + uint16_t reg_u16[EXTIOI_IRQS_NODETYPE_COUNT]; + u8 reg_u8[EXTIOI_IRQS_NODETYPE_COUNT * 2]; + } nodetype; + + /* one bit shows the state of one irq */ + union bounce { + u64 reg_u64[EXTIOI_IRQS_U64_NUMS]; + u32 reg_u32[EXTIOI_IRQS_U32_NUMS]; + u8 reg_u8[EXTIOI_IRQS_U8_NUMS]; + } bounce; + + union isr { + u64 reg_u64[EXTIOI_IRQS_U64_NUMS]; + u32 reg_u32[EXTIOI_IRQS_U32_NUMS]; + u8 reg_u8[EXTIOI_IRQS_U8_NUMS]; + } isr; + union coreisr { + u64 reg_u64[EXTIOI_ROUTE_MAX_VCPUS][EXTIOI_IRQS_U64_NUMS]; + u32 reg_u32[EXTIOI_ROUTE_MAX_VCPUS][EXTIOI_IRQS_U32_NUMS]; + u8 reg_u8[EXTIOI_ROUTE_MAX_VCPUS][EXTIOI_IRQS_U8_NUMS]; + } coreisr; + union enable { + u64 reg_u64[EXTIOI_IRQS_U64_NUMS]; + u32 reg_u32[EXTIOI_IRQS_U32_NUMS]; + u8 reg_u8[EXTIOI_IRQS_U8_NUMS]; + } enable; + + /* use one byte to config ipmap for 32 irqs at once */ + union ipmap { + u64 reg_u64; + u32 reg_u32[EXTIOI_IRQS_U32_NUMS / 4]; + u8 reg_u8[EXTIOI_IRQS_U8_NUMS / 4]; + } ipmap; + /* use one byte to config coremap for one irq */ + union coremap { + u64 reg_u64[EXTIOI_IRQS / 8]; + u32 reg_u32[EXTIOI_IRQS / 4]; + u8 reg_u8[EXTIOI_IRQS]; + } coremap; + + DECLARE_BITMAP(sw_coreisr[EXTIOI_ROUTE_MAX_VCPUS][LS3A_INTC_IP], EXTIOI_IRQS); + uint8_t sw_coremap[EXTIOI_IRQS]; +}; + +void extioi_set_irq(struct loongarch_extioi *s, int irq, int level); +int kvm_loongarch_register_extioi_device(void); +int kvm_loongarch_reset_extioi(struct kvm *kvm); +#endif /* LOONGARCH_EXTIOI_H */ diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 7e778847eb7089473982e95b54d5627a59d1ac4d..f8528e183445db43850d3b1f8299c300fad1073e 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -19,6 +19,9 @@ #include #include #include +#include +#include +#include /* Loongarch KVM register ids */ #define KVM_GET_IOC_CSR_IDX(id) ((id & KVM_CSR_IDX_MASK) >> LOONGARCH_REG_SHIFT) @@ -26,12 +29,27 @@ #define KVM_MAX_VCPUS 256 #define KVM_MAX_CPUCFG_REGS 21 -/* memory slots that does not exposed to userspace */ -#define KVM_PRIVATE_MEM_SLOTS 0 #define KVM_HALT_POLL_NS_DEFAULT 500000 #define KVM_REQ_TLB_FLUSH_GPA KVM_ARCH_REQ(0) #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(1) +#define KVM_REQ_PMU KVM_ARCH_REQ(2) + +/* KVM_IRQ_LINE irq field index values */ +#define KVM_LOONGARCH_IRQ_TYPE_SHIFT 24 +#define KVM_LOONGARCH_IRQ_TYPE_MASK 0xff +#define KVM_LOONGARCH_IRQ_VCPU_SHIFT 16 +#define KVM_LOONGARCH_IRQ_VCPU_MASK 0xff +#define KVM_LOONGARCH_IRQ_NUM_SHIFT 0 +#define KVM_LOONGARCH_IRQ_NUM_MASK 0xffff + +/* irq_type field */ +#define KVM_LOONGARCH_IRQ_TYPE_CPU_IP 0 +#define KVM_LOONGARCH_IRQ_TYPE_CPU_IO 1 +#define KVM_LOONGARCH_IRQ_TYPE_HT 2 +#define KVM_LOONGARCH_IRQ_TYPE_MSI 3 +#define KVM_LOONGARCH_IRQ_TYPE_IOAPIC 4 +#define KVM_LOONGARCH_IRQ_TYPE_ROUTE 5 #define KVM_GUESTDBG_SW_BP_MASK \ (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP) @@ -45,6 +63,12 @@ struct kvm_vm_stat { struct kvm_vm_stat_generic generic; u64 pages; u64 hugepages; + u64 ipi_read_exits; + u64 ipi_write_exits; + u64 extioi_read_exits; + u64 extioi_write_exits; + u64 pch_pic_read_exits; + u64 pch_pic_write_exits; }; struct kvm_vcpu_stat { @@ -62,12 +86,11 @@ struct kvm_arch_memory_slot { unsigned long flags; }; -#define KVM_REQ_PMU KVM_ARCH_REQ(0) #define HOST_MAX_PMNUM 16 struct kvm_context { unsigned long vpid_cache; struct kvm_vcpu *last_vcpu; - /* Save host pmu csr */ + /* Host PMU CSR */ u64 perf_ctrl[HOST_MAX_PMNUM]; u64 perf_cntr[HOST_MAX_PMNUM]; }; @@ -114,9 +137,14 @@ struct kvm_arch { unsigned int root_level; spinlock_t phyid_map_lock; struct kvm_phyid_map *phyid_map; + /* Enabled PV features */ + unsigned long pv_features; s64 time_offset; struct kvm_context __percpu *vmcs; + struct loongarch_ipi *ipi; + struct loongarch_extioi *extioi; + struct loongarch_pch_pic *pch_pic; }; #define CSR_MAX_NUMS 0x800 @@ -140,10 +168,15 @@ enum emulation_result { #define KVM_LARCH_FPU (0x1 << 0) #define KVM_LARCH_LSX (0x1 << 1) #define KVM_LARCH_LASX (0x1 << 2) -#define KVM_LARCH_SWCSR_LATEST (0x1 << 3) -#define KVM_LARCH_HWCSR_USABLE (0x1 << 4) -#define KVM_GUEST_PMU_ENABLE (0x1 << 5) -#define KVM_GUEST_PMU_ACTIVE (0x1 << 6) +#define KVM_LARCH_LBT (0x1 << 3) +#define KVM_LARCH_PMU (0x1 << 4) +#define KVM_LARCH_SWCSR_LATEST (0x1 << 5) +#define KVM_LARCH_HWCSR_USABLE (0x1 << 6) + +#define LOONGARCH_PV_FEAT_UPDATED BIT_ULL(63) +#define LOONGARCH_PV_FEAT_MASK (BIT(KVM_FEATURE_IPI) | \ + BIT(KVM_FEATURE_STEAL_TIME) | \ + BIT(KVM_FEATURE_VIRT_EXTIOI)) struct kvm_vcpu_arch { /* @@ -177,6 +210,7 @@ struct kvm_vcpu_arch { /* FPU state */ struct loongarch_fpu fpu FPU_ALIGN; + struct loongarch_lbt lbt; /* CSR state */ struct loongarch_csrs *csr; @@ -215,6 +249,8 @@ struct kvm_vcpu_arch { int last_sched_cpu; /* mp state */ struct kvm_mp_state mp_state; + /* ipi state */ + struct ipi_state ipi_state; /* cpucfg */ u32 cpucfg[KVM_MAX_CPUCFG_REGS]; @@ -251,14 +287,19 @@ static inline bool kvm_guest_has_lasx(struct kvm_vcpu_arch *arch) return arch->cpucfg[2] & CPUCFG2_LASX; } +static inline bool kvm_guest_has_lbt(struct kvm_vcpu_arch *arch) +{ + return arch->cpucfg[2] & (CPUCFG2_X86BT | CPUCFG2_ARMBT | CPUCFG2_MIPSBT); +} + static inline bool kvm_guest_has_pmu(struct kvm_vcpu_arch *arch) { - return arch->cpucfg[LOONGARCH_CPUCFG6] & CPUCFG6_PMP; + return arch->cpucfg[6] & CPUCFG6_PMP; } static inline int kvm_get_pmu_num(struct kvm_vcpu_arch *arch) { - return (arch->cpucfg[LOONGARCH_CPUCFG6] & CPUCFG6_PMNUM) >> CPUCFG6_PMNUM_SHIFT; + return (arch->cpucfg[6] & CPUCFG6_PMNUM) >> CPUCFG6_PMNUM_SHIFT; } /* Debug: dump vcpu state */ diff --git a/arch/loongarch/include/asm/kvm_ipi.h b/arch/loongarch/include/asm/kvm_ipi.h new file mode 100644 index 0000000000000000000000000000000000000000..729dfc1e3f401758601a9bf8673f9ee51f8156b4 --- /dev/null +++ b/arch/loongarch/include/asm/kvm_ipi.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#ifndef __LS3A_KVM_IPI_H +#define __LS3A_KVM_IPI_H + +#include + +#define LARCH_INT_IPI 12 + +struct loongarch_ipi { + spinlock_t lock; + struct kvm *kvm; + struct kvm_io_device device; + struct kvm_io_device mail_dev; +}; + +struct ipi_state { + spinlock_t lock; + uint32_t status; + uint32_t en; + uint32_t set; + uint32_t clear; + uint64_t buf[4]; +}; + +#define SMP_MAILBOX 0x1000 +#define KVM_IOCSR_IPI_ADDR_SIZE 0x48 + +#define CORE_STATUS_OFF 0x000 +#define CORE_EN_OFF 0x004 +#define CORE_SET_OFF 0x008 +#define CORE_CLEAR_OFF 0x00c +#define CORE_BUF_20 0x020 +#define CORE_BUF_28 0x028 +#define CORE_BUF_30 0x030 +#define CORE_BUF_38 0x038 +#define IOCSR_IPI_SEND 0x040 + +#define IOCSR_MAIL_SEND 0x048 +#define IOCSR_ANY_SEND 0x158 + +#define MAIL_SEND_ADDR (SMP_MAILBOX + IOCSR_MAIL_SEND) +#define KVM_IOCSR_MAIL_ADDR_SIZE 0x118 + +#define MAIL_SEND_OFFSET 0 +#define ANY_SEND_OFFSET (IOCSR_ANY_SEND - IOCSR_MAIL_SEND) + +int kvm_loongarch_register_ipi_device(void); +#endif diff --git a/arch/loongarch/include/asm/kvm_para.h b/arch/loongarch/include/asm/kvm_para.h index 335fb86778e2bc4d7a82c7e57670277a90efc481..d0f087870628a13a2633eccff2470d1c4ee8ad82 100644 --- a/arch/loongarch/include/asm/kvm_para.h +++ b/arch/loongarch/include/asm/kvm_para.h @@ -2,6 +2,8 @@ #ifndef _ASM_LOONGARCH_KVM_PARA_H #define _ASM_LOONGARCH_KVM_PARA_H +#include + /* * Hypercall code field */ @@ -39,9 +41,9 @@ struct kvm_steal_time { * Hypercall interface for KVM hypervisor * * a0: function identifier - * a1-a6: args + * a1-a5: args * Return value will be placed in a0. - * Up to 6 arguments are passed in a1, a2, a3, a4, a5, a6. + * Up to 5 arguments are passed in a1, a2, a3, a4, a5. */ static __always_inline long kvm_hypercall0(u64 fid) { diff --git a/arch/loongarch/include/asm/kvm_pch_pic.h b/arch/loongarch/include/asm/kvm_pch_pic.h new file mode 100644 index 0000000000000000000000000000000000000000..91bd5a5ec575d23662df8e85a28ce44024841db7 --- /dev/null +++ b/arch/loongarch/include/asm/kvm_pch_pic.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#ifndef LOONGARCH_PCH_PIC_H +#define LOONGARCH_PCH_PIC_H + +#include + +#define PCH_PIC_SIZE 0x3e8 + +#define PCH_PIC_INT_ID_START 0x0 +#define PCH_PIC_INT_ID_END 0x7 +#define PCH_PIC_MASK_START 0x20 +#define PCH_PIC_MASK_END 0x27 +#define PCH_PIC_HTMSI_EN_START 0x40 +#define PCH_PIC_HTMSI_EN_END 0x47 +#define PCH_PIC_EDGE_START 0x60 +#define PCH_PIC_EDGE_END 0x67 +#define PCH_PIC_CLEAR_START 0x80 +#define PCH_PIC_CLEAR_END 0x87 +#define PCH_PIC_AUTO_CTRL0_START 0xc0 +#define PCH_PIC_AUTO_CTRL0_END 0xc7 +#define PCH_PIC_AUTO_CTRL1_START 0xe0 +#define PCH_PIC_AUTO_CTRL1_END 0xe7 +#define PCH_PIC_ROUTE_ENTRY_START 0x100 +#define PCH_PIC_ROUTE_ENTRY_END 0x13f +#define PCH_PIC_HTMSI_VEC_START 0x200 +#define PCH_PIC_HTMSI_VEC_END 0x23f +#define PCH_PIC_INT_IRR_START 0x380 +#define PCH_PIC_INT_IRR_END 0x38f +#define PCH_PIC_INT_ISR_START 0x3a0 +#define PCH_PIC_INT_ISR_END 0x3af +#define PCH_PIC_POLARITY_START 0x3e0 +#define PCH_PIC_POLARITY_END 0x3e7 +#define PCH_PIC_INT_ID_VAL 0x7000000UL +#define PCH_PIC_INT_ID_VER 0x1UL + +struct loongarch_pch_pic { + spinlock_t lock; + struct kvm *kvm; + struct kvm_io_device device; + uint64_t mask; /* 1:disable irq, 0:enable irq */ + uint64_t htmsi_en; /* 1:msi */ + uint64_t edge; /* 1:edge triggered, 0:level triggered */ + uint64_t auto_ctrl0; /* only use default value 00b */ + uint64_t auto_ctrl1; /* only use default value 00b */ + uint64_t last_intirr; /* edge detection */ + uint64_t irr; /* interrupt request register */ + uint64_t isr; /* interrupt service register */ + uint64_t polarity; /* 0: high level trigger, 1: low level trigger */ + uint8_t route_entry[64]; /* default value 0, route to int0: extioi */ + uint8_t htmsi_vector[64]; /* irq route table for routing to extioi */ + uint64_t pch_pic_base; +}; + +void pch_pic_set_irq(struct loongarch_pch_pic *s, int irq, int level); +void pch_msi_set_irq(struct kvm *kvm, int irq, int level); +int kvm_loongarch_register_pch_pic_device(void); +#endif /* LOONGARCH_PCH_PIC_H */ diff --git a/arch/loongarch/include/asm/kvm_vcpu.h b/arch/loongarch/include/asm/kvm_vcpu.h index c416cb7125c0e5eb2ef0f48c6cb1bb54d3cb3804..d7e8f7d50ee0ce13360336f6c66ba24c746989ea 100644 --- a/arch/loongarch/include/asm/kvm_vcpu.h +++ b/arch/loongarch/include/asm/kvm_vcpu.h @@ -75,8 +75,13 @@ static inline void kvm_save_lasx(struct loongarch_fpu *fpu) { } static inline void kvm_restore_lasx(struct loongarch_fpu *fpu) { } #endif +#ifdef CONFIG_CPU_HAS_LBT +int kvm_own_lbt(struct kvm_vcpu *vcpu); +#else +static inline int kvm_own_lbt(struct kvm_vcpu *vcpu) { return -EINVAL; } +#endif + void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz); -void kvm_reset_timer(struct kvm_vcpu *vcpu); void kvm_save_timer(struct kvm_vcpu *vcpu); void kvm_restore_timer(struct kvm_vcpu *vcpu); @@ -125,4 +130,9 @@ static inline bool kvm_pvtime_supported(void) return !!sched_info_on(); } +static inline bool kvm_guest_has_pv_feature(struct kvm_vcpu *vcpu, unsigned int feature) +{ + return vcpu->kvm->arch.pv_features & BIT(feature); +} + #endif /* __ASM_LOONGARCH_KVM_VCPU_H__ */ diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index b4c628543b4f6d700e973f32489396ffd7a0c7d9..a2117ed1939a1e127a2d815e1bf350747bcdf445 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -161,16 +161,8 @@ /* * CPUCFG index area: 0x40000000 -- 0x400000ff - * SW emulation for KVM hypervirsor + * SW emulation for KVM hypervirsor, see arch/loongarch/include/uapi/asm/kvm_para.h */ -#define CPUCFG_KVM_BASE 0x40000000 -#define CPUCFG_KVM_SIZE 0x100 - -#define CPUCFG_KVM_SIG (CPUCFG_KVM_BASE + 0) -#define KVM_SIGNATURE "KVM\0" -#define CPUCFG_KVM_FEATURE (CPUCFG_KVM_BASE + 4) -#define KVM_FEATURE_IPI BIT(1) -#define KVM_FEATURE_STEAL_TIME BIT(2) #ifndef __ASSEMBLY__ @@ -887,7 +879,7 @@ #define LOONGARCH_CSR_DMWIN2 0x182 /* 64 direct map win2: MEM */ #define LOONGARCH_CSR_DMWIN3 0x183 /* 64 direct map win3: MEM */ -/* Direct Map window 0/1 */ +/* Direct Map window 0/1/2/3 */ #define CSR_DMW0_PLV0 _CONST64_(1 << 0) #define CSR_DMW0_VSEG _CONST64_(0x8000) #define CSR_DMW0_BASE (CSR_DMW0_VSEG << DMW_PABITS) @@ -899,6 +891,14 @@ #define CSR_DMW1_BASE (CSR_DMW1_VSEG << DMW_PABITS) #define CSR_DMW1_INIT (CSR_DMW1_BASE | CSR_DMW1_MAT | CSR_DMW1_PLV0) +#define CSR_DMW2_PLV0 _CONST64_(1 << 0) +#define CSR_DMW2_MAT _CONST64_(2 << 4) +#define CSR_DMW2_VSEG _CONST64_(0xa000) +#define CSR_DMW2_BASE (CSR_DMW2_VSEG << DMW_PABITS) +#define CSR_DMW2_INIT (CSR_DMW2_BASE | CSR_DMW2_MAT | CSR_DMW2_PLV0) + +#define CSR_DMW3_INIT 0x0 + /* Performance Counter registers */ #define LOONGARCH_CSR_PERFCTRL0 0x200 /* 32 perf event 0 config */ #define LOONGARCH_CSR_PERFCNTR0 0x201 /* 64 perf event 0 count value */ diff --git a/arch/loongarch/include/asm/paravirt.h b/arch/loongarch/include/asm/paravirt.h index dddec49671ae4254dc49acf69f79d499945b711f..63754678c3c60ef07aa81b1b202d04c8957d9429 100644 --- a/arch/loongarch/include/asm/paravirt.h +++ b/arch/loongarch/include/asm/paravirt.h @@ -31,5 +31,11 @@ static inline int pv_time_init(void) { return 0; } + +static inline int pv_spinlock_init(void) +{ + return 0; +} + #endif // CONFIG_PARAVIRT #endif diff --git a/arch/loongarch/include/asm/qspinlock.h b/arch/loongarch/include/asm/qspinlock.h new file mode 100644 index 0000000000000000000000000000000000000000..e76d3aa1e1ebe7dcae953bf6dc89d55802e05cfc --- /dev/null +++ b/arch/loongarch/include/asm/qspinlock.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH_QSPINLOCK_H +#define _ASM_LOONGARCH_QSPINLOCK_H + +#include + +#ifdef CONFIG_PARAVIRT + +DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); + +#define virt_spin_lock virt_spin_lock + +static inline bool virt_spin_lock(struct qspinlock *lock) +{ + int val; + + if (!static_branch_unlikely(&virt_spin_lock_key)) + return false; + + /* + * On hypervisors without PARAVIRT_SPINLOCKS support we fall + * back to a Test-and-Set spinlock, because fair locks have + * horrible lock 'holder' preemption issues. + */ + +__retry: + val = atomic_read(&lock->val); + + if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) { + cpu_relax(); + goto __retry; + } + + return true; +} + +#endif /* CONFIG_PARAVIRT */ + +#include + +#endif // _ASM_LOONGARCH_QSPINLOCK_H diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h index ee52fb1e99631653e3e40d6998afd159a7e5986d..eefb30c33ba33e1de9153676e8653d851bfa1c6c 100644 --- a/arch/loongarch/include/asm/setup.h +++ b/arch/loongarch/include/asm/setup.h @@ -12,6 +12,7 @@ #define VECSIZE 0x200 +extern bool disable_pci_irq_limit; extern unsigned long eentry; extern unsigned long tlbrentry; extern char init_command_line[COMMAND_LINE_SIZE]; diff --git a/arch/loongarch/include/asm/shmparam.h b/arch/loongarch/include/asm/shmparam.h new file mode 100644 index 0000000000000000000000000000000000000000..8af1e70cbf2c5acaf81f4a4cf81675f980f6bc2f --- /dev/null +++ b/arch/loongarch/include/asm/shmparam.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#ifndef _ASM_SHMPARAM_H +#define _ASM_SHMPARAM_H + +#define __ARCH_FORCE_SHMLBA 1 +#include + +#endif /* _ASM_SHMPARAM_H */ diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h index 630e5ebec21cb3700dc1d22b003111ef69906daa..cc232901e4dd2373f97eb38b3955936919526d2f 100644 --- a/arch/loongarch/include/asm/smp.h +++ b/arch/loongarch/include/asm/smp.h @@ -102,4 +102,7 @@ static inline void __cpu_die(unsigned int cpu) } #endif +int topo_add_cpu(int physid); +int topo_get_cpu(int physid); + #endif /* __ASM_SMP_H */ diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h index d9eafd3ee3d1e848675f54f033163ecc66e07d72..66736837085b61cd7c6d62461e5beb237e982fec 100644 --- a/arch/loongarch/include/asm/stackframe.h +++ b/arch/loongarch/include/asm/stackframe.h @@ -38,6 +38,17 @@ cfi_restore \reg \offset \docfi .endm + .macro SETUP_DMWINS temp + li.d \temp, CSR_DMW0_INIT # WUC, PLV0, 0x8000 xxxx xxxx xxxx + csrwr \temp, LOONGARCH_CSR_DMWIN0 + li.d \temp, CSR_DMW1_INIT # CAC, PLV0, 0x9000 xxxx xxxx xxxx + csrwr \temp, LOONGARCH_CSR_DMWIN1 + li.d \temp, CSR_DMW2_INIT # WUC, PLV0, 0xa000 xxxx xxxx xxxx + csrwr \temp, LOONGARCH_CSR_DMWIN2 + li.d \temp, CSR_DMW3_INIT # 0x0, unused + csrwr \temp, LOONGARCH_CSR_DMWIN3 + .endm + /* Jump to the runtime virtual address. */ .macro JUMP_VIRT_ADDR temp1 temp2 li.d \temp1, CACHE_BASE diff --git a/arch/loongarch/include/uapi/asm/bitsperlong.h b/arch/loongarch/include/uapi/asm/bitsperlong.h new file mode 100644 index 0000000000000000000000000000000000000000..00b4ba1e5cdf032f81aff9f728dcffea68571880 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/bitsperlong.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __ASM_LOONGARCH_BITSPERLONG_H +#define __ASM_LOONGARCH_BITSPERLONG_H + +#define __BITS_PER_LONG (__SIZEOF_LONG__ * 8) + +#include + +#endif /* __ASM_LOONGARCH_BITSPERLONG_H */ diff --git a/arch/loongarch/include/uapi/asm/kvm.h b/arch/loongarch/include/uapi/asm/kvm.h index 71baa77c3d3797bfca32a49101b257f7d32a16f3..d65a2993a406313f8f450d626f58f51af453db93 100644 --- a/arch/loongarch/include/uapi/asm/kvm.h +++ b/arch/loongarch/include/uapi/asm/kvm.h @@ -18,6 +18,7 @@ #define KVM_COALESCED_MMIO_PAGE_OFFSET 1 #define KVM_DIRTY_LOG_PAGE_OFFSET 64 +#define __KVM_HAVE_IRQ_LINE #define KVM_GUESTDBG_USE_SW_BP 0x00010000 @@ -66,6 +67,7 @@ struct kvm_fpu { #define KVM_REG_LOONGARCH_KVM (KVM_REG_LOONGARCH | 0x20000ULL) #define KVM_REG_LOONGARCH_FPSIMD (KVM_REG_LOONGARCH | 0x30000ULL) #define KVM_REG_LOONGARCH_CPUCFG (KVM_REG_LOONGARCH | 0x40000ULL) +#define KVM_REG_LOONGARCH_LBT (KVM_REG_LOONGARCH | 0x50000ULL) #define KVM_REG_LOONGARCH_MASK (KVM_REG_LOONGARCH | 0x70000ULL) #define KVM_CSR_IDX_MASK 0x7fff #define KVM_CPUCFG_IDX_MASK 0x7fff @@ -79,11 +81,30 @@ struct kvm_fpu { /* Debugging: Special instruction for software breakpoint */ #define KVM_REG_LOONGARCH_DEBUG_INST (KVM_REG_LOONGARCH_KVM | KVM_REG_SIZE_U64 | 3) +/* LBT registers */ +#define KVM_REG_LOONGARCH_LBT_SCR0 (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 1) +#define KVM_REG_LOONGARCH_LBT_SCR1 (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 2) +#define KVM_REG_LOONGARCH_LBT_SCR2 (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 3) +#define KVM_REG_LOONGARCH_LBT_SCR3 (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 4) +#define KVM_REG_LOONGARCH_LBT_EFLAGS (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 5) +#define KVM_REG_LOONGARCH_LBT_FTOP (KVM_REG_LOONGARCH_LBT | KVM_REG_SIZE_U64 | 6) + #define LOONGARCH_REG_SHIFT 3 #define LOONGARCH_REG_64(TYPE, REG) (TYPE | KVM_REG_SIZE_U64 | (REG << LOONGARCH_REG_SHIFT)) #define KVM_IOC_CSRID(REG) LOONGARCH_REG_64(KVM_REG_LOONGARCH_CSR, REG) #define KVM_IOC_CPUCFG(REG) LOONGARCH_REG_64(KVM_REG_LOONGARCH_CPUCFG, REG) +/* Device Control API on vm fd */ +#define KVM_LOONGARCH_VM_FEAT_CTRL 0 +#define KVM_LOONGARCH_VM_FEAT_LSX 0 +#define KVM_LOONGARCH_VM_FEAT_LASX 1 +#define KVM_LOONGARCH_VM_FEAT_X86BT 2 +#define KVM_LOONGARCH_VM_FEAT_ARMBT 3 +#define KVM_LOONGARCH_VM_FEAT_MIPSBT 4 +#define KVM_LOONGARCH_VM_FEAT_PMU 5 +#define KVM_LOONGARCH_VM_FEAT_PV_IPI 6 +#define KVM_LOONGARCH_VM_FEAT_PV_STEALTIME 7 + /* Device Control API on vcpu fd */ #define KVM_LOONGARCH_VCPU_CPUCFG 0 #define KVM_LOONGARCH_VCPU_PVTIME_CTRL 1 @@ -114,4 +135,15 @@ struct kvm_iocsr_entry { #define KVM_IRQCHIP_NUM_PINS 64 #define KVM_MAX_CORES 256 +#define KVM_LOONGARCH_VM_HAVE_IRQCHIP 0x40000001 + +#define KVM_DEV_LOONGARCH_IPI_GRP_REGS 0x40000002 + +#define KVM_DEV_LOONGARCH_EXTIOI_GRP_REGS 0x40000003 + +#define KVM_DEV_LOONGARCH_PCH_PIC_GRP_CTRL 0x40000004 +#define KVM_DEV_LOONGARCH_PCH_PIC_CTRL_INIT 0 + +#define KVM_DEV_LOONGARCH_PCH_PIC_GRP_REGS 0x40000005 + #endif /* __UAPI_ASM_LOONGARCH_KVM_H */ diff --git a/arch/loongarch/include/uapi/asm/kvm_para.h b/arch/loongarch/include/uapi/asm/kvm_para.h new file mode 100644 index 0000000000000000000000000000000000000000..b0604aa9b4bbd29dfafedbdf59f571bd445c2c2c --- /dev/null +++ b/arch/loongarch/include/uapi/asm/kvm_para.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_KVM_PARA_H +#define _UAPI_ASM_KVM_PARA_H + +#include + +/* + * CPUCFG index area: 0x40000000 -- 0x400000ff + * SW emulation for KVM hypervirsor + */ +#define CPUCFG_KVM_BASE 0x40000000 +#define CPUCFG_KVM_SIZE 0x100 +#define CPUCFG_KVM_SIG (CPUCFG_KVM_BASE + 0) +#define KVM_SIGNATURE "KVM\0" +#define CPUCFG_KVM_FEATURE (CPUCFG_KVM_BASE + 4) +#define KVM_FEATURE_IPI 1 +#define KVM_FEATURE_STEAL_TIME 2 +/* BIT 24 - 31 are features configurable by user space vmm */ +#define KVM_FEATURE_VIRT_EXTIOI 24 + +#endif /* _UAPI_ASM_KVM_PARA_H */ diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index 2c5066a003d1d00ab1940128f37dce57f29ba92c..f5b13966228f6c956da7e381d2ed1792c4dd957a 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -71,10 +71,10 @@ int set_processor_mask(u32 id, u32 flags) return -ENODEV; } - if (cpuid == loongson_sysconf.boot_cpu_id) - cpu = 0; - else - cpu = cpumask_next_zero(-1, cpu_present_mask); + + cpu = topo_add_cpu(cpuid); + if (cpu < 0) + return -EEXIST; if (flags & ACPI_MADT_ENABLED) { num_processors++; @@ -197,8 +197,6 @@ void __init acpi_boot_table_init(void) goto fdt_earlycon; } - loongson_sysconf.boot_cpu_id = read_csr_cpuid(); - /* * Process the Multiple APIC Description Table (MADT), if present */ @@ -248,7 +246,7 @@ void __init numa_set_distance(int from, int to, int distance) void __init acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) { - int pxm, node; + int pxm, node, cpu; if (srat_disabled()) return; @@ -277,6 +275,11 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) return; } + cpu = topo_get_cpu(pa->apic_id); + /* Check whether apic_id exists in MADT table */ + if (cpu < 0) + return; + early_numa_add_cpu(pa->apic_id, node); set_cpuid_to_node(pa->apic_id, node); @@ -315,12 +318,17 @@ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id, int *pcpu { int cpu; - cpu = set_processor_mask(physid, ACPI_MADT_ENABLED); + cpu = topo_get_cpu(physid); + /* Check whether apic_id exists in MADT table */ if (cpu < 0) { pr_info(PREFIX "Unable to map lapic to logical cpu number\n"); return cpu; } + num_processors++; + set_cpu_present(cpu, true); + __cpu_number_map[physid] = cpu; + __cpu_logical_map[cpu] = physid; acpi_map_cpu2node(handle, cpu, physid); *pcpu = cpu; diff --git a/arch/loongarch/kernel/fpu.S b/arch/loongarch/kernel/fpu.S index 69a85f2479fba1b34a0f56cd5618be707993d438..6ab640101457cc4d222716de1ab17f8b4090f9c4 100644 --- a/arch/loongarch/kernel/fpu.S +++ b/arch/loongarch/kernel/fpu.S @@ -530,6 +530,10 @@ SYM_FUNC_END(_restore_lasx_context) #ifdef CONFIG_CPU_HAS_LBT STACK_FRAME_NON_STANDARD _restore_fp +#ifdef CONFIG_CPU_HAS_LSX STACK_FRAME_NON_STANDARD _restore_lsx +#endif +#ifdef CONFIG_CPU_HAS_LASX STACK_FRAME_NON_STANDARD _restore_lasx #endif +#endif diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 4677ea8fa8e98cad8b11b5ea89d7811500be738e..5305f5e44462375fe8691f41ef6296f0b8f80866 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -44,11 +44,7 @@ SYM_DATA(kernel_fsize, .long _kernel_fsize); SYM_CODE_START(kernel_entry) # kernel entry point /* Config direct window and set PG */ - li.d t0, CSR_DMW0_INIT # UC, PLV0, 0x8000 xxxx xxxx xxxx - csrwr t0, LOONGARCH_CSR_DMWIN0 - li.d t0, CSR_DMW1_INIT # CA, PLV0, 0x9000 xxxx xxxx xxxx - csrwr t0, LOONGARCH_CSR_DMWIN1 - + SETUP_DMWINS t0 JUMP_VIRT_ADDR t0, t1 /* Enable PG */ @@ -124,11 +120,9 @@ SYM_CODE_END(kernel_entry) * function after setting up the stack and tp registers. */ SYM_CODE_START(smpboot_entry) - li.d t0, CSR_DMW0_INIT # UC, PLV0 - csrwr t0, LOONGARCH_CSR_DMWIN0 - li.d t0, CSR_DMW1_INIT # CA, PLV0 - csrwr t0, LOONGARCH_CSR_DMWIN1 - + + /* Config direct window and set PG */ + SETUP_DMWINS t0 JUMP_VIRT_ADDR t0, t1 #ifdef CONFIG_PAGE_SIZE_4KB diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c index 0268076cd78927a2123362c9e8fd4a3c0862fae2..38bbd62378d9b79477af642e724a79409ece42f5 100644 --- a/arch/loongarch/kernel/paravirt.c +++ b/arch/loongarch/kernel/paravirt.c @@ -12,6 +12,7 @@ static int has_steal_clock; struct static_key paravirt_steal_enabled; struct static_key paravirt_steal_rq_enabled; static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64); +DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); static u64 native_steal_clock(int cpu) { @@ -165,7 +166,7 @@ int __init pv_ipi_init(void) if (!cpu_has_hypervisor) return 0; - if (!kvm_para_available()) + if (!(feature & BIT(KVM_FEATURE_IPI))) return 0; feature = read_cpucfg(CPUCFG_KVM_FEATURE); @@ -200,7 +201,7 @@ static int pv_enable_steal_time(void) } addr |= KVM_STEAL_PHYS_VALID; - kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, KVM_FEATURE_STEAL_TIME, addr); + kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), addr); return 0; } @@ -208,7 +209,7 @@ static int pv_enable_steal_time(void) static void pv_disable_steal_time(void) { if (has_steal_clock) - kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, KVM_FEATURE_STEAL_TIME, 0); + kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), 0); } #ifdef CONFIG_SMP @@ -260,7 +261,7 @@ int __init pv_time_init(void) return 0; feature = read_cpucfg(CPUCFG_KVM_FEATURE); - if (!(feature & KVM_FEATURE_STEAL_TIME)) + if (!(feature & BIT(KVM_FEATURE_STEAL_TIME))) return 0; has_steal_clock = 1; @@ -294,3 +295,13 @@ int __init pv_time_init(void) return 0; } + +int __init pv_spinlock_init(void) +{ + if (!cpu_has_hypervisor) + return 0; + + static_branch_enable(&virt_spin_lock_key); + + return 0; +} diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c index aaa27823ea29d7d9c473c8877bdd2f233b693f91..14e5bb8e57c3e69ad28e68551804595d1acf0211 100644 --- a/arch/loongarch/kernel/relocate.c +++ b/arch/loongarch/kernel/relocate.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include "legacy_boot.h" @@ -171,7 +172,7 @@ unsigned long __init relocate_kernel(void) unsigned long kernel_length; unsigned long random_offset = 0; void *location_new = _text; /* Default to original kernel start */ - char *cmdline = early_ioremap(fw_arg1, COMMAND_LINE_SIZE); /* Boot command line is passed in fw_arg1 */ + char *cmdline = early_memremap_ro(fw_arg1, COMMAND_LINE_SIZE); /* Boot command line is passed in fw_arg1 */ if (fw_arg0 < 2) strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE); @@ -186,6 +187,7 @@ unsigned long __init relocate_kernel(void) random_offset = (unsigned long)location_new - (unsigned long)(_text); #endif reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS; + early_memunmap(cmdline, COMMAND_LINE_SIZE); if (random_offset) { kernel_length = (long)(_end) - (long)(_text); diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 0c3bfd0a57ff5a43a0c6cf56ca3c7f7e853296a1..ce5a98273f980fb0f2ba9cdacfea056497e5c26e 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -57,6 +57,7 @@ #define SMBIOS_FREQHIGH_OFFSET 0x17 #define SMBIOS_FREQLOW_MASK 0xFF #define SMBIOS_CORE_PACKAGE_OFFSET 0x23 +#define SMBIOS_THREAD_PACKAGE_OFFSET 0x25 #define LOONGSON_EFI_ENABLE (1 << 3) #ifdef CONFIG_EFI @@ -71,6 +72,8 @@ EXPORT_SYMBOL(cpu_data); struct loongson_board_info b_info; static const char dmi_empty_string[] = " "; +static int possible_cpus; +static bool bsp_added; /* * Setup information @@ -131,7 +134,7 @@ static void __init parse_cpu_table(const struct dmi_header *dm) cpu_clock_freq = freq_temp * 1000000; loongson_sysconf.cpuname = (void *)dmi_string_parse(dm, dmi_data[16]); - loongson_sysconf.cores_per_package = *(dmi_data + SMBIOS_CORE_PACKAGE_OFFSET); + loongson_sysconf.cores_per_package = *(dmi_data + SMBIOS_THREAD_PACKAGE_OFFSET); pr_info("CpuClock = %llu\n", cpu_clock_freq); } @@ -185,12 +188,14 @@ bool wc_enabled = false; EXPORT_SYMBOL(wc_enabled); +static int wc_arg = -1; + static int __init setup_writecombine(char *p) { if (!strcmp(p, "on")) - wc_enabled = true; + wc_arg = true; else if (!strcmp(p, "off")) - wc_enabled = false; + wc_arg = false; else pr_warn("Unknown writecombine setting \"%s\".\n", p); @@ -359,10 +364,75 @@ static void __init bootcmdline_init(char **cmdline_p) *cmdline_p = boot_command_line; } +int topo_get_cpu(int physid) +{ + int i; + + for (i = 0; i < possible_cpus; i++) + if (cpu_logical_map(i) == physid) + break; + + if (i == possible_cpus) + return -ENOENT; + + return i; +} + +int topo_add_cpu(int physid) +{ + int cpu; + + if (!bsp_added && (physid == loongson_sysconf.boot_cpu_id)) { + bsp_added = true; + return 0; + } + + cpu = topo_get_cpu(physid); + if (cpu >= 0) { + pr_warn("Adding duplicated physical cpuid 0x%x\n", physid); + return -EEXIST; + } + + if (possible_cpus >= nr_cpu_ids) + return -ERANGE; + + __cpu_logical_map[possible_cpus] = physid; + cpu = possible_cpus++; + return cpu; +} + +static void __init topo_init(void) +{ + loongson_sysconf.boot_cpu_id = read_csr_cpuid(); + __cpu_logical_map[0] = loongson_sysconf.boot_cpu_id; + possible_cpus++; +} + +static void __init writecombine_detect(void) +{ + u64 cpuname; + + if (wc_arg >= 0) { + wc_enabled = wc_arg; + return; + } + + cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME); + cpuname &= 0x0000ffffffffffff; + switch (cpuname) { + case 0x0000303030364333: + wc_enabled = false; + break; + default: + break; + } +} + void __init platform_init(void) { arch_reserve_vmcore(); arch_reserve_crashkernel(); + topo_init(); #ifdef CONFIG_ACPI_TABLE_UPGRADE acpi_table_upgrade(); @@ -382,6 +452,8 @@ void __init platform_init(void) smbios_parse(); pr_info("The BIOS Version: %s\n", b_info.bios_version); + writecombine_detect(); + pr_info("WriteCombine: %s\n", wc_enabled ? "on":"off"); efi_runtime_init(); } @@ -618,6 +690,8 @@ void __init setup_arch(char **cmdline_p) arch_mem_init(cmdline_p); resource_init(); + jump_label_init(); /* Initialise the static keys for paravirtualization */ + #ifdef CONFIG_SMP plat_smp_setup(); prefill_possible_map(); diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 035560315af5103a183c04a575301a6d12ee799e..bd13247a2d532cba9dd293ed96275a9eac7a5c42 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -288,11 +288,9 @@ static void __init fdt_smp_setup(void) if (cpuid >= nr_cpu_ids) continue; - if (cpuid == loongson_sysconf.boot_cpu_id) { - cpu = 0; - } else { - cpu = cpumask_next_zero(-1, cpu_present_mask); - } + cpu = topo_add_cpu(cpuid); + if (cpu < 0) + continue; num_processors++; set_cpu_possible(cpu, true); @@ -300,7 +298,7 @@ static void __init fdt_smp_setup(void) __cpu_number_map[cpuid] = cpu; __cpu_logical_map[cpu] = cpuid; - early_numa_add_cpu(cpu, 0); + early_numa_add_cpu(cpuid, 0); set_cpuid_to_node(cpuid, 0); } @@ -477,7 +475,7 @@ core_initcall(ipi_pm_init); #endif /* Preload SMP state for boot cpu */ -void smp_prepare_boot_cpu(void) +void __init smp_prepare_boot_cpu(void) { unsigned int cpu, node, rr_node; @@ -510,6 +508,8 @@ void smp_prepare_boot_cpu(void) rr_node = next_node_in(rr_node, node_online_map); } } + + pv_spinlock_init(); } /* called from main before smp_init() */ diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig index f1a9b518ab7b5b03de2549d5cd8c58e7455d71de..39c66452b1f0956a5d4a69fd48f9137340de584e 100644 --- a/arch/loongarch/kvm/Kconfig +++ b/arch/loongarch/kvm/Kconfig @@ -24,6 +24,10 @@ config KVM select HAVE_KVM_DIRTY_RING_ACQ_REL select HAVE_KVM_EVENTFD select HAVE_KVM_VCPU_ASYNC_IOCTL + select HAVE_KVM_IRQ_ROUTING + select HAVE_KVM_IRQCHIP + select HAVE_KVM_IRQFD + select HAVE_KVM_MSI select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING select KVM_GENERIC_MMU_NOTIFIER diff --git a/arch/loongarch/kvm/Makefile b/arch/loongarch/kvm/Makefile index 244467d7792a97c80c6558c39b2b7bf8a898fafd..f363e4b6fcf35ae6e615a40555ec6e744fa84687 100644 --- a/arch/loongarch/kvm/Makefile +++ b/arch/loongarch/kvm/Makefile @@ -18,5 +18,9 @@ kvm-y += timer.o kvm-y += tlb.o kvm-y += vcpu.o kvm-y += vm.o +kvm-y += intc/ipi.o +kvm-y += intc/extioi.o +kvm-y += intc/pch_pic.o +kvm-y += irqfd.o CFLAGS_exit.o += $(call cc-option,-Wno-override-init,) diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index ef37d82fb56f76f8b73b67779635ee77899eae59..7b40b018d7138fc10af383dac981f0e0720182a4 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -50,9 +50,7 @@ static int kvm_emu_cpucfg(struct kvm_vcpu *vcpu, larch_inst inst) vcpu->arch.gprs[rd] = *(unsigned int *)KVM_SIGNATURE; break; case CPUCFG_KVM_FEATURE: - ret = KVM_FEATURE_IPI; - if (kvm_pvtime_supported()) - ret |= KVM_FEATURE_STEAL_TIME; + ret = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK; vcpu->arch.gprs[rd] = ret; break; default: @@ -159,7 +157,7 @@ static int kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst) int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu) { int ret; - unsigned long val; + unsigned long *val; u32 addr, rd, rj, opcode; /* @@ -172,6 +170,7 @@ int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu) ret = EMULATE_DO_IOCSR; run->iocsr_io.phys_addr = addr; run->iocsr_io.is_write = 0; + val = &vcpu->arch.gprs[rd]; /* LoongArch is Little endian */ switch (opcode) { @@ -205,15 +204,21 @@ int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu) break; default: ret = EMULATE_FAIL; - break; + return ret; } - if (ret == EMULATE_DO_IOCSR) { - if (run->iocsr_io.is_write) { - val = vcpu->arch.gprs[rd]; - memcpy(run->iocsr_io.data, &val, run->iocsr_io.len); - } - vcpu->arch.io_gpr = rd; + if (run->iocsr_io.is_write) { + if (!kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, run->iocsr_io.len, val)) + ret = EMULATE_DONE; + else + /* Save data and let user space to write it */ + memcpy(run->iocsr_io.data, val, run->iocsr_io.len); + } else { + if (!kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, run->iocsr_io.len, val)) + ret = EMULATE_DONE; + else + /* Save register id for iocsr read completion */ + vcpu->arch.io_gpr = rd; } return ret; @@ -449,18 +454,30 @@ int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst) } if (ret == EMULATE_DO_MMIO) { + /* + * if mmio device such as pch pic is emulated in KVM, + * it need not return to user space to handle the mmio + * exception. + */ + ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, + run->mmio.len, &vcpu->arch.gprs[rd]); + if (!ret) { + update_pc(&vcpu->arch); + vcpu->mmio_needed = 0; + return EMULATE_DONE; + } /* Set for kvm_complete_mmio_read() use */ vcpu->arch.io_gpr = rd; run->mmio.is_write = 0; vcpu->mmio_is_write = 0; trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, run->mmio.len, run->mmio.phys_addr, NULL); - } else { - kvm_err("Read not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", - inst.word, vcpu->arch.pc, vcpu->arch.badv); - kvm_arch_vcpu_dump_regs(vcpu); - vcpu->mmio_needed = 0; + return EMULATE_DO_MMIO; } + kvm_err("Read not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", + inst.word, vcpu->arch.pc, vcpu->arch.badv); + kvm_arch_vcpu_dump_regs(vcpu); + vcpu->mmio_needed = 0; return ret; } @@ -602,19 +619,30 @@ int kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst) } if (ret == EMULATE_DO_MMIO) { + /* + * if mmio device such as pch pic is emulated in KVM, + * it need not return to user space to handle the mmio + * exception. + */ + ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, + run->mmio.len, data); + if (!ret) + return EMULATE_DONE; + run->mmio.is_write = 1; vcpu->mmio_needed = 1; vcpu->mmio_is_write = 1; trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, run->mmio.phys_addr, data); - } else { - vcpu->arch.pc = curr_pc; - kvm_err("Write not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", - inst.word, vcpu->arch.pc, vcpu->arch.badv); - kvm_arch_vcpu_dump_regs(vcpu); - /* Rollback PC if emulation was unsuccessful */ + return EMULATE_DO_MMIO; } + vcpu->arch.pc = curr_pc; + kvm_err("Write not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", + inst.word, vcpu->arch.pc, vcpu->arch.badv); + kvm_arch_vcpu_dump_regs(vcpu); + /* Rollback PC if emulation was unsuccessful */ + return ret; } @@ -756,6 +784,14 @@ static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu) return RESUME_GUEST; } +static int kvm_handle_lbt_disabled(struct kvm_vcpu *vcpu) +{ + if (kvm_own_lbt(vcpu)) + kvm_queue_exception(vcpu, EXCCODE_INE, 0); + + return RESUME_GUEST; +} + static int kvm_send_pv_ipi(struct kvm_vcpu *vcpu) { unsigned int min, cpu, i; @@ -789,19 +825,21 @@ static int kvm_send_pv_ipi(struct kvm_vcpu *vcpu) */ static void kvm_handle_service(struct kvm_vcpu *vcpu) { + long ret = KVM_HCALL_INVALID_CODE; unsigned long func = kvm_read_reg(vcpu, LOONGARCH_GPR_A0); - long ret; switch (func) { case KVM_HCALL_FUNC_IPI: - kvm_send_pv_ipi(vcpu); - ret = KVM_HCALL_SUCCESS; + if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_IPI)) { + kvm_send_pv_ipi(vcpu); + ret = KVM_HCALL_SUCCESS; + } break; case KVM_HCALL_FUNC_NOTIFY: - ret = kvm_save_notify(vcpu); + if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME)) + ret = kvm_save_notify(vcpu); break; default: - ret = KVM_HCALL_INVALID_CODE; break; } @@ -873,6 +911,7 @@ static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = { [EXCCODE_FPDIS] = kvm_handle_fpu_disabled, [EXCCODE_LSXDIS] = kvm_handle_lsx_disabled, [EXCCODE_LASXDIS] = kvm_handle_lasx_disabled, + [EXCCODE_BTDIS] = kvm_handle_lbt_disabled, [EXCCODE_GSPR] = kvm_handle_gspr, [EXCCODE_HVC] = kvm_handle_hypercall, }; diff --git a/arch/loongarch/kvm/intc/extioi.c b/arch/loongarch/kvm/intc/extioi.c new file mode 100644 index 0000000000000000000000000000000000000000..5327066f16aec95ca383f127e1aa1f048fa73a10 --- /dev/null +++ b/arch/loongarch/kvm/intc/extioi.c @@ -0,0 +1,804 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#include +#include +#include + +#define loongarch_ext_irq_lock(s, flags) spin_lock_irqsave(&s->lock, flags) +#define loongarch_ext_irq_unlock(s, flags) spin_unlock_irqrestore(&s->lock, flags) + +static void extioi_update_irq(struct loongarch_extioi *s, int irq, int level) +{ + int ipnum, cpu, found, irq_index, irq_mask; + struct kvm_interrupt vcpu_irq; + struct kvm_vcpu *vcpu; + + ipnum = s->ipmap.reg_u8[irq / 32]; + ipnum = count_trailing_zeros(ipnum); + ipnum = (ipnum >= 0 && ipnum < 4) ? ipnum : 0; + + cpu = s->sw_coremap[irq]; + vcpu = kvm_get_vcpu(s->kvm, cpu); + irq_index = irq / 32; + /* length of accessing core isr is 4 bytes */ + irq_mask = 1 << (irq & 0x1f); + + if (level) { + /* if not enable return false */ + if (((s->enable.reg_u32[irq_index]) & irq_mask) == 0) + return; + s->coreisr.reg_u32[cpu][irq_index] |= irq_mask; + found = find_first_bit(s->sw_coreisr[cpu][ipnum], EXTIOI_IRQS); + set_bit(irq, s->sw_coreisr[cpu][ipnum]); + } else { + s->coreisr.reg_u32[cpu][irq_index] &= ~irq_mask; + clear_bit(irq, s->sw_coreisr[cpu][ipnum]); + found = find_first_bit(s->sw_coreisr[cpu][ipnum], EXTIOI_IRQS); + } + + if (found < EXTIOI_IRQS) + /* other irq is handling, need not update parent irq level */ + return; + + vcpu_irq.irq = level ? INT_HWI0 + ipnum : -(INT_HWI0 + ipnum); + kvm_vcpu_ioctl_interrupt(vcpu, &vcpu_irq); +} + +static void extioi_set_sw_coreisr(struct loongarch_extioi *s) +{ + int ipnum, cpu, irq_index, irq_mask, irq; + + for (irq = 0; irq < EXTIOI_IRQS; irq++) { + ipnum = s->ipmap.reg_u8[irq / 32]; + ipnum = count_trailing_zeros(ipnum); + ipnum = (ipnum >= 0 && ipnum < 4) ? ipnum : 0; + irq_index = irq / 32; + /* length of accessing core isr is 4 bytes */ + irq_mask = 1 << (irq & 0x1f); + + cpu = s->coremap.reg_u8[irq]; + if (!!(s->coreisr.reg_u32[cpu][irq_index] & irq_mask)) + set_bit(irq, s->sw_coreisr[cpu][ipnum]); + else + clear_bit(irq, s->sw_coreisr[cpu][ipnum]); + } +} + +void extioi_set_irq(struct loongarch_extioi *s, int irq, int level) +{ + unsigned long *isr = (unsigned long *)s->isr.reg_u8; + unsigned long flags; + + level ? set_bit(irq, isr) : clear_bit(irq, isr); + if (!level) + return; + loongarch_ext_irq_lock(s, flags); + extioi_update_irq(s, irq, level); + loongarch_ext_irq_unlock(s, flags); +} + +static inline void extioi_enable_irq(struct kvm_vcpu *vcpu, struct loongarch_extioi *s, + int index, u8 mask, int level) +{ + u8 val; + int irq; + + val = mask & s->isr.reg_u8[index]; + irq = ffs(val); + while (irq != 0) { + /* + * enable bit change from 0 to 1, + * need to update irq by pending bits + */ + extioi_update_irq(s, irq - 1 + index * 8, level); + val &= ~(1 << (irq - 1)); + irq = ffs(val); + } +} + +static int loongarch_extioi_writeb(struct kvm_vcpu *vcpu, + struct loongarch_extioi *s, + gpa_t addr, int len, const void *val) +{ + int index, irq, ret = 0; + u8 data, old_data, cpu; + u8 coreisr, old_coreisr; + gpa_t offset; + + data = *(u8 *)val; + offset = addr - EXTIOI_BASE; + + switch (offset) { + case EXTIOI_NODETYPE_START ... EXTIOI_NODETYPE_END: + index = (offset - EXTIOI_NODETYPE_START); + s->nodetype.reg_u8[index] = data; + break; + case EXTIOI_IPMAP_START ... EXTIOI_IPMAP_END: + /* + * ipmap cannot be set at runtime, can be set only at the beginning + * of intr driver, need not update upper irq level + */ + index = (offset - EXTIOI_IPMAP_START); + s->ipmap.reg_u8[index] = data; + break; + case EXTIOI_ENABLE_START ... EXTIOI_ENABLE_END: + index = (offset - EXTIOI_ENABLE_START); + old_data = s->enable.reg_u8[index]; + s->enable.reg_u8[index] = data; + /* + * 1: enable irq. + * update irq when isr is set. + */ + data = s->enable.reg_u8[index] & ~old_data & s->isr.reg_u8[index]; + extioi_enable_irq(vcpu, s, index, data, 1); + /* + * 0: disable irq. + * update irq when isr is set. + */ + data = ~s->enable.reg_u8[index] & old_data & s->isr.reg_u8[index]; + extioi_enable_irq(vcpu, s, index, data, 0); + break; + case EXTIOI_BOUNCE_START ... EXTIOI_BOUNCE_END: + /* do not emulate hw bounced irq routing */ + index = offset - EXTIOI_BOUNCE_START; + s->bounce.reg_u8[index] = data; + break; + case EXTIOI_COREISR_START ... EXTIOI_COREISR_END: + /* length of accessing core isr is 8 bytes */ + index = (offset - EXTIOI_COREISR_START); + /* using attrs to get current cpu index */ + cpu = vcpu->vcpu_id; + coreisr = data; + old_coreisr = s->coreisr.reg_u8[cpu][index]; + /* write 1 to clear interrupt */ + s->coreisr.reg_u8[cpu][index] = old_coreisr & ~coreisr; + coreisr &= old_coreisr; + irq = ffs(coreisr); + while (irq != 0) { + extioi_update_irq(s, irq - 1 + index * 8, 0); + coreisr &= ~(1 << (irq - 1)); + irq = ffs(coreisr); + } + break; + case EXTIOI_COREMAP_START ... EXTIOI_COREMAP_END: + irq = offset - EXTIOI_COREMAP_START; + index = irq; + s->coremap.reg_u8[index] = data; + + cpu = data & 0xff; + cpu = ffs(cpu) - 1; + cpu = (cpu >= 4) ? 0 : cpu; + + if (s->sw_coremap[irq] == cpu) + break; + + if (test_bit(irq, (unsigned long *)s->isr.reg_u8)) { + /* + * lower irq at old cpu and raise irq at new cpu + */ + extioi_update_irq(s, irq, 0); + s->sw_coremap[irq] = cpu; + extioi_update_irq(s, irq, 1); + } else + s->sw_coremap[irq] = cpu; + + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static int loongarch_extioi_writew(struct kvm_vcpu *vcpu, + struct loongarch_extioi *s, + gpa_t addr, int len, const void *val) +{ + int i, index, irq, ret = 0; + u8 cpu; + u32 data, old_data; + u32 coreisr, old_coreisr; + gpa_t offset; + + data = *(u32 *)val; + offset = addr - EXTIOI_BASE; + + switch (offset) { + case EXTIOI_NODETYPE_START ... EXTIOI_NODETYPE_END: + index = (offset - EXTIOI_NODETYPE_START) >> 2; + s->nodetype.reg_u32[index] = data; + break; + case EXTIOI_IPMAP_START ... EXTIOI_IPMAP_END: + /* + * ipmap cannot be set at runtime, can be set only at the beginning + * of intr driver, need not update upper irq level + */ + index = (offset - EXTIOI_IPMAP_START) >> 2; + s->ipmap.reg_u32[index] = data; + break; + case EXTIOI_ENABLE_START ... EXTIOI_ENABLE_END: + index = (offset - EXTIOI_ENABLE_START) >> 2; + old_data = s->enable.reg_u32[index]; + s->enable.reg_u32[index] = data; + /* + * 1: enable irq. + * update irq when isr is set. + */ + data = s->enable.reg_u32[index] & ~old_data & s->isr.reg_u32[index]; + index = index << 2; + for (i = 0; i < sizeof(data); i++) { + u8 mask = (data >> (i * 8)) & 0xff; + + extioi_enable_irq(vcpu, s, index + i, mask, 1); + } + /* + * 0: disable irq. + * update irq when isr is set. + */ + data = ~s->enable.reg_u32[index] & old_data & s->isr.reg_u32[index]; + for (i = 0; i < sizeof(data); i++) { + u8 mask = (data >> (i * 8)) & 0xff; + + extioi_enable_irq(vcpu, s, index, mask, 0); + } + break; + case EXTIOI_BOUNCE_START ... EXTIOI_BOUNCE_END: + /* do not emulate hw bounced irq routing */ + index = (offset - EXTIOI_BOUNCE_START) >> 2; + s->bounce.reg_u32[index] = data; + break; + case EXTIOI_COREISR_START ... EXTIOI_COREISR_END: + /* length of accessing core isr is 8 bytes */ + index = (offset - EXTIOI_COREISR_START) >> 2; + /* using attrs to get current cpu index */ + cpu = vcpu->vcpu_id; + coreisr = data; + old_coreisr = s->coreisr.reg_u32[cpu][index]; + /* write 1 to clear interrupt */ + s->coreisr.reg_u32[cpu][index] = old_coreisr & ~coreisr; + coreisr &= old_coreisr; + irq = ffs(coreisr); + while (irq != 0) { + extioi_update_irq(s, irq - 1 + index * 32, 0); + coreisr &= ~(1 << (irq - 1)); + irq = ffs(coreisr); + } + break; + case EXTIOI_COREMAP_START ... EXTIOI_COREMAP_END: + irq = offset - EXTIOI_COREMAP_START; + index = irq >> 2; + + s->coremap.reg_u32[index] = data; + + for (i = 0; i < sizeof(data); i++) { + cpu = data & 0xff; + cpu = ffs(cpu) - 1; + cpu = (cpu >= 4) ? 0 : cpu; + data = data >> 8; + + if (s->sw_coremap[irq + i] == cpu) + continue; + + if (test_bit(irq, (unsigned long *)s->isr.reg_u8)) { + /* + * lower irq at old cpu and raise irq at new cpu + */ + extioi_update_irq(s, irq + i, 0); + s->sw_coremap[irq + i] = cpu; + extioi_update_irq(s, irq + i, 1); + } else + s->sw_coremap[irq + i] = cpu; + } + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static int loongarch_extioi_writel(struct kvm_vcpu *vcpu, + struct loongarch_extioi *s, + gpa_t addr, int len, const void *val) +{ + int i, index, irq, bits, ret = 0; + u8 cpu; + u64 data, old_data; + u64 coreisr, old_coreisr; + gpa_t offset; + + data = *(u64 *)val; + offset = addr - EXTIOI_BASE; + + switch (offset) { + case EXTIOI_NODETYPE_START ... EXTIOI_NODETYPE_END: + index = (offset - EXTIOI_NODETYPE_START) >> 3; + s->nodetype.reg_u64[index] = data; + break; + case EXTIOI_IPMAP_START ... EXTIOI_IPMAP_END: + /* + * ipmap cannot be set at runtime, can be set only at the beginning + * of intr driver, need not update upper irq level + */ + index = (offset - EXTIOI_IPMAP_START) >> 3; + s->ipmap.reg_u64 = data; + break; + case EXTIOI_ENABLE_START ... EXTIOI_ENABLE_END: + index = (offset - EXTIOI_ENABLE_START) >> 3; + old_data = s->enable.reg_u64[index]; + s->enable.reg_u64[index] = data; + /* + * 1: enable irq. + * update irq when isr is set. + */ + data = s->enable.reg_u64[index] & ~old_data & s->isr.reg_u64[index]; + index = index << 3; + for (i = 0; i < sizeof(data); i++) { + u8 mask = (data >> (i * 8)) & 0xff; + + extioi_enable_irq(vcpu, s, index + i, mask, 1); + } + /* + * 0: disable irq. + * update irq when isr is set. + */ + data = ~s->enable.reg_u64[index] & old_data & s->isr.reg_u64[index]; + for (i = 0; i < sizeof(data); i++) { + u8 mask = (data >> (i * 8)) & 0xff; + + extioi_enable_irq(vcpu, s, index, mask, 0); + } + break; + case EXTIOI_BOUNCE_START ... EXTIOI_BOUNCE_END: + /* do not emulate hw bounced irq routing */ + index = (offset - EXTIOI_BOUNCE_START) >> 3; + s->bounce.reg_u64[index] = data; + break; + case EXTIOI_COREISR_START ... EXTIOI_COREISR_END: + /* length of accessing core isr is 8 bytes */ + index = (offset - EXTIOI_COREISR_START) >> 3; + /* using attrs to get current cpu index */ + cpu = vcpu->vcpu_id; + coreisr = data; + old_coreisr = s->coreisr.reg_u64[cpu][index]; + /* write 1 to clear interrupt */ + s->coreisr.reg_u64[cpu][index] = old_coreisr & ~coreisr; + coreisr &= old_coreisr; + + bits = sizeof(u64) * 8; + irq = find_first_bit((void *)&coreisr, bits); + while (irq < bits) { + extioi_update_irq(s, irq + index * bits, 0); + bitmap_clear((void *)&coreisr, irq, 1); + irq = find_first_bit((void *)&coreisr, bits); + } + break; + case EXTIOI_COREMAP_START ... EXTIOI_COREMAP_END: + irq = offset - EXTIOI_COREMAP_START; + index = irq >> 3; + + s->coremap.reg_u64[index] = data; + + for (i = 0; i < sizeof(data); i++) { + cpu = data & 0xff; + cpu = ffs(cpu) - 1; + cpu = (cpu >= 4) ? 0 : cpu; + data = data >> 8; + + if (s->sw_coremap[irq + i] == cpu) + continue; + + if (test_bit(irq, (unsigned long *)s->isr.reg_u8)) { + /* + * lower irq at old cpu and raise irq at new cpu + */ + extioi_update_irq(s, irq + i, 0); + s->sw_coremap[irq + i] = cpu; + extioi_update_irq(s, irq + i, 1); + } else + s->sw_coremap[irq + i] = cpu; + } + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static int kvm_loongarch_extioi_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + int ret; + struct loongarch_extioi *extioi = vcpu->kvm->arch.extioi; + unsigned long flags; + + if (!extioi) { + kvm_err("%s: extioi irqchip not valid!\n", __func__); + return -EINVAL; + } + + vcpu->kvm->stat.extioi_write_exits++; + loongarch_ext_irq_lock(extioi, flags); + + switch (len) { + case 1: + ret = loongarch_extioi_writeb(vcpu, extioi, addr, len, val); + break; + case 4: + ret = loongarch_extioi_writew(vcpu, extioi, addr, len, val); + break; + case 8: + ret = loongarch_extioi_writel(vcpu, extioi, addr, len, val); + break; + default: + WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx,size %d\n", + __func__, addr, len); + } + + loongarch_ext_irq_unlock(extioi, flags); + + + return ret; +} + +static int loongarch_extioi_readb(struct kvm_vcpu *vcpu, struct loongarch_extioi *s, + gpa_t addr, int len, void *val) +{ + int index, ret = 0; + gpa_t offset; + u64 data; + + offset = addr - EXTIOI_BASE; + switch (offset) { + case EXTIOI_NODETYPE_START ... EXTIOI_NODETYPE_END: + index = offset - EXTIOI_NODETYPE_START; + data = s->nodetype.reg_u8[index]; + break; + case EXTIOI_IPMAP_START ... EXTIOI_IPMAP_END: + index = offset - EXTIOI_IPMAP_START; + data = s->ipmap.reg_u8[index]; + break; + case EXTIOI_ENABLE_START ... EXTIOI_ENABLE_END: + index = offset - EXTIOI_ENABLE_START; + data = s->enable.reg_u8[index]; + break; + case EXTIOI_BOUNCE_START ... EXTIOI_BOUNCE_END: + index = offset - EXTIOI_BOUNCE_START; + data = s->bounce.reg_u8[index]; + break; + case EXTIOI_COREISR_START ... EXTIOI_COREISR_END: + /* length of accessing core isr is 8 bytes */ + index = offset - EXTIOI_COREISR_START; + data = s->coreisr.reg_u8[vcpu->vcpu_id][index]; + break; + case EXTIOI_COREMAP_START ... EXTIOI_COREMAP_END: + index = offset - EXTIOI_COREMAP_START; + data = s->coremap.reg_u8[index]; + break; + default: + ret = -EINVAL; + break; + } + + *(u8 *)val = data; + + return ret; +} + +static int loongarch_extioi_readw(struct kvm_vcpu *vcpu, struct loongarch_extioi *s, + gpa_t addr, int len, void *val) +{ + int index, ret = 0; + gpa_t offset; + u64 data; + + offset = addr - EXTIOI_BASE; + switch (offset) { + case EXTIOI_NODETYPE_START ... EXTIOI_NODETYPE_END: + index = (offset - EXTIOI_NODETYPE_START) >> 2; + data = s->nodetype.reg_u32[index]; + break; + case EXTIOI_IPMAP_START ... EXTIOI_IPMAP_END: + index = (offset - EXTIOI_IPMAP_START) >> 2; + data = s->ipmap.reg_u32[index]; + break; + case EXTIOI_ENABLE_START ... EXTIOI_ENABLE_END: + index = (offset - EXTIOI_ENABLE_START) >> 2; + data = s->enable.reg_u32[index]; + break; + case EXTIOI_BOUNCE_START ... EXTIOI_BOUNCE_END: + index = (offset - EXTIOI_BOUNCE_START) >> 2; + data = s->bounce.reg_u32[index]; + break; + case EXTIOI_COREISR_START ... EXTIOI_COREISR_END: + /* length of accessing core isr is 8 bytes */ + index = (offset - EXTIOI_COREISR_START) >> 2; + data = s->coreisr.reg_u32[vcpu->vcpu_id][index]; + break; + case EXTIOI_COREMAP_START ... EXTIOI_COREMAP_END: + index = (offset - EXTIOI_COREMAP_START) >> 2; + data = s->coremap.reg_u32[index]; + break; + default: + ret = -EINVAL; + break; + } + + *(u32 *)val = data; + + return ret; +} + +static int loongarch_extioi_readl(struct kvm_vcpu *vcpu, struct loongarch_extioi *s, + gpa_t addr, int len, void *val) +{ + int index, ret = 0; + gpa_t offset; + u64 data; + + offset = addr - EXTIOI_BASE; + switch (offset) { + case EXTIOI_NODETYPE_START ... EXTIOI_NODETYPE_END: + index = (offset - EXTIOI_NODETYPE_START) >> 3; + data = s->nodetype.reg_u64[index]; + break; + case EXTIOI_IPMAP_START ... EXTIOI_IPMAP_END: + index = (offset - EXTIOI_IPMAP_START) >> 3; + data = s->ipmap.reg_u64; + break; + case EXTIOI_ENABLE_START ... EXTIOI_ENABLE_END: + index = (offset - EXTIOI_ENABLE_START) >> 3; + data = s->enable.reg_u64[index]; + break; + case EXTIOI_BOUNCE_START ... EXTIOI_BOUNCE_END: + index = (offset - EXTIOI_BOUNCE_START) >> 3; + data = s->bounce.reg_u64[index]; + break; + case EXTIOI_COREISR_START ... EXTIOI_COREISR_END: + /* length of accessing core isr is 8 bytes */ + index = (offset - EXTIOI_COREISR_START) >> 3; + data = s->coreisr.reg_u64[vcpu->vcpu_id][index]; + break; + case EXTIOI_COREMAP_START ... EXTIOI_COREMAP_END: + index = (offset - EXTIOI_COREMAP_START) >> 3; + data = s->coremap.reg_u64[index]; + break; + default: + ret = -EINVAL; + break; + } + + *(u64 *)val = data; + + return ret; +} + +static int kvm_loongarch_extioi_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, void *val) +{ + int ret; + struct loongarch_extioi *extioi = vcpu->kvm->arch.extioi; + unsigned long flags; + + if (!extioi) { + kvm_err("%s: extioi irqchip not valid!\n", __func__); + return -EINVAL; + } + + vcpu->kvm->stat.extioi_read_exits++; + loongarch_ext_irq_lock(extioi, flags); + + switch (len) { + case 1: + ret = loongarch_extioi_readb(vcpu, extioi, addr, len, val); + break; + case 4: + ret = loongarch_extioi_readw(vcpu, extioi, addr, len, val); + break; + case 8: + ret = loongarch_extioi_readl(vcpu, extioi, addr, len, val); + break; + default: + WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx,size %d\n", + __func__, addr, len); + } + + loongarch_ext_irq_unlock(extioi, flags); + + return ret; +} + +static const struct kvm_io_device_ops kvm_loongarch_extioi_ops = { + .read = kvm_loongarch_extioi_read, + .write = kvm_loongarch_extioi_write, +}; + +static int kvm_loongarch_extioi_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + bool is_write) +{ + int len, addr; + void __user *data; + void *p = NULL; + struct loongarch_extioi *s; + unsigned long flags; + + s = dev->kvm->arch.extioi; + addr = attr->attr; + data = (void __user *)attr->addr; + + loongarch_ext_irq_lock(s, flags); + switch (addr) { + case EXTIOI_NODETYPE_START: + p = s->nodetype.reg_u8; + len = sizeof(s->nodetype); + break; + case EXTIOI_IPMAP_START: + p = s->ipmap.reg_u8; + len = sizeof(s->ipmap); + break; + case EXTIOI_ENABLE_START: + p = s->enable.reg_u8; + len = sizeof(s->enable); + break; + case EXTIOI_BOUNCE_START: + p = s->bounce.reg_u8; + len = sizeof(s->bounce); + break; + case EXTIOI_ISR_START: + p = s->isr.reg_u8; + len = sizeof(s->isr); + break; + case EXTIOI_COREISR_START: + p = s->coreisr.reg_u8; + len = sizeof(s->coreisr); + break; + case EXTIOI_COREMAP_START: + p = s->coremap.reg_u8; + len = sizeof(s->coremap); + break; + case EXTIOI_SW_COREMAP_FLAG: + p = s->sw_coremap; + len = sizeof(s->sw_coremap); + break; + default: + loongarch_ext_irq_unlock(s, flags); + kvm_err("%s: unknown extioi register, addr = %d\n", __func__, addr); + return -EINVAL; + } + + loongarch_ext_irq_unlock(s, flags); + + if (is_write) { + if (copy_from_user(p, data, len)) + return -EFAULT; + } else { + if (copy_to_user(data, p, len)) + return -EFAULT; + } + + if ((addr == EXTIOI_COREISR_START) && is_write) { + loongarch_ext_irq_lock(s, flags); + extioi_set_sw_coreisr(s); + loongarch_ext_irq_unlock(s, flags); + } + + return 0; +} + +static int kvm_loongarch_extioi_get_attr(struct kvm_device *dev, + struct kvm_device_attr *attr) +{ + if (attr->group == KVM_DEV_LOONGARCH_EXTIOI_GRP_REGS) + return kvm_loongarch_extioi_regs_access(dev, attr, false); + + return -EINVAL; +} + +static int kvm_loongarch_extioi_set_attr(struct kvm_device *dev, + struct kvm_device_attr *attr) +{ + if (attr->group == KVM_DEV_LOONGARCH_EXTIOI_GRP_REGS) + return kvm_loongarch_extioi_regs_access(dev, attr, true); + + return -EINVAL; +} + +static void kvm_loongarch_extioi_destroy(struct kvm_device *dev) +{ + struct kvm *kvm; + struct loongarch_extioi *extioi; + struct kvm_io_device *device; + + if (!dev) + return; + + kvm = dev->kvm; + if (!kvm) + return; + + extioi = kvm->arch.extioi; + if (!extioi) + return; + + device = &extioi->device; + kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, device); + kfree(extioi); +} + +static int kvm_loongarch_extioi_create(struct kvm_device *dev, u32 type) +{ + int ret; + struct loongarch_extioi *s; + struct kvm_io_device *device; + struct kvm *kvm = dev->kvm; + + /* extioi has been created */ + if (kvm->arch.extioi) + return -EINVAL; + + s = kzalloc(sizeof(struct loongarch_extioi), GFP_KERNEL); + if (!s) + return -ENOMEM; + spin_lock_init(&s->lock); + s->kvm = kvm; + + /* + * Initialize IOCSR device + */ + device = &s->device; + kvm_iodevice_init(device, &kvm_loongarch_extioi_ops); + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, EXTIOI_BASE, EXTIOI_SIZE, device); + mutex_unlock(&kvm->slots_lock); + if (ret < 0) { + kfree(s); + return -EFAULT; + } + + kvm->arch.extioi = s; + + kvm_info("create extioi device successfully\n"); + return 0; +} + +static struct kvm_device_ops kvm_loongarch_extioi_dev_ops = { + .name = "kvm-loongarch-extioi", + .create = kvm_loongarch_extioi_create, + .destroy = kvm_loongarch_extioi_destroy, + .set_attr = kvm_loongarch_extioi_set_attr, + .get_attr = kvm_loongarch_extioi_get_attr, +}; + +int kvm_loongarch_register_extioi_device(void) +{ + return kvm_register_device_ops(&kvm_loongarch_extioi_dev_ops, + KVM_DEV_TYPE_LA_EXTIOI); +} + +int kvm_loongarch_reset_extioi(struct kvm *kvm) +{ + struct loongarch_extioi *extioi = kvm->arch.extioi; + unsigned long flags; + u8 offset, size; + u8 *pstart; + + if (!extioi) + return -EINVAL; + + pstart = (char *)&extioi->nodetype; + offset = (char *)&extioi->nodetype - (char *)extioi; + size = sizeof(struct loongarch_extioi) - offset; + + loongarch_ext_irq_lock(extioi, flags); + memset(pstart, 0, size); + loongarch_ext_irq_unlock(extioi, flags); + + return 0; +} diff --git a/arch/loongarch/kvm/intc/ipi.c b/arch/loongarch/kvm/intc/ipi.c new file mode 100644 index 0000000000000000000000000000000000000000..12024d9fdd0b5a751474cedbc45bab6bf624c760 --- /dev/null +++ b/arch/loongarch/kvm/intc/ipi.c @@ -0,0 +1,538 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#include +#include +#include + +static void ipi_send(struct kvm *kvm, uint64_t data) +{ + struct kvm_vcpu *vcpu; + struct kvm_interrupt irq; + int cpu, action, status; + + cpu = ((data & 0xffffffff) >> 16) & 0x3ff; + vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); + if (unlikely(vcpu == NULL)) { + kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); + return; + } + + action = 1 << (data & 0x1f); + + spin_lock(&vcpu->arch.ipi_state.lock); + status = vcpu->arch.ipi_state.status; + vcpu->arch.ipi_state.status |= action; + if (status == 0) { + irq.irq = LARCH_INT_IPI; + kvm_vcpu_ioctl_interrupt(vcpu, &irq); + } + spin_unlock(&vcpu->arch.ipi_state.lock); +} + +static void ipi_clear(struct kvm_vcpu *vcpu, uint64_t data) +{ + struct kvm_interrupt irq; + + spin_lock(&vcpu->arch.ipi_state.lock); + vcpu->arch.ipi_state.status &= ~data; + if (!vcpu->arch.ipi_state.status) { + irq.irq = -LARCH_INT_IPI; + kvm_vcpu_ioctl_interrupt(vcpu, &irq); + } + spin_unlock(&vcpu->arch.ipi_state.lock); +} + +static uint64_t read_mailbox(struct kvm_vcpu *vcpu, int offset, int len) +{ + void *pbuf; + uint64_t ret = 0; + + spin_lock(&vcpu->arch.ipi_state.lock); + pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20); + if (len == 1) + ret = *(unsigned char *)pbuf; + else if (len == 2) + ret = *(unsigned short *)pbuf; + else if (len == 4) + ret = *(unsigned int *)pbuf; + else if (len == 8) + ret = *(unsigned long *)pbuf; + else + kvm_err("%s: unknown data len: %d\n", __func__, len); + spin_unlock(&vcpu->arch.ipi_state.lock); + + return ret; +} + +static void write_mailbox(struct kvm_vcpu *vcpu, int offset, + uint64_t data, int len) +{ + void *pbuf; + + spin_lock(&vcpu->arch.ipi_state.lock); + pbuf = (void *)vcpu->arch.ipi_state.buf + (offset - 0x20); + if (len == 1) + *(unsigned char *)pbuf = (unsigned char)data; + else if (len == 2) + *(unsigned short *)pbuf = (unsigned short)data; + else if (len == 4) + *(unsigned int *)pbuf = (unsigned int)data; + else if (len == 8) + *(unsigned long *)pbuf = (unsigned long)data; + else + kvm_err("%s: unknown data len: %d\n", __func__, len); + spin_unlock(&vcpu->arch.ipi_state.lock); +} + +static int loongarch_ipi_writel(struct kvm_vcpu *vcpu, gpa_t addr, + int len, const void *val) +{ + uint64_t data; + uint32_t offset; + int ret = 0; + + data = *(uint64_t *)val; + + offset = (uint32_t)(addr & 0xff); + WARN_ON_ONCE(offset & (len - 1)); + + switch (offset) { + case CORE_STATUS_OFF: + kvm_err("CORE_SET_OFF Can't be write\n"); + ret = -EINVAL; + break; + case CORE_EN_OFF: + spin_lock(&vcpu->arch.ipi_state.lock); + vcpu->arch.ipi_state.en = data; + spin_unlock(&vcpu->arch.ipi_state.lock); + break; + case IOCSR_IPI_SEND: + ipi_send(vcpu->kvm, data); + break; + case CORE_SET_OFF: + kvm_info("CORE_SET_OFF simulation is required\n"); + ret = -EINVAL; + break; + case CORE_CLEAR_OFF: + /* Just clear the status of the current vcpu */ + ipi_clear(vcpu, data); + break; + case CORE_BUF_20 ... CORE_BUF_38 + 7: + if (offset + len > CORE_BUF_38 + 8) { + kvm_err("%s: invalid offset or len: offset = %d, len = %d\n", + __func__, offset, len); + ret = -EINVAL; + break; + } + write_mailbox(vcpu, offset, data, len); + break; + default: + kvm_err("%s: unknown addr: %llx\n", __func__, addr); + ret = -EINVAL; + break; + } + + return ret; +} + +static int loongarch_ipi_readl(struct kvm_vcpu *vcpu, gpa_t addr, + int len, void *val) +{ + uint32_t offset; + uint64_t res = 0; + int ret = 0; + + offset = (uint32_t)(addr & 0xff); + WARN_ON_ONCE(offset & (len - 1)); + + switch (offset) { + case CORE_STATUS_OFF: + spin_lock(&vcpu->arch.ipi_state.lock); + res = vcpu->arch.ipi_state.status; + spin_unlock(&vcpu->arch.ipi_state.lock); + break; + case CORE_EN_OFF: + spin_lock(&vcpu->arch.ipi_state.lock); + res = vcpu->arch.ipi_state.en; + spin_unlock(&vcpu->arch.ipi_state.lock); + break; + case CORE_SET_OFF: + res = 0; + break; + case CORE_CLEAR_OFF: + res = 0; + break; + case CORE_BUF_20 ... CORE_BUF_38 + 7: + if (offset + len > CORE_BUF_38 + 8) { + kvm_err("%s: invalid offset or len: offset = %d, len = %d\n", + __func__, offset, len); + ret = -EINVAL; + break; + } + res = read_mailbox(vcpu, offset, len); + break; + default: + kvm_err("%s: unknown addr: %llx\n", __func__, addr); + ret = -EINVAL; + break; + } + + *(uint64_t *)val = res; + + return ret; +} + +static int kvm_loongarch_ipi_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + struct loongarch_ipi *ipi; + int ret; + + ipi = vcpu->kvm->arch.ipi; + if (!ipi) { + kvm_err("%s: ipi irqchip not valid!\n", __func__); + return -EINVAL; + } + + ipi->kvm->stat.ipi_write_exits++; + ret = loongarch_ipi_writel(vcpu, addr, len, val); + + return ret; +} + +static int kvm_loongarch_ipi_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, void *val) +{ + struct loongarch_ipi *ipi; + int ret; + + ipi = vcpu->kvm->arch.ipi; + if (!ipi) { + kvm_err("%s: ipi irqchip not valid!\n", __func__); + return -EINVAL; + } + + ipi->kvm->stat.ipi_read_exits++; + ret = loongarch_ipi_readl(vcpu, addr, len, val); + + return ret; +} + +static int send_ipi_data(struct kvm_vcpu *vcpu, gpa_t addr, uint64_t data) +{ + int i, ret; + uint32_t val = 0, mask = 0; + /* + * Bit 27-30 is mask for byte writing. + * If the mask is 0, we need not to do anything. + */ + if ((data >> 27) & 0xf) { + /* Read the old val */ + ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, sizeof(val), &val); + if (unlikely(ret)) { + kvm_err("%s: : read date from addr %llx failed\n", __func__, addr); + return ret; + } + /* Construct the mask by scanning the bit 27-30 */ + for (i = 0; i < 4; i++) { + if (data & (0x1 << (27 + i))) + mask |= (0xff << (i * 8)); + } + /* Save the old part of val */ + val &= mask; + } + + val |= ((uint32_t)(data >> 32) & ~mask); + ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, sizeof(val), &val); + if (unlikely(ret)) + kvm_err("%s: : write date to addr %llx failed\n", __func__, addr); + + return ret; +} + +static int mail_send(struct kvm *kvm, uint64_t data) +{ + struct kvm_vcpu *vcpu; + int cpu, mailbox; + int offset, ret; + + cpu = ((data & 0xffffffff) >> 16) & 0x3ff; + vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); + if (unlikely(vcpu == NULL)) { + kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); + return -EINVAL; + } + + mailbox = ((data & 0xffffffff) >> 2) & 0x7; + offset = SMP_MAILBOX + CORE_BUF_20 + mailbox * 4; + ret = send_ipi_data(vcpu, offset, data); + + return ret; +} + +static int any_send(struct kvm *kvm, uint64_t data) +{ + struct kvm_vcpu *vcpu; + int cpu, offset, ret; + + cpu = ((data & 0xffffffff) >> 16) & 0x3ff; + vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); + if (unlikely(vcpu == NULL)) { + kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); + return -EINVAL; + } + + offset = data & 0xffff; + ret = send_ipi_data(vcpu, offset, data); + return ret; +} + +static int kvm_loongarch_mail_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + struct loongarch_ipi *ipi; + int ret; + + ipi = vcpu->kvm->arch.ipi; + if (!ipi) { + kvm_err("%s: ipi irqchip not valid!\n", __func__); + return -EINVAL; + } + + addr &= 0xfff; + addr -= IOCSR_MAIL_SEND; + + switch (addr) { + case MAIL_SEND_OFFSET: + ret = mail_send(vcpu->kvm, *(uint64_t *)val); + break; + case ANY_SEND_OFFSET: + ret = any_send(vcpu->kvm, *(uint64_t *)val); + break; + default: + kvm_err("%s: invalid addr %llx!\n", __func__, addr); + ret = -EINVAL; + break; + } + + return ret; +} + +static const struct kvm_io_device_ops kvm_loongarch_ipi_ops = { + .read = kvm_loongarch_ipi_read, + .write = kvm_loongarch_ipi_write, +}; + +static const struct kvm_io_device_ops kvm_loongarch_mail_ops = { + .write = kvm_loongarch_mail_write, +}; + +static int kvm_loongarch_ipi_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + bool is_write) +{ + uint64_t val; + int cpu, addr; + void *p = NULL; + int len = 4; + struct kvm_vcpu *vcpu; + + cpu = (attr->attr >> 16) & 0x3ff; + addr = attr->attr & 0xff; + + vcpu = kvm_get_vcpu(dev->kvm, cpu); + if (unlikely(vcpu == NULL)) { + kvm_err("%s: invalid target cpu: %d\n", __func__, cpu); + return -EINVAL; + } + switch (addr) { + case CORE_STATUS_OFF: + p = &vcpu->arch.ipi_state.status; + break; + case CORE_EN_OFF: + p = &vcpu->arch.ipi_state.en; + break; + case CORE_SET_OFF: + p = &vcpu->arch.ipi_state.set; + break; + case CORE_CLEAR_OFF: + p = &vcpu->arch.ipi_state.clear; + break; + case CORE_BUF_20: + p = &vcpu->arch.ipi_state.buf[0]; + len = 8; + break; + case CORE_BUF_28: + p = &vcpu->arch.ipi_state.buf[1]; + len = 8; + break; + case CORE_BUF_30: + p = &vcpu->arch.ipi_state.buf[2]; + len = 8; + break; + case CORE_BUF_38: + p = &vcpu->arch.ipi_state.buf[3]; + len = 8; + break; + default: + kvm_err("%s: unknown ipi register, addr = %d\n", __func__, addr); + return -EINVAL; + } + + if (is_write) { + if (len == 4) { + if (get_user(val, (uint32_t __user *)attr->addr)) + return -EFAULT; + *(uint32_t *)p = (uint32_t)val; + } else if (len == 8) { + if (get_user(val, (uint64_t __user *)attr->addr)) + return -EFAULT; + *(uint64_t *)p = val; + } + } else { + if (len == 4) { + val = *(uint32_t *)p; + return put_user(val, (uint32_t __user *)attr->addr); + } else if (len == 8) { + val = *(uint64_t *)p; + return put_user(val, (uint64_t __user *)attr->addr); + } + } + + return 0; +} + +static int kvm_loongarch_ipi_get_attr(struct kvm_device *dev, + struct kvm_device_attr *attr) +{ + switch (attr->group) { + case KVM_DEV_LOONGARCH_IPI_GRP_REGS: + return kvm_loongarch_ipi_regs_access(dev, attr, false); + default: + kvm_err("%s: unknown group (%d)\n", __func__, attr->group); + return -EINVAL; + } +} + +static int kvm_loongarch_ipi_set_attr(struct kvm_device *dev, + struct kvm_device_attr *attr) +{ + switch (attr->group) { + case KVM_DEV_LOONGARCH_IPI_GRP_REGS: + return kvm_loongarch_ipi_regs_access(dev, attr, true); + default: + kvm_err("%s: unknown group (%d)\n", __func__, attr->group); + return -EINVAL; + } +} + +static void kvm_loongarch_ipi_destroy(struct kvm_device *dev) +{ + struct kvm *kvm; + struct loongarch_ipi *ipi; + struct kvm_io_device *device; + + if (!dev) + return; + + kvm = dev->kvm; + if (!kvm) + return; + + ipi = kvm->arch.ipi; + if (!ipi) + return; + + device = &ipi->device; + kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, device); + + device = &ipi->mail_dev; + kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, device); + + kfree(ipi); +} + +static int kvm_loongarch_ipi_create(struct kvm_device *dev, u32 type) +{ + struct kvm *kvm; + struct loongarch_ipi *s; + unsigned long addr; + struct kvm_io_device *device; + int ret; + + kvm_info("begin create loongarch ipi in kvm ...\n"); + if (!dev) { + kvm_err("%s: kvm_device ptr is invalid!\n", __func__); + return -EINVAL; + } + + kvm = dev->kvm; + if (kvm->arch.ipi) { + kvm_err("%s: loongarch ipi has been created!\n", __func__); + return -EINVAL; + } + + s = kzalloc(sizeof(struct loongarch_ipi), GFP_KERNEL); + if (!s) + return -ENOMEM; + spin_lock_init(&s->lock); + s->kvm = kvm; + + /* + * Initialize IOCSR device + */ + device = &s->device; + kvm_iodevice_init(device, &kvm_loongarch_ipi_ops); + addr = SMP_MAILBOX; + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, addr, + KVM_IOCSR_IPI_ADDR_SIZE, device); + mutex_unlock(&kvm->slots_lock); + if (ret < 0) { + kvm_err("%s: initialize IOCSR dev failed, ret = %d\n", __func__, ret); + goto err; + } + + device = &s->mail_dev; + kvm_iodevice_init(device, &kvm_loongarch_mail_ops); + addr = MAIL_SEND_ADDR; + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, addr, + KVM_IOCSR_MAIL_ADDR_SIZE, device); + mutex_unlock(&kvm->slots_lock); + if (ret < 0) { + device = &s->device; + kvm_io_bus_unregister_dev(kvm, KVM_IOCSR_BUS, device); + kvm_err("%s: initialize mail box dev failed, ret = %d\n", __func__, ret); + goto err; + } + + kvm->arch.ipi = s; + kvm_info("create loongarch ipi in kvm done!\n"); + + return 0; + +err: + kfree(s); + return -EFAULT; +} + +static struct kvm_device_ops kvm_loongarch_ipi_dev_ops = { + .name = "kvm-loongarch-ipi", + .create = kvm_loongarch_ipi_create, + .destroy = kvm_loongarch_ipi_destroy, + .set_attr = kvm_loongarch_ipi_set_attr, + .get_attr = kvm_loongarch_ipi_get_attr, +}; + +int kvm_loongarch_register_ipi_device(void) +{ + return kvm_register_device_ops(&kvm_loongarch_ipi_dev_ops, + KVM_DEV_TYPE_LA_IPI); +} diff --git a/arch/loongarch/kvm/intc/pch_pic.c b/arch/loongarch/kvm/intc/pch_pic.c new file mode 100644 index 0000000000000000000000000000000000000000..7d053dbcd5c066829fa6c12ab15a6aac4d7957be --- /dev/null +++ b/arch/loongarch/kvm/intc/pch_pic.c @@ -0,0 +1,540 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include + +/* update the isr according to irq level and route irq to extioi */ +static void pch_pic_update_irq(struct loongarch_pch_pic *s, int irq, int level) +{ + u64 mask = (1 << irq); + + /* + * set isr and route irq to extioi and + * the route table is in htmsi_vector[] + */ + if (level) { + if (mask & s->irr & ~s->mask) { + s->isr |= mask; + irq = s->htmsi_vector[irq]; + extioi_set_irq(s->kvm->arch.extioi, irq, level); + } + } else { + if (mask & s->isr & ~s->irr) { + s->isr &= ~mask; + irq = s->htmsi_vector[irq]; + extioi_set_irq(s->kvm->arch.extioi, irq, level); + } + } +} + +/* msi irq handler */ +void pch_msi_set_irq(struct kvm *kvm, int irq, int level) +{ + extioi_set_irq(kvm->arch.extioi, irq, level); +} + +/* called when a irq is triggered in pch pic */ +void pch_pic_set_irq(struct loongarch_pch_pic *s, int irq, int level) +{ + u64 mask = (1 << irq); + + spin_lock(&s->lock); + if (level) + /* set irr */ + s->irr |= mask; + else { + /* 0 level signal in edge triggered irq does not mean to clear irq + * The irr register variable is cleared when the cpu writes to the + * PCH_PIC_CLEAR_START address area + */ + if (s->edge & mask) { + spin_unlock(&s->lock); + return; + } + s->irr &= ~mask; + } + pch_pic_update_irq(s, irq, level); + spin_unlock(&s->lock); +} + +/* update batch irqs, the irq_mask is a bitmap of irqs */ +static void pch_pic_update_batch_irqs(struct loongarch_pch_pic *s, u64 irq_mask, int level) +{ + int irq, bits; + + /* find each irq by irqs bitmap and update each irq */ + bits = sizeof(irq_mask) * 8; + irq = find_first_bit((void *)&irq_mask, bits); + while (irq < bits) { + pch_pic_update_irq(s, irq, level); + bitmap_clear((void *)&irq_mask, irq, 1); + irq = find_first_bit((void *)&irq_mask, bits); + } +} + +/* + * pch pic register is 64-bit, but it is accessed by 32-bit, + * so we use high to get whether low or high 32 bits we want + * to read. + */ +static u32 pch_pic_read_reg(u64 *s, int high) +{ + u64 val = *s; + + /* read the high 32 bits when the high is 1 */ + return high ? (u32)(val >> 32) : (u32)val; +} + +/* + * pch pic register is 64-bit, but it is accessed by 32-bit, + * so we use high to get whether low or high 32 bits we want + * to write. + */ +static u32 pch_pic_write_reg(u64 *s, int high, u32 v) +{ + u64 val = *s, data = v; + + if (high) { + /* + * Clear val high 32 bits + * write the high 32 bits when the high is 1 + */ + *s = (val << 32 >> 32) | (data << 32); + val >>= 32; + } else + /* + * Clear val low 32 bits + * write the low 32 bits when the high is 0 + */ + *s = (val >> 32 << 32) | v; + + return (u32)val; +} + +static int loongarch_pch_pic_write(struct loongarch_pch_pic *s, gpa_t addr, + int len, const void *val) +{ + u32 old, data, offset, index; + u64 irq; + int ret; + + ret = 0; + data = *(u32 *)val; + offset = addr - s->pch_pic_base; + + spin_lock(&s->lock); + switch (offset) { + case PCH_PIC_MASK_START ... PCH_PIC_MASK_END: + offset -= PCH_PIC_MASK_START; + /* get whether high or low 32 bits we want to write */ + index = offset >> 2; + old = pch_pic_write_reg(&s->mask, index, data); + + /* enable irq when mask value change to 0 */ + irq = (old & ~data) << (32 * index); + pch_pic_update_batch_irqs(s, irq, 1); + + /* disable irq when mask value change to 1 */ + irq = (~old & data) << (32 * index); + pch_pic_update_batch_irqs(s, irq, 0); + break; + case PCH_PIC_HTMSI_EN_START ... PCH_PIC_HTMSI_EN_END: + offset -= PCH_PIC_HTMSI_EN_START; + index = offset >> 2; + pch_pic_write_reg(&s->htmsi_en, index, data); + break; + case PCH_PIC_EDGE_START ... PCH_PIC_EDGE_END: + offset -= PCH_PIC_EDGE_START; + index = offset >> 2; + /* 1: edge triggered, 0: level triggered */ + pch_pic_write_reg(&s->edge, index, data); + break; + case PCH_PIC_CLEAR_START ... PCH_PIC_CLEAR_END: + offset -= PCH_PIC_CLEAR_START; + index = offset >> 2; + /* write 1 to clear edge irq */ + old = pch_pic_read_reg(&s->irr, index); + /* + * get the irq bitmap which is edge triggered and + * already set and to be cleared + */ + irq = old & pch_pic_read_reg(&s->edge, index) & data; + /* write irr to the new state where irqs have been cleared */ + pch_pic_write_reg(&s->irr, index, old & ~irq); + /* update cleared irqs */ + pch_pic_update_batch_irqs(s, irq, 0); + break; + case PCH_PIC_AUTO_CTRL0_START ... PCH_PIC_AUTO_CTRL0_END: + offset -= PCH_PIC_AUTO_CTRL0_START; + index = offset >> 2; + /* we only use default mode: fixed interrupt distribution mode */ + pch_pic_write_reg(&s->auto_ctrl0, index, 0); + break; + case PCH_PIC_AUTO_CTRL1_START ... PCH_PIC_AUTO_CTRL1_END: + offset -= PCH_PIC_AUTO_CTRL1_START; + index = offset >> 2; + /* we only use default mode: fixed interrupt distribution mode */ + pch_pic_write_reg(&s->auto_ctrl1, index, 0); + break; + case PCH_PIC_ROUTE_ENTRY_START ... PCH_PIC_ROUTE_ENTRY_END: + offset -= PCH_PIC_ROUTE_ENTRY_START; + /* only route to int0: extioi */ + s->route_entry[offset] = 1; + break; + case PCH_PIC_HTMSI_VEC_START ... PCH_PIC_HTMSI_VEC_END: + /* route table to extioi */ + offset -= PCH_PIC_HTMSI_VEC_START; + s->htmsi_vector[offset] = (u8)data; + break; + case PCH_PIC_POLARITY_START ... PCH_PIC_POLARITY_END: + offset -= PCH_PIC_POLARITY_START; + index = offset >> 2; + + /* we only use defalut value 0: high level triggered */ + pch_pic_write_reg(&s->polarity, index, 0); + break; + default: + ret = -EINVAL; + break; + } + + spin_unlock(&s->lock); + return ret; +} + +static int kvm_loongarch_pch_pic_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + int ret; + struct loongarch_pch_pic *s = vcpu->kvm->arch.pch_pic; + + if (!s) { + kvm_err("%s: pch pic irqchip not valid!\n", __func__); + return -EINVAL; + } + + /* statistics of pch pic writing */ + vcpu->kvm->stat.pch_pic_write_exits++; + ret = loongarch_pch_pic_write(s, addr, len, val); + + return ret; +} + +static int loongarch_pch_pic_read(struct loongarch_pch_pic *s, gpa_t addr, int len, void *val) +{ + int offset, index, ret = 0; + u32 data = 0; + u64 int_id = 0; + + offset = addr - s->pch_pic_base; + + spin_lock(&s->lock); + switch (offset) { + case PCH_PIC_INT_ID_START ... PCH_PIC_INT_ID_END: + /* int id version */ + int_id |= (u64)PCH_PIC_INT_ID_VER << 32; + /* irq number */ + int_id |= (u64)31 << (32 + 16); + /* int id value */ + int_id |= PCH_PIC_INT_ID_VAL; + *(u64 *)val = int_id; + break; + case PCH_PIC_MASK_START ... PCH_PIC_MASK_END: + offset -= PCH_PIC_MASK_START; + index = offset >> 2; + /* read mask reg */ + data = pch_pic_read_reg(&s->mask, index); + *(u32 *)val = data; + break; + case PCH_PIC_HTMSI_EN_START ... PCH_PIC_HTMSI_EN_END: + offset -= PCH_PIC_HTMSI_EN_START; + index = offset >> 2; + /* read htmsi enable reg */ + data = pch_pic_read_reg(&s->htmsi_en, index); + *(u32 *)val = data; + break; + case PCH_PIC_EDGE_START ... PCH_PIC_EDGE_END: + offset -= PCH_PIC_EDGE_START; + index = offset >> 2; + /* read edge enable reg */ + data = pch_pic_read_reg(&s->edge, index); + *(u32 *)val = data; + break; + case PCH_PIC_AUTO_CTRL0_START ... PCH_PIC_AUTO_CTRL0_END: + case PCH_PIC_AUTO_CTRL1_START ... PCH_PIC_AUTO_CTRL1_END: + /* we only use default mode: fixed interrupt distribution mode */ + *(u32 *)val = 0; + break; + case PCH_PIC_ROUTE_ENTRY_START ... PCH_PIC_ROUTE_ENTRY_END: + /* only route to int0: extioi */ + *(u8 *)val = 1; + break; + case PCH_PIC_HTMSI_VEC_START ... PCH_PIC_HTMSI_VEC_END: + offset -= PCH_PIC_HTMSI_VEC_START; + /* read htmsi vector */ + data = s->htmsi_vector[offset]; + *(u8 *)val = data; + break; + case PCH_PIC_POLARITY_START ... PCH_PIC_POLARITY_END: + /* we only use defalut value 0: high level triggered */ + *(u32 *)val = 0; + break; + default: + ret = -EINVAL; + } + spin_unlock(&s->lock); + return ret; +} + +static int kvm_loongarch_pch_pic_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, void *val) +{ + int ret; + struct loongarch_pch_pic *s = vcpu->kvm->arch.pch_pic; + + if (!s) { + kvm_err("%s: pch pic irqchip not valid!\n", __func__); + return -EINVAL; + } + + /* statistics of pch pic reading */ + vcpu->kvm->stat.pch_pic_read_exits++; + ret = loongarch_pch_pic_read(s, addr, len, val); + return ret; +} + +static const struct kvm_io_device_ops kvm_loongarch_pch_pic_ops = { + .read = kvm_loongarch_pch_pic_read, + .write = kvm_loongarch_pch_pic_write, +}; + +static int kvm_loongarch_pch_pic_init(struct kvm_device *dev, u64 addr) +{ + int ret; + struct loongarch_pch_pic *s = dev->kvm->arch.pch_pic; + struct kvm_io_device *device; + struct kvm *kvm = dev->kvm; + + s->pch_pic_base = addr; + device = &s->device; + /* init device by pch pic writing and reading ops */ + kvm_iodevice_init(device, &kvm_loongarch_pch_pic_ops); + mutex_lock(&kvm->slots_lock); + /* register pch pic device */ + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, addr, PCH_PIC_SIZE, device); + mutex_unlock(&kvm->slots_lock); + if (ret < 0) + return -EFAULT; + + return 0; +} + +/* used by user space to get or set pch pic registers */ +static int kvm_loongarch_pch_pic_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + bool is_write) +{ + int addr, len = 8, ret = 0; + void __user *data; + void *p = NULL; + struct loongarch_pch_pic *s; + + s = dev->kvm->arch.pch_pic; + addr = attr->attr; + data = (void __user *)attr->addr; + + spin_lock(&s->lock); + /* get pointer to pch pic register by addr */ + switch (addr) { + case PCH_PIC_MASK_START: + p = &s->mask; + break; + case PCH_PIC_HTMSI_EN_START: + p = &s->htmsi_en; + break; + case PCH_PIC_EDGE_START: + p = &s->edge; + break; + case PCH_PIC_AUTO_CTRL0_START: + p = &s->auto_ctrl0; + break; + case PCH_PIC_AUTO_CTRL1_START: + p = &s->auto_ctrl1; + break; + case PCH_PIC_ROUTE_ENTRY_START: + p = s->route_entry; + len = 64; + break; + case PCH_PIC_HTMSI_VEC_START: + p = s->htmsi_vector; + len = 64; + break; + case PCH_PIC_INT_IRR_START: + p = &s->irr; + break; + case PCH_PIC_INT_ISR_START: + p = &s->isr; + break; + case PCH_PIC_POLARITY_START: + p = &s->polarity; + break; + default: + ret = -EINVAL; + } + + /* write or read value according to is_write */ + if (is_write) { + if (copy_from_user(p, data, len)) + ret = -EFAULT; + } else { + if (copy_to_user(data, p, len)) + ret = -EFAULT; + } + + spin_unlock(&s->lock); + return ret; +} + +static int kvm_loongarch_pch_pic_get_attr(struct kvm_device *dev, + struct kvm_device_attr *attr) +{ + /* only support pch pic group registers */ + if (attr->group == KVM_DEV_LOONGARCH_PCH_PIC_GRP_REGS) + return kvm_loongarch_pch_pic_regs_access(dev, attr, false); + + return -EINVAL; +} + +static int kvm_loongarch_pch_pic_set_attr(struct kvm_device *dev, + struct kvm_device_attr *attr) +{ + int ret = -EINVAL; + u64 addr; + void __user *uaddr = (void __user *)(long)attr->addr; + + switch (attr->group) { + case KVM_DEV_LOONGARCH_PCH_PIC_GRP_CTRL: + switch (attr->attr) { + case KVM_DEV_LOONGARCH_PCH_PIC_CTRL_INIT: + if (copy_from_user(&addr, uaddr, sizeof(addr))) + return -EFAULT; + + if (!dev->kvm->arch.pch_pic) { + kvm_err("%s: please create pch_pic irqchip first!\n", __func__); + ret = -EFAULT; + break; + } + + ret = kvm_loongarch_pch_pic_init(dev, addr); + break; + default: + kvm_err("%s: unknown group (%d) attr (%lld)\n", __func__, attr->group, + attr->attr); + ret = -EINVAL; + break; + } + break; + case KVM_DEV_LOONGARCH_PCH_PIC_GRP_REGS: + ret = kvm_loongarch_pch_pic_regs_access(dev, attr, true); + break; + default: + break; + } + + return ret; +} + +static int kvm_setup_default_irq_routing(struct kvm *kvm) +{ + struct kvm_irq_routing_entry *entries; + + u32 nr = KVM_IRQCHIP_NUM_PINS; + int i, ret; + + entries = kcalloc(nr, sizeof(*entries), GFP_KERNEL); + if (!entries) + return -ENOMEM; + + for (i = 0; i < nr; i++) { + entries[i].gsi = i; + entries[i].type = KVM_IRQ_ROUTING_IRQCHIP; + entries[i].u.irqchip.irqchip = 0; + entries[i].u.irqchip.pin = i; + } + ret = kvm_set_irq_routing(kvm, entries, nr, 0); + kfree(entries); + + return 0; +} + +static void kvm_loongarch_pch_pic_destroy(struct kvm_device *dev) +{ + struct kvm *kvm; + struct loongarch_pch_pic *s; + struct kvm_io_device *device; + + if (!dev) + return; + + kvm = dev->kvm; + if (!kvm) + return; + + s = kvm->arch.pch_pic; + if (!s) + return; + + device = &s->device; + /* unregister pch pic device and free it's memory */ + kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, device); + kfree(s); +} + +static int kvm_loongarch_pch_pic_create(struct kvm_device *dev, u32 type) +{ + int ret; + struct loongarch_pch_pic *s; + struct kvm *kvm = dev->kvm; + + /* pch pic should not has been created */ + if (kvm->arch.pch_pic) + return -EINVAL; + + ret = kvm_setup_default_irq_routing(kvm); + if (ret) + return -ENOMEM; + + s = kzalloc(sizeof(struct loongarch_pch_pic), GFP_KERNEL); + if (!s) + return -ENOMEM; + + spin_lock_init(&s->lock); + s->kvm = kvm; + + + kvm->arch.pch_pic = s; + + kvm_info("create pch pic device successfully\n"); + return 0; +} + +static struct kvm_device_ops kvm_loongarch_pch_pic_dev_ops = { + .name = "kvm-loongarch-pch-pic", + .create = kvm_loongarch_pch_pic_create, + .destroy = kvm_loongarch_pch_pic_destroy, + .set_attr = kvm_loongarch_pch_pic_set_attr, + .get_attr = kvm_loongarch_pch_pic_get_attr, +}; + +int kvm_loongarch_register_pch_pic_device(void) +{ + return kvm_register_device_ops(&kvm_loongarch_pch_pic_dev_ops, + KVM_DEV_TYPE_LA_IOAPIC); +} diff --git a/arch/loongarch/kvm/irqfd.c b/arch/loongarch/kvm/irqfd.c new file mode 100644 index 0000000000000000000000000000000000000000..bf67f329ebc962cc6b7e9320074c998e99713921 --- /dev/null +++ b/arch/loongarch/kvm/irqfd.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#include +#include +#include + +static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + /* ioapic pin (0 ~ 64) <---> gsi(0 ~ 64) */ + pch_pic_set_irq(kvm->arch.pch_pic, e->irqchip.pin, level); + + return 0; +} + +/* + * kvm_set_routing_entry: populate a kvm routing entry + * from a user routing entry + * + * @kvm: the VM this entry is applied to + * @e: kvm kernel routing entry handle + * @ue: user api routing entry handle + * return 0 on success, -EINVAL on errors. + */ +int kvm_set_routing_entry(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue) +{ + int r = -EINVAL; + + switch (ue->type) { + case KVM_IRQ_ROUTING_IRQCHIP: + e->set = kvm_set_ioapic_irq; + + e->irqchip.irqchip = ue->u.irqchip.irqchip; + e->irqchip.pin = ue->u.irqchip.pin; + + if (e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) + goto out; + break; + case KVM_IRQ_ROUTING_MSI: + e->set = kvm_set_msi; + e->msi.address_lo = ue->u.msi.address_lo; + e->msi.address_hi = ue->u.msi.address_hi; + e->msi.data = ue->u.msi.data; + break; + default: + goto out; + } + r = 0; +out: + return r; +} + +int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + if (e->type == KVM_IRQ_ROUTING_MSI) { + pch_msi_set_irq(kvm, e->msi.data, 1); + return 0; + } + + return -EWOULDBLOCK; +} + +/** + * kvm_set_msi: inject the MSI corresponding to the + * MSI routing entry + * + * This is the entry point for irqfd MSI injection + * and userspace MSI injection. + */ +int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + if (!level) + return -1; + + pch_msi_set_irq(kvm, e->msi.data, level); + return 0; +} diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 844736b99d387466f48b75fa49ca22ab03bb27d5..1f50f6723739cde230b961ed429cee62b693db44 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "trace.h" unsigned long vpid_mask; @@ -313,7 +315,7 @@ void kvm_arch_hardware_disable(void) static int kvm_loongarch_env_init(void) { - int cpu, order; + int cpu, order, ret; void *addr; struct kvm_context *context; @@ -368,7 +370,20 @@ static int kvm_loongarch_env_init(void) kvm_init_gcsr_flag(); - return 0; + /* Register loongarch ipi interrupt controller interface. */ + ret = kvm_loongarch_register_ipi_device(); + if (ret) + return ret; + + /* Register loongarch extioi interrupt controller interface. */ + ret = kvm_loongarch_register_extioi_device(); + if (ret) + return ret; + + /* Register loongarch pch pic interrupt controller interface. */ + ret = kvm_loongarch_register_pch_pic_device(); + + return ret; } static void kvm_loongarch_env_exit(void) diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S index 80e988985a6adfa97cce7e92d1ebdef4b0cc1ca4..0c292f81849277178110a94f09e6b37e90d0dae6 100644 --- a/arch/loongarch/kvm/switch.S +++ b/arch/loongarch/kvm/switch.S @@ -277,6 +277,10 @@ SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest) #ifdef CONFIG_CPU_HAS_LBT STACK_FRAME_NON_STANDARD kvm_restore_fpu +#ifdef CONFIG_CPU_HAS_LSX STACK_FRAME_NON_STANDARD kvm_restore_lsx +#endif +#ifdef CONFIG_CPU_HAS_LASX STACK_FRAME_NON_STANDARD kvm_restore_lasx #endif +#endif diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c index bcc6b6d063d914dbf820b43f2c1308803646b395..74a4b5c272d60e99523e12e89d5e663d53009c2b 100644 --- a/arch/loongarch/kvm/timer.c +++ b/arch/loongarch/kvm/timer.c @@ -188,10 +188,3 @@ void kvm_save_timer(struct kvm_vcpu *vcpu) kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ESTAT); preempt_enable(); } - -void kvm_reset_timer(struct kvm_vcpu *vcpu) -{ - write_gcsr_timercfg(0); - kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_TCFG, 0); - hrtimer_cancel(&vcpu->arch.swtimer); -} diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index bdfac8a614120684089dbcea3b84db476fc9ca7d..50bd40d36eb9e5eecf0c522180ff7b4b33c0cb57 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -31,63 +32,19 @@ const struct kvm_stats_header kvm_vcpu_stats_header = { sizeof(kvm_vcpu_stats_desc), }; -static void kvm_update_stolen_time(struct kvm_vcpu *vcpu) -{ - u32 version; - u64 steal; - gpa_t gpa; - struct kvm_memslots *slots; - struct kvm_steal_time __user *st; - struct gfn_to_hva_cache *ghc; - - ghc = &vcpu->arch.st.cache; - gpa = vcpu->arch.st.guest_addr; - if (!(gpa & KVM_STEAL_PHYS_VALID)) - return; - - gpa &= KVM_STEAL_PHYS_MASK; - slots = kvm_memslots(vcpu->kvm); - if (slots->generation != ghc->generation || gpa != ghc->gpa) { - if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) { - ghc->gpa = INVALID_GPA; - return; - } - } - - st = (struct kvm_steal_time __user *)ghc->hva; - unsafe_get_user(version, &st->version, out); - if (version & 1) - version += 1; /* first time write, random junk */ - - version += 1; - unsafe_put_user(version, &st->version, out); - smp_wmb(); - - unsafe_get_user(steal, &st->steal, out); - steal += current->sched_info.run_delay - vcpu->arch.st.last_steal; - vcpu->arch.st.last_steal = current->sched_info.run_delay; - unsafe_put_user(steal, &st->steal, out); - - smp_wmb(); - version += 1; - unsafe_put_user(version, &st->version, out); -out: - mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); -} - static inline void kvm_save_host_pmu(struct kvm_vcpu *vcpu) { struct kvm_context *context; context = this_cpu_ptr(vcpu->kvm->arch.vmcs); - context->perf_ctrl[0] = write_csr_perfctrl0(0); - context->perf_ctrl[1] = write_csr_perfctrl1(0); - context->perf_ctrl[2] = write_csr_perfctrl2(0); - context->perf_ctrl[3] = write_csr_perfctrl3(0); context->perf_cntr[0] = read_csr_perfcntr0(); context->perf_cntr[1] = read_csr_perfcntr1(); context->perf_cntr[2] = read_csr_perfcntr2(); context->perf_cntr[3] = read_csr_perfcntr3(); + context->perf_ctrl[0] = write_csr_perfctrl0(0); + context->perf_ctrl[1] = write_csr_perfctrl1(0); + context->perf_ctrl[2] = write_csr_perfctrl2(0); + context->perf_ctrl[3] = write_csr_perfctrl3(0); } static inline void kvm_restore_host_pmu(struct kvm_vcpu *vcpu) @@ -105,18 +62,19 @@ static inline void kvm_restore_host_pmu(struct kvm_vcpu *vcpu) write_csr_perfctrl3(context->perf_ctrl[3]); } + static inline void kvm_save_guest_pmu(struct kvm_vcpu *vcpu) { struct loongarch_csrs *csr = vcpu->arch.csr; - kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); - kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); - kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); - kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR0); kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR1); kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR2); kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR3); + kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); + kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); + kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); + kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); } static inline void kvm_restore_guest_pmu(struct kvm_vcpu *vcpu) @@ -133,70 +91,109 @@ static inline void kvm_restore_guest_pmu(struct kvm_vcpu *vcpu) kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); } +static int kvm_own_pmu(struct kvm_vcpu *vcpu) +{ + unsigned long val; + + if (!kvm_guest_has_pmu(&vcpu->arch)) + return -EINVAL; + + kvm_save_host_pmu(vcpu); + + /* Set PM0-PM(num) to guest */ + val = read_csr_gcfg() & ~CSR_GCFG_GPERF; + val |= (kvm_get_pmu_num(&vcpu->arch) + 1) << CSR_GCFG_GPERF_SHIFT; + write_csr_gcfg(val); + + kvm_restore_guest_pmu(vcpu); + + return 0; +} + static void kvm_lose_pmu(struct kvm_vcpu *vcpu) { unsigned long val; struct loongarch_csrs *csr = vcpu->arch.csr; - if (!(vcpu->arch.aux_inuse & KVM_GUEST_PMU_ENABLE)) - return; - if (!(vcpu->arch.aux_inuse & KVM_GUEST_PMU_ACTIVE)) + if (!(vcpu->arch.aux_inuse & KVM_LARCH_PMU)) return; kvm_save_guest_pmu(vcpu); + /* Disable pmu access from guest */ write_csr_gcfg(read_csr_gcfg() & ~CSR_GCFG_GPERF); /* - * Clear KVM_GUEST_PMU_ENABLE if the guest is not using PMU CSRs - * when exiting the guest, so that the next time trap into the guest. - * we don't need to deal with PMU CSRs contexts. + * Clear KVM_LARCH_PMU if the guest is not using PMU CSRs when + * exiting the guest, so that the next time trap into the guest. + * We don't need to deal with PMU CSRs contexts. */ val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); if (!(val & KVM_PMU_EVENT_ENABLED)) - vcpu->arch.aux_inuse &= ~KVM_GUEST_PMU_ENABLE; - kvm_restore_host_pmu(vcpu); + vcpu->arch.aux_inuse &= ~KVM_LARCH_PMU; - /* KVM_GUEST_PMU_ACTIVE needs to be cleared when exiting the guest */ - vcpu->arch.aux_inuse &= ~KVM_GUEST_PMU_ACTIVE; + kvm_restore_host_pmu(vcpu); } -static void kvm_own_pmu(struct kvm_vcpu *vcpu) +static void kvm_restore_pmu(struct kvm_vcpu *vcpu) { - unsigned long val; - - kvm_save_host_pmu(vcpu); - /* Set PM0-PM(num) to guest */ - val = read_csr_gcfg() & ~CSR_GCFG_GPERF; - val |= (kvm_get_pmu_num(&vcpu->arch) + 1) << CSR_GCFG_GPERF_SHIFT; - write_csr_gcfg(val); - kvm_restore_guest_pmu(vcpu); + if ((vcpu->arch.aux_inuse & KVM_LARCH_PMU)) + kvm_make_request(KVM_REQ_PMU, vcpu); } -static void kvm_restore_pmu(struct kvm_vcpu *vcpu) +static void kvm_check_pmu(struct kvm_vcpu *vcpu) { - if (!(vcpu->arch.aux_inuse & KVM_GUEST_PMU_ENABLE)) - return; - - kvm_make_request(KVM_REQ_PMU, vcpu); + if (kvm_check_request(KVM_REQ_PMU, vcpu)) { + kvm_own_pmu(vcpu); + vcpu->arch.aux_inuse |= KVM_LARCH_PMU; + } } -static void kvm_check_pmu(struct kvm_vcpu *vcpu) +static void kvm_update_stolen_time(struct kvm_vcpu *vcpu) { - if (!kvm_check_request(KVM_REQ_PMU, vcpu)) + u32 version; + u64 steal; + gpa_t gpa; + struct kvm_memslots *slots; + struct kvm_steal_time __user *st; + struct gfn_to_hva_cache *ghc; + + ghc = &vcpu->arch.st.cache; + gpa = vcpu->arch.st.guest_addr; + if (!(gpa & KVM_STEAL_PHYS_VALID)) return; - kvm_own_pmu(vcpu); + gpa &= KVM_STEAL_PHYS_MASK; + slots = kvm_memslots(vcpu->kvm); + if (slots->generation != ghc->generation || gpa != ghc->gpa) { + if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) { + ghc->gpa = INVALID_GPA; + return; + } + } - /* - * Set KVM_GUEST PMU_ENABLE and GUEST_PMU_ACTIVE - * when guest has KVM_REQ_PMU request. - */ - vcpu->arch.aux_inuse |= KVM_GUEST_PMU_ENABLE; - vcpu->arch.aux_inuse |= KVM_GUEST_PMU_ACTIVE; + st = (struct kvm_steal_time __user *)ghc->hva; + unsafe_get_user(version, &st->version, out); + if (version & 1) + version += 1; /* first time write, random junk */ + + version += 1; + unsafe_put_user(version, &st->version, out); + smp_wmb(); + + unsafe_get_user(steal, &st->steal, out); + steal += current->sched_info.run_delay - vcpu->arch.st.last_steal; + vcpu->arch.st.last_steal = current->sched_info.run_delay; + unsafe_put_user(steal, &st->steal, out); + + smp_wmb(); + version += 1; + unsafe_put_user(version, &st->version, out); +out: + mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); } /* @@ -602,10 +599,11 @@ static int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 val) if (id >= LOONGARCH_CSR_PERFCTRL0 && id <= LOONGARCH_CSR_PERFCNTR3) { unsigned long val; - val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); - val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); - val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); - val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); + val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0) | + kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1) | + kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2) | + kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); + if (val & KVM_PMU_EVENT_ENABLED) kvm_make_request(KVM_REQ_PMU, vcpu); } @@ -639,6 +637,12 @@ static int _kvm_get_cpucfg_mask(int id, u64 *v) *v |= CPUCFG2_LSX; if (cpu_has_lasx) *v |= CPUCFG2_LASX; + if (cpu_has_lbt_x86) + *v |= CPUCFG2_X86BT; + if (cpu_has_lbt_arm) + *v |= CPUCFG2_ARMBT; + if (cpu_has_lbt_mips) + *v |= CPUCFG2_MIPSBT; return 0; case LOONGARCH_CPUCFG3: @@ -672,7 +676,7 @@ static int _kvm_get_cpucfg_mask(int id, u64 *v) static int kvm_check_cpucfg(int id, u64 val) { - int ret, host; + int ret; u64 mask = 0; ret = _kvm_get_cpucfg_mask(id, &mask); @@ -700,9 +704,8 @@ static int kvm_check_cpucfg(int id, u64 val) return 0; case LOONGARCH_CPUCFG6: if (val & CPUCFG6_PMP) { - host = read_cpucfg(LOONGARCH_CPUCFG6); + u32 host = read_cpucfg(LOONGARCH_CPUCFG6); if ((val & CPUCFG6_PMBITS) != (host & CPUCFG6_PMBITS)) - /* Guest pmbits must be the same with host */ return -EINVAL; if ((val & CPUCFG6_PMNUM) > (host & CPUCFG6_PMNUM)) return -EINVAL; @@ -737,6 +740,34 @@ static int kvm_get_one_reg(struct kvm_vcpu *vcpu, else ret = -EINVAL; break; + case KVM_REG_LOONGARCH_LBT: + if (!kvm_guest_has_lbt(&vcpu->arch)) + return -ENXIO; + + switch (reg->id) { + case KVM_REG_LOONGARCH_LBT_SCR0: + *v = vcpu->arch.lbt.scr0; + break; + case KVM_REG_LOONGARCH_LBT_SCR1: + *v = vcpu->arch.lbt.scr1; + break; + case KVM_REG_LOONGARCH_LBT_SCR2: + *v = vcpu->arch.lbt.scr2; + break; + case KVM_REG_LOONGARCH_LBT_SCR3: + *v = vcpu->arch.lbt.scr3; + break; + case KVM_REG_LOONGARCH_LBT_EFLAGS: + *v = vcpu->arch.lbt.eflags; + break; + case KVM_REG_LOONGARCH_LBT_FTOP: + *v = vcpu->arch.fpu.ftop; + break; + default: + ret = -EINVAL; + break; + } + break; case KVM_REG_LOONGARCH_KVM: switch (reg->id) { case KVM_REG_LOONGARCH_COUNTER: @@ -795,9 +826,36 @@ static int kvm_set_one_reg(struct kvm_vcpu *vcpu, if (ret) break; vcpu->arch.cpucfg[id] = (u32)v; - if (id == LOONGARCH_CPUCFG6) { - vcpu->arch.max_pmu_csrid = LOONGARCH_CSR_PERFCTRL0 + - 2 * kvm_get_pmu_num(&vcpu->arch) + 1; + if (id == LOONGARCH_CPUCFG6) + vcpu->arch.max_pmu_csrid = + LOONGARCH_CSR_PERFCTRL0 + 2 * kvm_get_pmu_num(&vcpu->arch) + 1; + break; + case KVM_REG_LOONGARCH_LBT: + if (!kvm_guest_has_lbt(&vcpu->arch)) + return -ENXIO; + + switch (reg->id) { + case KVM_REG_LOONGARCH_LBT_SCR0: + vcpu->arch.lbt.scr0 = v; + break; + case KVM_REG_LOONGARCH_LBT_SCR1: + vcpu->arch.lbt.scr1 = v; + break; + case KVM_REG_LOONGARCH_LBT_SCR2: + vcpu->arch.lbt.scr2 = v; + break; + case KVM_REG_LOONGARCH_LBT_SCR3: + vcpu->arch.lbt.scr3 = v; + break; + case KVM_REG_LOONGARCH_LBT_EFLAGS: + vcpu->arch.lbt.eflags = v; + break; + case KVM_REG_LOONGARCH_LBT_FTOP: + vcpu->arch.fpu.ftop = v; + break; + default: + ret = -EINVAL; + break; } break; case KVM_REG_LOONGARCH_KVM: @@ -811,7 +869,9 @@ static int kvm_set_one_reg(struct kvm_vcpu *vcpu, vcpu->kvm->arch.time_offset = (signed long)(v - drdtime()); break; case KVM_REG_LOONGARCH_VCPU_RESET: - kvm_reset_timer(vcpu); + vcpu->arch.st.guest_addr = 0; + if (vcpu->vcpu_id == 0) + kvm_loongarch_reset_extioi(vcpu->kvm); memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending)); memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear)); break; @@ -895,6 +955,8 @@ static int kvm_loongarch_cpucfg_has_attr(struct kvm_vcpu *vcpu, case LOONGARCH_CPUCFG2: case LOONGARCH_CPUCFG6: return 0; + case CPUCFG_KVM_FEATURE: + return 0; default: return -ENXIO; } @@ -905,8 +967,8 @@ static int kvm_loongarch_cpucfg_has_attr(struct kvm_vcpu *vcpu, static int kvm_loongarch_pvtime_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) { - if (!kvm_pvtime_supported() || - attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) + if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME) + || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) return -ENXIO; return 0; @@ -938,9 +1000,18 @@ static int kvm_loongarch_cpucfg_get_attr(struct kvm_vcpu *vcpu, uint64_t val; uint64_t __user *uaddr = (uint64_t __user *)attr->addr; - ret = _kvm_get_cpucfg_mask(attr->attr, &val); - if (ret) - return ret; + switch (attr->attr) { + case 0 ... (KVM_MAX_CPUCFG_REGS - 1): + ret = _kvm_get_cpucfg_mask(attr->attr, &val); + if (ret) + return ret; + break; + case CPUCFG_KVM_FEATURE: + val = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK; + break; + default: + return -ENXIO; + } put_user(val, uaddr); @@ -953,8 +1024,8 @@ static int kvm_loongarch_pvtime_get_attr(struct kvm_vcpu *vcpu, u64 gpa; u64 __user *user = (u64 __user *)attr->addr; - if (!kvm_pvtime_supported() || - attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) + if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME) + || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) return -ENXIO; gpa = vcpu->arch.st.guest_addr; @@ -986,7 +1057,28 @@ static int kvm_loongarch_vcpu_get_attr(struct kvm_vcpu *vcpu, static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) { - return -ENXIO; + u64 val, valid; + u64 __user *user = (u64 __user *)attr->addr; + struct kvm *kvm = vcpu->kvm; + + switch (attr->attr) { + case CPUCFG_KVM_FEATURE: + if (get_user(val, user)) + return -EFAULT; + + valid = LOONGARCH_PV_FEAT_MASK; + if (val & ~valid) + return -EINVAL; + + /* All vCPUs need set the same PV features */ + if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED) + && ((kvm->arch.pv_features & valid) != val)) + return -EINVAL; + kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED; + return 0; + default: + return -ENXIO; + } } static int kvm_loongarch_pvtime_set_attr(struct kvm_vcpu *vcpu, @@ -996,8 +1088,8 @@ static int kvm_loongarch_pvtime_set_attr(struct kvm_vcpu *vcpu, u64 gpa, __user *user = (u64 __user *)attr->addr; struct kvm *kvm = vcpu->kvm; - if (!kvm_pvtime_supported() || - attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) + if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME) + || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) return -ENXIO; if (get_user(gpa, user)) @@ -1142,12 +1234,66 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) return 0; } +#ifdef CONFIG_CPU_HAS_LBT +int kvm_own_lbt(struct kvm_vcpu *vcpu) +{ + if (!kvm_guest_has_lbt(&vcpu->arch)) + return -EINVAL; + + preempt_disable(); + set_csr_euen(CSR_EUEN_LBTEN); + _restore_lbt(&vcpu->arch.lbt); + vcpu->arch.aux_inuse |= KVM_LARCH_LBT; + preempt_enable(); + + return 0; +} + +static void kvm_lose_lbt(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) { + _save_lbt(&vcpu->arch.lbt); + clear_csr_euen(CSR_EUEN_LBTEN); + vcpu->arch.aux_inuse &= ~KVM_LARCH_LBT; + } + preempt_enable(); +} + +static void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr) +{ + /* + * If TM is enabled, top register save/restore will + * cause lbt exception, here enable lbt in advance + */ + if (fcsr & FPU_CSR_TM) + kvm_own_lbt(vcpu); +} + +static void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { + if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) + return; + kvm_check_fcsr(vcpu, read_fcsr(LOONGARCH_FCSR0)); + } +} +#else +static inline void kvm_lose_lbt(struct kvm_vcpu *vcpu) { } +static inline void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr) { } +static inline void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu) { } +#endif + /* Enable FPU and restore context */ void kvm_own_fpu(struct kvm_vcpu *vcpu) { preempt_disable(); - /* Enable FPU */ + /* + * Enable FPU for guest + * Set FR and FRE according to guest context + */ + kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr); set_csr_euen(CSR_EUEN_FPEN); kvm_restore_fpu(&vcpu->arch.fpu); @@ -1167,6 +1313,7 @@ int kvm_own_lsx(struct kvm_vcpu *vcpu) preempt_disable(); /* Enable LSX for guest */ + kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr); set_csr_euen(CSR_EUEN_LSXEN | CSR_EUEN_FPEN); switch (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { case KVM_LARCH_FPU: @@ -1201,6 +1348,7 @@ int kvm_own_lasx(struct kvm_vcpu *vcpu) preempt_disable(); + kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr); set_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | KVM_LARCH_LSX)) { case KVM_LARCH_LSX: @@ -1232,6 +1380,7 @@ void kvm_lose_fpu(struct kvm_vcpu *vcpu) { preempt_disable(); + kvm_check_fcsr_alive(vcpu); if (vcpu->arch.aux_inuse & KVM_LARCH_LASX) { kvm_save_lasx(&vcpu->arch.fpu); vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU | KVM_LARCH_LASX); @@ -1254,6 +1403,7 @@ void kvm_lose_fpu(struct kvm_vcpu *vcpu) /* Disable FPU */ clear_csr_euen(CSR_EUEN_FPEN); } + kvm_lose_lbt(vcpu); preempt_enable(); } @@ -1327,6 +1477,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Init */ vcpu->arch.last_sched_cpu = -1; + /* Init ipi_state lock */ + spin_lock_init(&vcpu->arch.ipi_state.lock); + /* * Initialize guest register state to valid architectural reset state. */ diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c index 6b2e4f66ad26193aa043f01ce93941a4b4bae711..e7e43e6a49134631b514bb687cf1b6d70d15031f 100644 --- a/arch/loongarch/kvm/vm.c +++ b/arch/loongarch/kvm/vm.c @@ -5,6 +5,9 @@ #include #include +#include +#include +#include const struct _kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), @@ -39,6 +42,12 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) spin_lock_init(&kvm->arch.phyid_map_lock); kvm_init_vmcs(kvm); + + /* Enable all PV features by default */ + kvm->arch.pv_features = BIT(KVM_FEATURE_IPI); + if (kvm_pvtime_supported()) + kvm->arch.pv_features |= BIT(KVM_FEATURE_STEAL_TIME); + kvm->arch.gpa_size = BIT(cpu_vabits - 1); kvm->arch.root_level = CONFIG_PGTABLE_LEVELS - 1; kvm->arch.invalid_ptes[0] = 0; @@ -69,6 +78,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) int r; switch (ext) { + case KVM_CAP_IRQCHIP: case KVM_CAP_ONE_REG: case KVM_CAP_ENABLE_CAP: case KVM_CAP_READONLY_MEM: @@ -77,6 +87,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_IOEVENTFD: case KVM_CAP_MP_STATE: case KVM_CAP_SET_GUEST_DEBUG: + case KVM_CAP_VM_ATTRIBUTES: r = 1; break; case KVM_CAP_NR_VCPUS: @@ -99,7 +110,114 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) return r; } +static int kvm_vm_feature_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) +{ + switch (attr->attr) { + case KVM_LOONGARCH_VM_FEAT_LSX: + if (cpu_has_lsx) + return 0; + return -ENXIO; + case KVM_LOONGARCH_VM_FEAT_LASX: + if (cpu_has_lasx) + return 0; + return -ENXIO; + case KVM_LOONGARCH_VM_FEAT_X86BT: + if (cpu_has_lbt_x86) + return 0; + return -ENXIO; + case KVM_LOONGARCH_VM_FEAT_ARMBT: + if (cpu_has_lbt_arm) + return 0; + return -ENXIO; + case KVM_LOONGARCH_VM_FEAT_MIPSBT: + if (cpu_has_lbt_mips) + return 0; + return -ENXIO; + case KVM_LOONGARCH_VM_FEAT_PMU: + if (cpu_has_pmp) + return 0; + return -ENXIO; + case KVM_LOONGARCH_VM_FEAT_PV_IPI: + return 0; + case KVM_LOONGARCH_VM_FEAT_PV_STEALTIME: + if (kvm_pvtime_supported()) + return 0; + return -ENXIO; + default: + return -ENXIO; + } +} + +static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) +{ + switch (attr->group) { + case KVM_LOONGARCH_VM_FEAT_CTRL: + return kvm_vm_feature_has_attr(kvm, attr); + case KVM_LOONGARCH_VM_HAVE_IRQCHIP: + return 0; + default: + return -ENXIO; + } +} + int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { - return -ENOIOCTLCMD; + int r; + void __user *argp = (void __user *)arg; + struct kvm *kvm = filp->private_data; + struct kvm_device_attr attr; + + switch (ioctl) { + case KVM_CREATE_IRQCHIP: { + r = 1; + break; + } + case KVM_HAS_DEVICE_ATTR: { + if (copy_from_user(&attr, argp, sizeof(attr))) + return -EFAULT; + + return kvm_vm_has_attr(kvm, &attr); + } + default: + return -EINVAL; + } + + return r; +} + +int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *data, + bool line_status) +{ + bool level; + struct loongarch_pch_pic *s; + int type, vcpu, irq, vcpus, val, ret = 0; + + level = data->level; + val = data->irq; + s = kvm->arch.pch_pic; + vcpus = atomic_read(&kvm->online_vcpus); + + type = (val >> KVM_LOONGARCH_IRQ_TYPE_SHIFT) & KVM_LOONGARCH_IRQ_TYPE_MASK; + vcpu = (val >> KVM_LOONGARCH_IRQ_VCPU_SHIFT) & KVM_LOONGARCH_IRQ_VCPU_MASK; + irq = (val >> KVM_LOONGARCH_IRQ_NUM_SHIFT) & KVM_LOONGARCH_IRQ_NUM_MASK; + + switch (type) { + case KVM_LOONGARCH_IRQ_TYPE_IOAPIC: + if (irq < KVM_IRQCHIP_NUM_PINS) + pch_pic_set_irq(s, irq, level); + else if (irq < 256) + pch_msi_set_irq(kvm, irq, level); + else + ret = -EINVAL; + break; + default: + ret = -EINVAL; + } + + return ret; +} + +bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) +{ + return (bool)((!!kvm->arch.extioi) && (!!kvm->arch.pch_pic)); } diff --git a/arch/loongarch/power/suspend_asm.S b/arch/loongarch/power/suspend_asm.S index e2fc3b4e31f0019164f57de05c05324cc2bb391d..c28ad52b7bafba21fdaaae3e9aa42df36c0c0351 100644 --- a/arch/loongarch/power/suspend_asm.S +++ b/arch/loongarch/power/suspend_asm.S @@ -73,11 +73,7 @@ SYM_FUNC_START(loongarch_suspend_enter) * Reload all of the registers and return. */ SYM_INNER_LABEL(loongarch_wakeup_start, SYM_L_GLOBAL) - li.d t0, CSR_DMW0_INIT # UC, PLV0 - csrwr t0, LOONGARCH_CSR_DMWIN0 - li.d t0, CSR_DMW1_INIT # CA, PLV0 - csrwr t0, LOONGARCH_CSR_DMWIN1 - + SETUP_DMWINS t0 JUMP_VIRT_ADDR t0, t1 /* Enable PG */ diff --git a/drivers/firmware/efi/libstub/loongarch.c b/drivers/firmware/efi/libstub/loongarch.c index d0ef93551c44f64affb5a152b277a8b0c4e827fb..3782d0a187d1f50d91a803ccd5a57094204314ca 100644 --- a/drivers/firmware/efi/libstub/loongarch.c +++ b/drivers/firmware/efi/libstub/loongarch.c @@ -74,6 +74,8 @@ efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, /* Config Direct Mapping */ csr_write64(CSR_DMW0_INIT, LOONGARCH_CSR_DMWIN0); csr_write64(CSR_DMW1_INIT, LOONGARCH_CSR_DMWIN1); + csr_write64(CSR_DMW2_INIT, LOONGARCH_CSR_DMWIN2); + csr_write64(CSR_DMW3_INIT, LOONGARCH_CSR_DMWIN3); real_kernel_entry = (void *)kernel_entry_address(kernel_addr, image); diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c index 62cc8a598f06ee61ac818ffbb6db35f59ef0ee3c..dd7c2481c94f2785fc6229efe7139a2a4beffe64 100644 --- a/drivers/gpu/drm/radeon/cik.c +++ b/drivers/gpu/drm/radeon/cik.c @@ -8093,7 +8093,9 @@ int cik_irq_process(struct radeon_device *rdev) if (queue_thermal) schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; +#ifdef CONFIG_LOONGARCH WREG32(IH_RB_RPTR, rptr); +#endif atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c index 0dc5db59e95bfc5de8279afee245e97fcc176355..84ce0e5fc72a830f252814abfc1ea489f8e2a555 100644 --- a/drivers/gpu/drm/radeon/evergreen.c +++ b/drivers/gpu/drm/radeon/evergreen.c @@ -4922,7 +4922,9 @@ int evergreen_irq_process(struct radeon_device *rdev) if (queue_thermal && rdev->pm.dpm_enabled) schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; +#ifdef CONFIG_LOONGARCH WREG32(IH_RB_RPTR, rptr); +#endif atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c index 664b10b58c0f23d1891a13e4a0be70a88cb9aa48..43c1fde01708ddd2d25ca288a06cb2ec6ef77cb4 100644 --- a/drivers/gpu/drm/radeon/r600.c +++ b/drivers/gpu/drm/radeon/r600.c @@ -4328,7 +4328,9 @@ int r600_irq_process(struct radeon_device *rdev) if (queue_thermal && rdev->pm.dpm_enabled) schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; +#ifdef CONFIG_LOONGARCH WREG32(IH_RB_RPTR, rptr); +#endif atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c index 95cb2b4afb48c77f5531c4b0bf7132a56219c5a6..32c8803431a1efcbbe74024c0a2806b3f5e0714e 100644 --- a/drivers/gpu/drm/radeon/si.c +++ b/drivers/gpu/drm/radeon/si.c @@ -6442,7 +6442,9 @@ int si_irq_process(struct radeon_device *rdev) if (queue_thermal && rdev->pm.dpm_enabled) schedule_work(&rdev->pm.dpm.thermal.work); rdev->ih.rptr = rptr; +#ifdef CONFIG_LOONGARCH WREG32(IH_RB_RPTR, rptr); +#endif atomic_set(&rdev->ih.lock, 0); /* make sure wptr hasn't changed while processing */ diff --git a/drivers/irqchip/irq-loongarch-avec.c b/drivers/irqchip/irq-loongarch-avec.c index e8c43b3fd826695162156830e773c2d10ff9d23b..2f79a53e35a9ae96dbbeb9bb0fd557f563489f9f 100644 --- a/drivers/irqchip/irq-loongarch-avec.c +++ b/drivers/irqchip/irq-loongarch-avec.c @@ -30,6 +30,7 @@ struct pending_list { struct list_head head; }; +bool disable_pci_irq_limit; static struct cpumask intersect_mask; static DEFINE_PER_CPU(struct pending_list, pending_list); #endif @@ -83,7 +84,7 @@ static void avecintc_sync(struct avecintc_data *adata) plist = per_cpu_ptr(&pending_list, adata->prev_cpu); list_add_tail(&adata->entry, &plist->head); adata->moving = 1; - smp_ops.send_ipi_single(adata->prev_cpu, SMP_CLEAR_VECTOR); + smp_ops.send_ipi_single(adata->prev_cpu, ACTION_CLEAR_VECTOR); } } @@ -132,6 +133,7 @@ static int avecintc_set_affinity(struct irq_data *data, const struct cpumask *de static int avecintc_cpu_online(unsigned int cpu) { + long value; if (!loongarch_avec.vector_matrix) return 0; @@ -141,6 +143,10 @@ static int avecintc_cpu_online(unsigned int cpu) pending_list_init(cpu); + value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC); + value |= IOCSR_MISC_FUNC_AVEC_EN; + iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC); + raw_spin_unlock(&loongarch_avec.lock); return 0; @@ -193,7 +199,7 @@ void complete_irq_moving(void) } if (isr & (1UL << (vector % VECTORS_PER_REG))) { - smp_ops.send_ipi_single(cpu, SMP_CLEAR_VECTOR); + smp_ops.send_ipi_single(cpu, ACTION_CLEAR_VECTOR); continue; } list_del(&adata->entry); @@ -367,6 +373,7 @@ static int __init avecintc_init(struct irq_domain *parent) int ret, parent_irq; unsigned long value; + disable_pci_irq_limit = true; raw_spin_lock_init(&loongarch_avec.lock); loongarch_avec.fwnode = irq_domain_alloc_named_fwnode("AVECINTC"); diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 161c3ac171a3b7a05197a922e2aeb4c1c4bcaa50..d280d06e63c12a5ceba4f4f7ab4cf73ac4abdad8 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -409,6 +409,8 @@ static int msi_capability_init(struct pci_dev *dev, int nvec, } #ifdef CONFIG_LOONGARCH +#include + static unsigned int pci_irq_numbers = 32; static int __init pci_irq_limit(char *str) @@ -430,9 +432,11 @@ int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, int rc; #ifdef CONFIG_LOONGARCH - if (maxvec > 32) { - maxvec = pci_irq_numbers; - minvec = min_t(int, pci_irq_numbers, minvec); + if (!disable_pci_irq_limit) { + if (maxvec > 32) { + maxvec = pci_irq_numbers; + minvec = min_t(int, pci_irq_numbers, minvec); + } } #endif @@ -818,9 +822,11 @@ int __pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, int int hwsize, rc, nvec = maxvec; #ifdef CONFIG_LOONGARCH - if (maxvec > 32) { - nvec = pci_irq_numbers; - minvec = min_t(int, pci_irq_numbers, minvec); + if (!disable_pci_irq_limit) { + if (maxvec > 32) { + nvec = pci_irq_numbers; + minvec = min_t(int, pci_irq_numbers, minvec); + } } #endif diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 7e46dcb9ba33c34d6c2760db686cd3f804e0f6d7..73c57f76f8f4ba60a8c321e01b728b9421059e10 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -32,6 +32,9 @@ #include #include #include +#ifdef CONFIG_MACH_LOONGSON64 +#include +#endif #include "pci.h" DEFINE_MUTEX(pci_slot_mutex); @@ -172,6 +175,15 @@ static bool pci_bridge_d3_disable; /* Force bridge_d3 for all PCIe ports */ static bool pci_bridge_d3_force; +#ifdef CONFIG_MACH_LOONGSON64 + +#ifndef CONFIG_PM_SLEEP +suspend_state_t pm_suspend_target_state; +#define pm_suspend_target_state (PM_SUSPEND_ON) +#endif + +#endif + static int __init pcie_port_pm_setup(char *str) { if (!strcmp(str, "off")) @@ -6206,8 +6218,9 @@ int pcie_set_readrq(struct pci_dev *dev, int rq) { u16 v; int ret; +#ifdef CONFIG_MACH_LOONGSON64 struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus); - +#endif if (rq < 128 || rq > 4096 || !is_power_of_2(rq)) return -EINVAL; @@ -6225,7 +6238,9 @@ int pcie_set_readrq(struct pci_dev *dev, int rq) v = (ffs(rq) - 8) << 12; - if (bridge->no_inc_mrrs) { +#ifdef CONFIG_MACH_LOONGSON64 + if (pm_suspend_target_state == PM_SUSPEND_ON && + bridge->no_inc_mrrs) { int max_mrrs = pcie_get_readrq(dev); if (rq > max_mrrs) { @@ -6233,6 +6248,7 @@ int pcie_set_readrq(struct pci_dev *dev, int rq) return -EINVAL; } } +#endif ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_READRQ, v); diff --git a/drivers/platform/loongarch/Kconfig b/drivers/platform/loongarch/Kconfig index 5633e4d73991a69e4386e2771386d7c9a0181134..9ec1a86ef7faeac6f29d6ece028e2534da435416 100644 --- a/drivers/platform/loongarch/Kconfig +++ b/drivers/platform/loongarch/Kconfig @@ -16,6 +16,14 @@ menuconfig LOONGARCH_PLATFORM_DEVICES if LOONGARCH_PLATFORM_DEVICES +config CPU_HWMON + bool "Loongson CPU HWMon Driver" + depends on MACH_LOONGSON64 + select HWMON + default y + help + Loongson-3A/3B/3C CPU HWMon (temperature sensor) driver. + config LOONGSON_LAPTOP tristate "Generic Loongson-3 Laptop Driver" depends on ACPI diff --git a/drivers/platform/loongarch/Makefile b/drivers/platform/loongarch/Makefile index f43ab03db1a2d5bde40f59cbd403c866ca7f1026..695688bed4232f2e1c99ae3be4f16e9799331e01 100644 --- a/drivers/platform/loongarch/Makefile +++ b/drivers/platform/loongarch/Makefile @@ -1 +1,2 @@ obj-$(CONFIG_LOONGSON_LAPTOP) += loongson-laptop.o +obj-$(CONFIG_CPU_HWMON) += cpu_hwmon.o diff --git a/drivers/platform/loongarch/cpu_hwmon.c b/drivers/platform/loongarch/cpu_hwmon.c new file mode 100644 index 0000000000000000000000000000000000000000..c705d088c44a8476047491eccac51009f26d1824 --- /dev/null +++ b/drivers/platform/loongarch/cpu_hwmon.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include + +#include + +static int nr_packages; +static struct device *cpu_hwmon_dev; + +static int loongson3_cpu_temp(int cpu) +{ + u32 reg; + + reg = iocsr_read32(LOONGARCH_IOCSR_CPUTEMP) & 0xff; + + return (int)((s8)reg) * 1000; +} + +static ssize_t cpu_temp_label(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int id = (to_sensor_dev_attr(attr))->index - 1; + + return sprintf(buf, "CPU %d Temperature\n", id); +} + +static ssize_t get_cpu_temp(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int id = (to_sensor_dev_attr(attr))->index - 1; + int value = loongson3_cpu_temp(id); + + return sprintf(buf, "%d\n", value); +} + +static SENSOR_DEVICE_ATTR(temp1_input, 0444, get_cpu_temp, NULL, 1); +static SENSOR_DEVICE_ATTR(temp1_label, 0444, cpu_temp_label, NULL, 1); +static SENSOR_DEVICE_ATTR(temp2_input, 0444, get_cpu_temp, NULL, 2); +static SENSOR_DEVICE_ATTR(temp2_label, 0444, cpu_temp_label, NULL, 2); +static SENSOR_DEVICE_ATTR(temp3_input, 0444, get_cpu_temp, NULL, 3); +static SENSOR_DEVICE_ATTR(temp3_label, 0444, cpu_temp_label, NULL, 3); +static SENSOR_DEVICE_ATTR(temp4_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp4_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp5_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp5_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp6_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp6_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp7_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp7_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp8_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp8_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp9_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp9_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp10_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp10_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp11_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp11_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp12_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp12_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp13_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp13_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp14_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp14_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp15_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp15_label, 0444, cpu_temp_label, NULL, 4); +static SENSOR_DEVICE_ATTR(temp16_input, 0444, get_cpu_temp, NULL, 4); +static SENSOR_DEVICE_ATTR(temp16_label, 0444, cpu_temp_label, NULL, 4); + +static struct attribute *cpu_hwmon_attributes[] = { + &sensor_dev_attr_temp1_input.dev_attr.attr, + &sensor_dev_attr_temp1_label.dev_attr.attr, + &sensor_dev_attr_temp2_input.dev_attr.attr, + &sensor_dev_attr_temp2_label.dev_attr.attr, + &sensor_dev_attr_temp3_input.dev_attr.attr, + &sensor_dev_attr_temp3_label.dev_attr.attr, + &sensor_dev_attr_temp4_input.dev_attr.attr, + &sensor_dev_attr_temp4_label.dev_attr.attr, + &sensor_dev_attr_temp5_input.dev_attr.attr, + &sensor_dev_attr_temp5_label.dev_attr.attr, + &sensor_dev_attr_temp6_input.dev_attr.attr, + &sensor_dev_attr_temp6_label.dev_attr.attr, + &sensor_dev_attr_temp7_input.dev_attr.attr, + &sensor_dev_attr_temp7_label.dev_attr.attr, + &sensor_dev_attr_temp8_input.dev_attr.attr, + &sensor_dev_attr_temp8_label.dev_attr.attr, + &sensor_dev_attr_temp9_input.dev_attr.attr, + &sensor_dev_attr_temp9_label.dev_attr.attr, + &sensor_dev_attr_temp10_input.dev_attr.attr, + &sensor_dev_attr_temp10_label.dev_attr.attr, + &sensor_dev_attr_temp11_input.dev_attr.attr, + &sensor_dev_attr_temp11_label.dev_attr.attr, + &sensor_dev_attr_temp12_input.dev_attr.attr, + &sensor_dev_attr_temp12_label.dev_attr.attr, + &sensor_dev_attr_temp13_input.dev_attr.attr, + &sensor_dev_attr_temp13_label.dev_attr.attr, + &sensor_dev_attr_temp14_input.dev_attr.attr, + &sensor_dev_attr_temp14_label.dev_attr.attr, + &sensor_dev_attr_temp15_input.dev_attr.attr, + &sensor_dev_attr_temp15_label.dev_attr.attr, + &sensor_dev_attr_temp16_input.dev_attr.attr, + &sensor_dev_attr_temp16_label.dev_attr.attr, + NULL +}; +static umode_t cpu_hwmon_is_visible(struct kobject *kobj, + struct attribute *attr, int i) +{ + int id = i / 2; + + if (id < nr_packages) + return attr->mode; + return 0; +} + +static struct attribute_group cpu_hwmon_group = { + .attrs = cpu_hwmon_attributes, + .is_visible = cpu_hwmon_is_visible, +}; + +static const struct attribute_group *cpu_hwmon_groups[] = { + &cpu_hwmon_group, + NULL +}; + +static int cpu_initial_threshold = 72000; +static int cpu_thermal_threshold = 96000; +module_param(cpu_thermal_threshold, int, 0644); +MODULE_PARM_DESC(cpu_thermal_threshold, "cpu thermal threshold (96000 (default))"); + +static struct delayed_work thermal_work; + +static void do_thermal_timer(struct work_struct *work) +{ + int i, value, temp_max = 0; + + for (i = 0; i < nr_packages; i++) { + value = loongson3_cpu_temp(i); + if (value > temp_max) + temp_max = value; + } + + if (temp_max <= cpu_thermal_threshold) + schedule_delayed_work(&thermal_work, msecs_to_jiffies(5000)); + else + orderly_poweroff(true); +} + +static int __init loongson_hwmon_init(void) +{ + int i, value, temp_max = 0; + + pr_info("Loongson Hwmon Enter...\n"); + + nr_packages = loongson_sysconf.nr_cpus / + loongson_sysconf.cores_per_package; + + cpu_hwmon_dev = hwmon_device_register_with_groups(NULL, "cpu_hwmon", + NULL, cpu_hwmon_groups); + if (IS_ERR(cpu_hwmon_dev)) { + pr_err("Hwmon register fail with %ld!\n", PTR_ERR(cpu_hwmon_dev)); + return PTR_ERR(cpu_hwmon_dev); + } + + for (i = 0; i < nr_packages; i++) { + value = loongson3_cpu_temp(i); + if (value > temp_max) + temp_max = value; + } + + pr_info("Initial CPU temperature is %d (highest).\n", temp_max); + if (temp_max > cpu_initial_threshold) + cpu_thermal_threshold += temp_max - cpu_initial_threshold; + + INIT_DEFERRABLE_WORK(&thermal_work, do_thermal_timer); + schedule_delayed_work(&thermal_work, msecs_to_jiffies(20000)); + + return 0; +} + +static void __exit loongson_hwmon_exit(void) +{ + cancel_delayed_work_sync(&thermal_work); + hwmon_device_unregister(cpu_hwmon_dev); +} + +module_init(loongson_hwmon_init); +module_exit(loongson_hwmon_exit); + +MODULE_AUTHOR("Huacai Chen "); +MODULE_DESCRIPTION("Loongson CPU Hwmon driver"); +MODULE_LICENSE("GPL"); diff --git a/include/drm/drm_cache.h b/include/drm/drm_cache.h index 08e0e3ffad1319d8a32a21b37e1b070ae0a9acaa..667fb0368ef6d5a6a9d987636a72e77aeca8de3a 100644 --- a/include/drm/drm_cache.h +++ b/include/drm/drm_cache.h @@ -74,7 +74,7 @@ static inline bool drm_arch_can_wc_memory(void) * cache coherency machanism. This means WUC can only used for write-only * memory regions. */ - return false; + return wc_enabled; #else return true; #endif diff --git a/include/linux/compiler.h b/include/linux/compiler.h index df29ddb3c8622e080bba9e90c81587f59d948b03..266a74943831407be3fdc9661179155081b202f4 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -133,7 +133,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, #define annotate_unreachable() __annotate_unreachable(__COUNTER__) /* Annotate a C jump table to allow objtool to follow the code flow */ -#define __annotate_jump_table __section(".rodata..c_jump_table") +#define __annotate_jump_table __section(".rodata..c_jump_table,\"a\",@progbits #") #else /* !CONFIG_OBJTOOL */ #define annotate_reachable() diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 36545ce04f62acec4d66a19dbdb1840e8b6469e0..cd0b218a80d33683d981412669f78b1728338f19 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -216,6 +216,7 @@ enum kvm_bus { KVM_PIO_BUS, KVM_VIRTIO_CCW_NOTIFY_BUS, KVM_FAST_MMIO_BUS, + KVM_IOCSR_BUS, KVM_NR_BUSES }; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 10952900d67ae0d595abf6a063da728184620788..0e485c57dfeda9a7d8b2e8d3e0145953d44c9c7e 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1423,7 +1423,15 @@ enum kvm_device_type { #define KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_RISCV_AIA, #define KVM_DEV_TYPE_RISCV_AIA KVM_DEV_TYPE_RISCV_AIA + KVM_DEV_TYPE_LA_IOAPIC = 0x100, +#define KVM_DEV_TYPE_LA_IOAPIC KVM_DEV_TYPE_LA_IOAPIC + KVM_DEV_TYPE_LA_IPI, +#define KVM_DEV_TYPE_LA_IPI KVM_DEV_TYPE_LA_IPI + KVM_DEV_TYPE_LA_EXTIOI, +#define KVM_DEV_TYPE_LA_EXTIOI KVM_DEV_TYPE_LA_EXTIOI + KVM_DEV_TYPE_MAX, + }; struct kvm_vfio_spapr_tce { diff --git a/tools/include/uapi/asm/bitsperlong.h b/tools/include/uapi/asm/bitsperlong.h index c65267afc3415b194b9f60f8c56024daf6e8a535..4be3d005acee9f27ce4e22b75f9947bd8e994fa2 100644 --- a/tools/include/uapi/asm/bitsperlong.h +++ b/tools/include/uapi/asm/bitsperlong.h @@ -13,6 +13,8 @@ #include "../../../arch/ia64/include/uapi/asm/bitsperlong.h" #elif defined(__alpha__) #include "../../../arch/alpha/include/uapi/asm/bitsperlong.h" +#elif defined(__loongarch__) +#include "../../../arch/loongarch/include/uapi/asm/bitsperlong.h" #else #include #endif diff --git a/tools/objtool/arch/loongarch/decode.c b/tools/objtool/arch/loongarch/decode.c index aee479d2191c21a893a7ea04f01e93d68290978f..69b66994f2a1557275212bf470b413af299f74f2 100644 --- a/tools/objtool/arch/loongarch/decode.c +++ b/tools/objtool/arch/loongarch/decode.c @@ -122,7 +122,7 @@ static bool decode_insn_reg2i12_fomat(union loongarch_instruction inst, switch (inst.reg2i12_format.opcode) { case addid_op: if ((inst.reg2i12_format.rd == CFI_SP) || (inst.reg2i12_format.rj == CFI_SP)) { - /* addi.d sp,sp,si12 or addi.d fp,sp,si12 */ + /* addi.d sp,sp,si12 or addi.d fp,sp,si12 or addi.d sp,fp,si12 */ insn->immediate = sign_extend64(inst.reg2i12_format.immediate, 11); ADD_OP(op) { op->src.type = OP_SRC_ADD; @@ -132,6 +132,15 @@ static bool decode_insn_reg2i12_fomat(union loongarch_instruction inst, op->dest.reg = inst.reg2i12_format.rd; } } + if ((inst.reg2i12_format.rd == CFI_SP) && (inst.reg2i12_format.rj == CFI_FP)) { + /* addi.d sp,fp,si12 */ + struct symbol *func = find_func_containing(insn->sec, insn->offset); + + if (!func) + return false; + + func->frame_pointer = true; + } break; case ldd_op: if (inst.reg2i12_format.rj == CFI_SP) { diff --git a/tools/objtool/arch/x86/Build b/tools/objtool/arch/x86/Build index 9f7869b5c5e0caf1c0128e0f177d98e931017583..3dedb2fd8f3a0c3d7502e2536167f5e895fe1d44 100644 --- a/tools/objtool/arch/x86/Build +++ b/tools/objtool/arch/x86/Build @@ -1,5 +1,6 @@ objtool-y += special.o objtool-y += decode.o +objtool-y += orc.o inat_tables_script = ../arch/x86/tools/gen-insn-attr-x86.awk inat_tables_maps = ../arch/x86/lib/x86-opcode-map.txt diff --git a/tools/objtool/arch/x86/orc.c b/tools/objtool/arch/x86/orc.c new file mode 100644 index 0000000000000000000000000000000000000000..b6cd943e87f936ef93ce03609d4596b6fd96553a --- /dev/null +++ b/tools/objtool/arch/x86/orc.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include +#include + +#include +#include +#include +#include + +int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, struct instruction *insn) +{ + struct cfi_reg *bp = &cfi->regs[CFI_BP]; + + memset(orc, 0, sizeof(*orc)); + + if (!cfi) { + /* + * This is usually either unreachable nops/traps (which don't + * trigger unreachable instruction warnings), or + * STACK_FRAME_NON_STANDARD functions. + */ + orc->type = ORC_TYPE_UNDEFINED; + return 0; + } + + switch (cfi->type) { + case UNWIND_HINT_TYPE_UNDEFINED: + orc->type = ORC_TYPE_UNDEFINED; + return 0; + case UNWIND_HINT_TYPE_END_OF_STACK: + orc->type = ORC_TYPE_END_OF_STACK; + return 0; + case UNWIND_HINT_TYPE_CALL: + orc->type = ORC_TYPE_CALL; + break; + case UNWIND_HINT_TYPE_REGS: + orc->type = ORC_TYPE_REGS; + break; + case UNWIND_HINT_TYPE_REGS_PARTIAL: + orc->type = ORC_TYPE_REGS_PARTIAL; + break; + default: + WARN_INSN(insn, "unknown unwind hint type %d", cfi->type); + return -1; + } + + orc->signal = cfi->signal; + + switch (cfi->cfa.base) { + case CFI_SP: + orc->sp_reg = ORC_REG_SP; + break; + case CFI_SP_INDIRECT: + orc->sp_reg = ORC_REG_SP_INDIRECT; + break; + case CFI_BP: + orc->sp_reg = ORC_REG_BP; + break; + case CFI_BP_INDIRECT: + orc->sp_reg = ORC_REG_BP_INDIRECT; + break; + case CFI_R10: + orc->sp_reg = ORC_REG_R10; + break; + case CFI_R13: + orc->sp_reg = ORC_REG_R13; + break; + case CFI_DI: + orc->sp_reg = ORC_REG_DI; + break; + case CFI_DX: + orc->sp_reg = ORC_REG_DX; + break; + default: + WARN_INSN(insn, "unknown CFA base reg %d", cfi->cfa.base); + return -1; + } + + switch (bp->base) { + case CFI_UNDEFINED: + orc->bp_reg = ORC_REG_UNDEFINED; + break; + case CFI_CFA: + orc->bp_reg = ORC_REG_PREV_SP; + break; + case CFI_BP: + orc->bp_reg = ORC_REG_BP; + break; + default: + WARN_INSN(insn, "unknown BP base reg %d", bp->base); + return -1; + } + + orc->sp_offset = cfi->cfa.offset; + orc->bp_offset = bp->offset; + + return 0; +} + +int write_orc_entry(struct elf *elf, struct section *orc_sec, + struct section *ip_sec, unsigned int idx, + struct section *insn_sec, unsigned long insn_off, + struct orc_entry *o) +{ + struct orc_entry *orc; + + /* populate ORC data */ + orc = (struct orc_entry *)orc_sec->data->d_buf + idx; + memcpy(orc, o, sizeof(*orc)); + orc->sp_offset = bswap_if_needed(elf, orc->sp_offset); + orc->bp_offset = bswap_if_needed(elf, orc->bp_offset); + + /* populate reloc for ip */ + if (!elf_init_reloc_text_sym(elf, ip_sec, idx * sizeof(int), idx, + insn_sec, insn_off)) + return -1; + + return 0; +} + +static const char *reg_name(unsigned int reg) +{ + switch (reg) { + case ORC_REG_PREV_SP: + return "prevsp"; + case ORC_REG_DX: + return "dx"; + case ORC_REG_DI: + return "di"; + case ORC_REG_BP: + return "bp"; + case ORC_REG_SP: + return "sp"; + case ORC_REG_R10: + return "r10"; + case ORC_REG_R13: + return "r13"; + case ORC_REG_BP_INDIRECT: + return "bp(ind)"; + case ORC_REG_SP_INDIRECT: + return "sp(ind)"; + default: + return "?"; + } +} + +static const char *orc_type_name(unsigned int type) +{ + switch (type) { + case ORC_TYPE_UNDEFINED: + return "(und)"; + case ORC_TYPE_END_OF_STACK: + return "end"; + case ORC_TYPE_CALL: + return "call"; + case ORC_TYPE_REGS: + return "regs"; + case ORC_TYPE_REGS_PARTIAL: + return "regs (partial)"; + default: + return "?"; + } +} + +static void print_reg(unsigned int reg, int offset) +{ + if (reg == ORC_REG_BP_INDIRECT) + printf("(bp%+d)", offset); + else if (reg == ORC_REG_SP_INDIRECT) + printf("(sp)%+d", offset); + else if (reg == ORC_REG_UNDEFINED) + printf("(und)"); + else + printf("%s%+d", reg_name(reg), offset); +} + +void orc_print_dump(struct elf *dummy_elf, struct orc_entry *orc, int i) +{ + printf("type:%s", orc_type_name(orc[i].type)); + + printf(" sp:"); + print_reg(orc[i].sp_reg, bswap_if_needed(dummy_elf, orc[i].sp_offset)); + + printf(" bp:"); + print_reg(orc[i].bp_reg, bswap_if_needed(dummy_elf, orc[i].bp_offset)); + + printf(" signal:%d\n", orc[i].signal); +} diff --git a/tools/objtool/check.c b/tools/objtool/check.c index e255ac8aae578cd949c83dbe0dfb064d7df64ae0..1d3be9c53fd2b9524bfd8733cc368a04458ff7ef 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -20,6 +20,7 @@ #include #include #include +#include struct alternative { struct alternative *next; @@ -584,7 +585,7 @@ static int add_dead_ends(struct objtool_file *file) struct section *rsec; struct reloc *reloc; struct instruction *insn; - s64 addend; + unsigned long offset; /* * Check for manually annotated dead ends. @@ -594,27 +595,28 @@ static int add_dead_ends(struct objtool_file *file) goto reachable; for_each_reloc(rsec, reloc) { - - if (reloc->sym->type != STT_SECTION) { + if (reloc->sym->type == STT_SECTION) { + offset = reloc_addend(reloc); + } else if (reloc->sym->local_label) { + offset = reloc->sym->offset; + } else { WARN("unexpected relocation symbol type in %s", rsec->name); return -1; } - addend = reloc_addend(reloc); - - insn = find_insn(file, reloc->sym->sec, addend); + insn = find_insn(file, reloc->sym->sec, offset); if (insn) insn = prev_insn_same_sec(file, insn); - else if (addend == reloc->sym->sec->sh.sh_size) { + else if (offset == reloc->sym->sec->sh.sh_size) { insn = find_last_insn(file, reloc->sym->sec); if (!insn) { WARN("can't find unreachable insn at %s+0x%" PRIx64, - reloc->sym->sec->name, addend); + reloc->sym->sec->name, offset); return -1; } } else { WARN("can't find unreachable insn at %s+0x%" PRIx64, - reloc->sym->sec->name, addend); + reloc->sym->sec->name, offset); return -1; } @@ -633,27 +635,28 @@ static int add_dead_ends(struct objtool_file *file) return 0; for_each_reloc(rsec, reloc) { - - if (reloc->sym->type != STT_SECTION) { + if (reloc->sym->type == STT_SECTION) { + offset = reloc_addend(reloc); + } else if (reloc->sym->local_label) { + offset = reloc->sym->offset; + } else { WARN("unexpected relocation symbol type in %s", rsec->name); return -1; } - addend = reloc_addend(reloc); - - insn = find_insn(file, reloc->sym->sec, addend); + insn = find_insn(file, reloc->sym->sec, offset); if (insn) insn = prev_insn_same_sec(file, insn); - else if (addend == reloc->sym->sec->sh.sh_size) { + else if (offset == reloc->sym->sec->sh.sh_size) { insn = find_last_insn(file, reloc->sym->sec); if (!insn) { WARN("can't find reachable insn at %s+0x%" PRIx64, - reloc->sym->sec->name, addend); + reloc->sym->sec->name, offset); return -1; } } else { WARN("can't find reachable insn at %s+0x%" PRIx64, - reloc->sym->sec->name, addend); + reloc->sym->sec->name, offset); return -1; } @@ -2208,6 +2211,7 @@ static int read_unwind_hints(struct objtool_file *file) struct unwind_hint *hint; struct instruction *insn; struct reloc *reloc; + unsigned long offset; int i; sec = find_section_by_name(file->elf, ".discard.unwind_hints"); @@ -2235,7 +2239,16 @@ static int read_unwind_hints(struct objtool_file *file) return -1; } - insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); + if (reloc->sym->type == STT_SECTION) { + offset = reloc_addend(reloc); + } else if (reloc->sym->local_label) { + offset = reloc->sym->offset; + } else { + WARN("unexpected relocation symbol type in %s", sec->rsec->name); + return -1; + } + + insn = find_insn(file, reloc->sym->sec, offset); if (!insn) { WARN("can't find insn for unwind_hints[%d]", i); return -1; @@ -2506,6 +2519,9 @@ static int classify_symbols(struct objtool_file *file) struct symbol *func; for_each_sym(file, func) { + if (func->type == STT_NOTYPE && strstarts(func->name, ".L")) + func->local_label = true; + if (func->bind != STB_GLOBAL) continue; @@ -2959,10 +2975,27 @@ static int update_cfi_state(struct instruction *insn, break; } - if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { + if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && + insn->sym->frame_pointer) { + /* addi.d fp,sp,imm on LoongArch */ + if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { + cfa->base = CFI_BP; + cfa->offset = 0; + } + break; + } - /* lea disp(%rbp), %rsp */ - cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); + if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { + /* addi.d sp,fp,imm on LoongArch */ + if (cfa->base == CFI_BP && cfa->offset == 0) { + if (insn->sym->frame_pointer) { + cfa->base = CFI_SP; + cfa->offset = -op->src.offset; + } + } else { + /* lea disp(%rbp), %rsp */ + cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); + } break; } diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index 9f71e988eca45fd6bc81734b9a52d76b895e9517..d7e815c2fd1567bdd86fdbb78e26b5ab68238398 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -67,6 +67,8 @@ struct symbol { u8 profiling_func : 1; u8 warned : 1; u8 embedded_insn : 1; + u8 local_label : 1; + u8 frame_pointer : 1; struct list_head pv_target; struct reloc *relocs; }; diff --git a/tools/objtool/include/objtool/orc.h b/tools/objtool/include/objtool/orc.h new file mode 100644 index 0000000000000000000000000000000000000000..15a32def1071d0e9bfe266530b6435bfec2b33f8 --- /dev/null +++ b/tools/objtool/include/objtool/orc.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _OBJTOOL_ORC_H +#define _OBJTOOL_ORC_H + +#include + +int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, struct instruction *insn); +void orc_print_dump(struct elf *dummy_elf, struct orc_entry *orc, int i); +int write_orc_entry(struct elf *elf, struct section *orc_sec, + struct section *ip_sec, unsigned int idx, + struct section *insn_sec, unsigned long insn_off, + struct orc_entry *o); + +#endif /* _OBJTOOL_ORC_H */ diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c index 0e183bb1c72051157cb1e6daf30fa12ff3f0ce7c..a62247efb64f2e06c293e129ade3abae4fd456c1 100644 --- a/tools/objtool/orc_dump.c +++ b/tools/objtool/orc_dump.c @@ -6,65 +6,10 @@ #include #include #include +#include #include #include -static const char *reg_name(unsigned int reg) -{ - switch (reg) { - case ORC_REG_PREV_SP: - return "prevsp"; - case ORC_REG_DX: - return "dx"; - case ORC_REG_DI: - return "di"; - case ORC_REG_BP: - return "bp"; - case ORC_REG_SP: - return "sp"; - case ORC_REG_R10: - return "r10"; - case ORC_REG_R13: - return "r13"; - case ORC_REG_BP_INDIRECT: - return "bp(ind)"; - case ORC_REG_SP_INDIRECT: - return "sp(ind)"; - default: - return "?"; - } -} - -static const char *orc_type_name(unsigned int type) -{ - switch (type) { - case ORC_TYPE_UNDEFINED: - return "(und)"; - case ORC_TYPE_END_OF_STACK: - return "end"; - case ORC_TYPE_CALL: - return "call"; - case ORC_TYPE_REGS: - return "regs"; - case ORC_TYPE_REGS_PARTIAL: - return "regs (partial)"; - default: - return "?"; - } -} - -static void print_reg(unsigned int reg, int offset) -{ - if (reg == ORC_REG_BP_INDIRECT) - printf("(bp%+d)", offset); - else if (reg == ORC_REG_SP_INDIRECT) - printf("(sp)%+d", offset); - else if (reg == ORC_REG_UNDEFINED) - printf("(und)"); - else - printf("%s%+d", reg_name(reg), offset); -} - int orc_dump(const char *_objname) { int fd, nr_entries, i, *orc_ip = NULL, orc_size = 0; @@ -205,17 +150,7 @@ int orc_dump(const char *_objname) printf("%llx:", (unsigned long long)(orc_ip_addr + (i * sizeof(int)) + orc_ip[i])); } - printf("type:%s", orc_type_name(orc[i].type)); - - printf(" sp:"); - - print_reg(orc[i].sp_reg, bswap_if_needed(&dummy_elf, orc[i].sp_offset)); - - printf(" bp:"); - - print_reg(orc[i].bp_reg, bswap_if_needed(&dummy_elf, orc[i].bp_offset)); - - printf(" signal:%d\n", orc[i].signal); + orc_print_dump(&dummy_elf, orc, i); } elf_end(elf); diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index bae343908867105f25ac35c61083e4d0ecfeca73..922e6aac7cea7c089fb48b37e94a01718b8c00ab 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -10,121 +10,10 @@ #include #include +#include #include #include -static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, - struct instruction *insn) -{ - struct cfi_reg *bp = &cfi->regs[CFI_BP]; - - memset(orc, 0, sizeof(*orc)); - - if (!cfi) { - /* - * This is usually either unreachable nops/traps (which don't - * trigger unreachable instruction warnings), or - * STACK_FRAME_NON_STANDARD functions. - */ - orc->type = ORC_TYPE_UNDEFINED; - return 0; - } - - switch (cfi->type) { - case UNWIND_HINT_TYPE_UNDEFINED: - orc->type = ORC_TYPE_UNDEFINED; - return 0; - case UNWIND_HINT_TYPE_END_OF_STACK: - orc->type = ORC_TYPE_END_OF_STACK; - return 0; - case UNWIND_HINT_TYPE_CALL: - orc->type = ORC_TYPE_CALL; - break; - case UNWIND_HINT_TYPE_REGS: - orc->type = ORC_TYPE_REGS; - break; - case UNWIND_HINT_TYPE_REGS_PARTIAL: - orc->type = ORC_TYPE_REGS_PARTIAL; - break; - default: - WARN_INSN(insn, "unknown unwind hint type %d", cfi->type); - return -1; - } - - orc->signal = cfi->signal; - - switch (cfi->cfa.base) { - case CFI_SP: - orc->sp_reg = ORC_REG_SP; - break; - case CFI_SP_INDIRECT: - orc->sp_reg = ORC_REG_SP_INDIRECT; - break; - case CFI_BP: - orc->sp_reg = ORC_REG_BP; - break; - case CFI_BP_INDIRECT: - orc->sp_reg = ORC_REG_BP_INDIRECT; - break; - case CFI_R10: - orc->sp_reg = ORC_REG_R10; - break; - case CFI_R13: - orc->sp_reg = ORC_REG_R13; - break; - case CFI_DI: - orc->sp_reg = ORC_REG_DI; - break; - case CFI_DX: - orc->sp_reg = ORC_REG_DX; - break; - default: - WARN_INSN(insn, "unknown CFA base reg %d", cfi->cfa.base); - return -1; - } - - switch (bp->base) { - case CFI_UNDEFINED: - orc->bp_reg = ORC_REG_UNDEFINED; - break; - case CFI_CFA: - orc->bp_reg = ORC_REG_PREV_SP; - break; - case CFI_BP: - orc->bp_reg = ORC_REG_BP; - break; - default: - WARN_INSN(insn, "unknown BP base reg %d", bp->base); - return -1; - } - - orc->sp_offset = cfi->cfa.offset; - orc->bp_offset = bp->offset; - - return 0; -} - -static int write_orc_entry(struct elf *elf, struct section *orc_sec, - struct section *ip_sec, unsigned int idx, - struct section *insn_sec, unsigned long insn_off, - struct orc_entry *o) -{ - struct orc_entry *orc; - - /* populate ORC data */ - orc = (struct orc_entry *)orc_sec->data->d_buf + idx; - memcpy(orc, o, sizeof(*orc)); - orc->sp_offset = bswap_if_needed(elf, orc->sp_offset); - orc->bp_offset = bswap_if_needed(elf, orc->bp_offset); - - /* populate reloc for ip */ - if (!elf_init_reloc_text_sym(elf, ip_sec, idx * sizeof(int), idx, - insn_sec, insn_off)) - return -1; - - return 0; -} - struct orc_list_entry { struct list_head list; struct orc_entry orc; diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c index 86d267db87bb13b6bb7050b0325415abf21eea2e..7bc74969a819afb5198fc2dfb7c9e83b80f967e0 100644 --- a/virt/kvm/dirty_ring.c +++ b/virt/kvm/dirty_ring.c @@ -55,6 +55,9 @@ static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask) struct kvm_memory_slot *memslot; int as_id, id; + if (!mask) + return; + as_id = slot >> 16; id = (u16)slot;