| Message ID | 20241206163102.843505-4-cleger@rivosinc.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | riscv: add support for SBI Supervisor Software Events | expand |
Hi Clement, On Fri, Dec 06, 2024 at 05:30:59PM +0100, Clément Léger wrote: > Add driver level interface to use RISC-V SSE arch support. This interface > allows registering SSE handlers, and receive them. This will be used by > PMU and GHES driver. > > Signed-off-by: Himanshu Chauhan <hchauhan@ventanamicro.com> > Co-developed-by: Himanshu Chauhan <hchauhan@ventanamicro.com> > Signed-off-by: Clément Léger <cleger@rivosinc.com> > --- > MAINTAINERS | 14 + > drivers/firmware/Kconfig | 1 + > drivers/firmware/Makefile | 1 + > drivers/firmware/riscv/Kconfig | 15 + > drivers/firmware/riscv/Makefile | 3 + > drivers/firmware/riscv/riscv_sse.c | 691 +++++++++++++++++++++++++++++ > include/linux/riscv_sse.h | 56 +++ > 7 files changed, 781 insertions(+) > create mode 100644 drivers/firmware/riscv/Kconfig > create mode 100644 drivers/firmware/riscv/Makefile > create mode 100644 drivers/firmware/riscv/riscv_sse.c > create mode 100644 include/linux/riscv_sse.h > > diff --git a/MAINTAINERS b/MAINTAINERS > index 686109008d8e..a3ddde7fe9fb 100644 > --- a/MAINTAINERS > +++ b/MAINTAINERS > @@ -20125,6 +20125,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git > F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml > F: drivers/iommu/riscv/ > > +RISC-V FIRMWARE DRIVERS > +M: Conor Dooley <conor@kernel.org> > +L: linux-riscv@lists.infradead.org > +S: Maintained > +T: git git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git > +F: drivers/firmware/riscv/* > + > RISC-V MICROCHIP FPGA SUPPORT > M: Conor Dooley <conor.dooley@microchip.com> > M: Daire McNamara <daire.mcnamara@microchip.com> > @@ -20177,6 +20184,13 @@ F: drivers/perf/riscv_pmu.c > F: drivers/perf/riscv_pmu_legacy.c > F: drivers/perf/riscv_pmu_sbi.c > > +RISC-V SSE DRIVER > +M: Clément Léger <cleger@rivosinc.com> > +L: linux-riscv@lists.infradead.org > +S: Maintained > +F: drivers/firmware/riscv/riscv_sse.c > +F: include/linux/riscv_sse.h > + I request you to add me as a reviewer to these SSE files. Himanshu Chauhan <himanshu@thechauhan.dev> Thanks Regards Himanshu > RISC-V THEAD SoC SUPPORT > M: Drew Fustini <drew@pdp7.com> > M: Guo Ren <guoren@kernel.org> > diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig > index 71d8b26c4103..9e996a1fd511 100644 > --- a/drivers/firmware/Kconfig > +++ b/drivers/firmware/Kconfig > @@ -267,6 +267,7 @@ source "drivers/firmware/meson/Kconfig" > source "drivers/firmware/microchip/Kconfig" > source "drivers/firmware/psci/Kconfig" > source "drivers/firmware/qcom/Kconfig" > +source "drivers/firmware/riscv/Kconfig" > source "drivers/firmware/smccc/Kconfig" > source "drivers/firmware/tegra/Kconfig" > source "drivers/firmware/xilinx/Kconfig" > diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile > index 7a8d486e718f..c0f5009949a8 100644 > --- a/drivers/firmware/Makefile > +++ b/drivers/firmware/Makefile > @@ -33,6 +33,7 @@ obj-y += efi/ > obj-y += imx/ > obj-y += psci/ > obj-y += qcom/ > +obj-y += riscv/ > obj-y += smccc/ > obj-y += tegra/ > obj-y += xilinx/ > diff --git a/drivers/firmware/riscv/Kconfig b/drivers/firmware/riscv/Kconfig > new file mode 100644 > index 000000000000..8056ed3262d9 > --- /dev/null > +++ b/drivers/firmware/riscv/Kconfig > @@ -0,0 +1,15 @@ > +# SPDX-License-Identifier: GPL-2.0-only > +menu "Risc-V Specific firmware drivers" > +depends on RISCV > + > +config RISCV_SSE > + bool "Enable SBI Supervisor Software Events support" > + depends on RISCV_SBI > + default y > + help > + The Supervisor Software Events support allow the SBI to deliver > + NMI-like notifications to the supervisor mode software. When enable, > + this option provides support to register callbacks on specific SSE > + events. > + > +endmenu > diff --git a/drivers/firmware/riscv/Makefile b/drivers/firmware/riscv/Makefile > new file mode 100644 > index 000000000000..4ccfcbbc28ea > --- /dev/null > +++ b/drivers/firmware/riscv/Makefile > @@ -0,0 +1,3 @@ > +# SPDX-License-Identifier: GPL-2.0 > + > +obj-$(CONFIG_RISCV_SSE) += riscv_sse.o > diff --git a/drivers/firmware/riscv/riscv_sse.c b/drivers/firmware/riscv/riscv_sse.c > new file mode 100644 > index 000000000000..c165e32cc9a5 > --- /dev/null > +++ b/drivers/firmware/riscv/riscv_sse.c > @@ -0,0 +1,691 @@ > +// SPDX-License-Identifier: GPL-2.0-only > +/* > + * Copyright (C) 2024 Rivos Inc. > + */ > + > +#define pr_fmt(fmt) "sse: " fmt > + > +#include <linux/cpu.h> > +#include <linux/cpuhotplug.h> > +#include <linux/cpu_pm.h> > +#include <linux/hardirq.h> > +#include <linux/list.h> > +#include <linux/percpu-defs.h> > +#include <linux/reboot.h> > +#include <linux/riscv_sse.h> > +#include <linux/slab.h> > + > +#include <asm/sbi.h> > +#include <asm/sse.h> > + > +struct sse_event { > + struct list_head list; > + u32 evt; > + u32 priority; > + sse_event_handler *handler; > + void *handler_arg; > + bool is_enabled; > + /* Only valid for global events */ > + unsigned int cpu; > + > + union { > + struct sse_registered_event *global; > + struct sse_registered_event __percpu *local; > + }; > +}; > + > +static int sse_hp_state; > +static bool sse_available; > +static DEFINE_SPINLOCK(events_list_lock); > +static LIST_HEAD(events); > +static DEFINE_MUTEX(sse_mutex); > + > +struct sse_registered_event { > + struct sse_event_arch_data arch; > + struct sse_event *evt; > + unsigned long attr_buf; > +}; > + > +void sse_handle_event(struct sse_event_arch_data *arch_event, > + struct pt_regs *regs) > +{ > + int ret; > + struct sse_registered_event *reg_evt = > + container_of(arch_event, struct sse_registered_event, arch); > + struct sse_event *evt = reg_evt->evt; > + > + ret = evt->handler(evt->evt, evt->handler_arg, regs); > + if (ret) > + pr_warn("event %x handler failed with error %d\n", evt->evt, > + ret); > +} > + > +static bool sse_event_is_global(u32 evt) > +{ > + return !!(evt & SBI_SSE_EVENT_GLOBAL); > +} > + > +static > +struct sse_event *sse_event_get(u32 evt) > +{ > + struct sse_event *sse_evt = NULL, *tmp; > + > + scoped_guard(spinlock, &events_list_lock) { > + list_for_each_entry(tmp, &events, list) { > + if (tmp->evt == evt) { > + return sse_evt; > + } > + } > + } > + > + return NULL; > +} > + > +static phys_addr_t sse_event_get_phys(struct sse_registered_event *reg_evt, > + void *addr) > +{ > + phys_addr_t phys; > + > + if (sse_event_is_global(reg_evt->evt->evt)) > + phys = virt_to_phys(addr); > + else > + phys = per_cpu_ptr_to_phys(addr); > + > + return phys; > +} > + > +static int sse_sbi_event_func(struct sse_event *event, unsigned long func) > +{ > + struct sbiret ret; > + u32 evt = event->evt; > + > + ret = sbi_ecall(SBI_EXT_SSE, func, evt, 0, 0, 0, 0, 0); > + if (ret.error) > + pr_debug("Failed to execute func %lx, event %x, error %ld\n", > + func, evt, ret.error); > + > + return sbi_err_map_linux_errno(ret.error); > +} > + > +static int sse_sbi_disable_event(struct sse_event *event) > +{ > + return sse_sbi_event_func(event, SBI_SSE_EVENT_DISABLE); > +} > + > +static int sse_sbi_enable_event(struct sse_event *event) > +{ > + return sse_sbi_event_func(event, SBI_SSE_EVENT_ENABLE); > +} > + > +static int sse_event_attr_get_no_lock(struct sse_registered_event *reg_evt, > + unsigned long attr_id, unsigned long *val) > +{ > + struct sbiret sret; > + u32 evt = reg_evt->evt->evt; > + unsigned long phys; > + > + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); > + > + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, > + attr_id, 1, phys, 0, 0); > + if (sret.error) { > + pr_debug("Failed to get event %x attr %lx, error %ld\n", evt, > + attr_id, sret.error); > + return sbi_err_map_linux_errno(sret.error); > + } > + > + *val = reg_evt->attr_buf; > + > + return 0; > +} > + > +static int sse_event_attr_set_nolock(struct sse_registered_event *reg_evt, > + unsigned long attr_id, unsigned long val) > +{ > + struct sbiret sret; > + u32 evt = reg_evt->evt->evt; > + unsigned long phys; > + > + reg_evt->attr_buf = val; > + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); > + > + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, evt, > + attr_id, 1, phys, 0, 0); > + if (sret.error && sret.error != SBI_ERR_INVALID_STATE) { > + pr_debug("Failed to set event %x attr %lx, error %ld\n", evt, > + attr_id, sret.error); > + return sbi_err_map_linux_errno(sret.error); > + } > + > + return 0; > +} > + > +static int sse_event_set_target_cpu_nolock(struct sse_event *event, > + unsigned int cpu) > +{ > + unsigned int hart_id = cpuid_to_hartid_map(cpu); > + struct sse_registered_event *reg_evt = event->global; > + u32 evt = event->evt; > + bool was_enabled; > + int ret; > + > + if (!sse_event_is_global(evt)) > + return -EINVAL; > + > + was_enabled = event->is_enabled; > + if (was_enabled) > + sse_sbi_disable_event(event); > + do { > + ret = sse_event_attr_set_nolock(reg_evt, > + SBI_SSE_ATTR_PREFERRED_HART, > + hart_id); > + } while (ret == -EINVAL); > + > + if (ret == 0) > + event->cpu = cpu; > + > + if (was_enabled) > + sse_sbi_enable_event(event); > + > + return 0; > +} > + > +int sse_event_set_target_cpu(struct sse_event *event, unsigned int cpu) > +{ > + int ret; > + > + scoped_guard(mutex, &sse_mutex) { > + cpus_read_lock(); > + > + if (!cpu_online(cpu)) > + return -EINVAL; > + > + ret = sse_event_set_target_cpu_nolock(event, cpu); > + > + cpus_read_unlock(); > + } > + > + return ret; > +} > + > +static int sse_event_init_registered(unsigned int cpu, > + struct sse_registered_event *reg_evt, > + struct sse_event *event) > +{ > + reg_evt->evt = event; > + arch_sse_init_event(®_evt->arch, event->evt, cpu); > + > + return 0; > +} > + > +static void sse_event_free_registered(struct sse_registered_event *reg_evt) > +{ > + arch_sse_free_event(®_evt->arch); > +} > + > +static int sse_event_alloc_global(struct sse_event *event) > +{ > + int err; > + struct sse_registered_event *reg_evt; > + > + reg_evt = kzalloc(sizeof(*reg_evt), GFP_KERNEL); > + if (!reg_evt) > + return -ENOMEM; > + > + event->global = reg_evt; > + err = sse_event_init_registered(smp_processor_id(), reg_evt, > + event); > + if (err) > + kfree(reg_evt); > + > + return err; > +} > + > +static int sse_event_alloc_local(struct sse_event *event) > +{ > + int err; > + unsigned int cpu, err_cpu; > + struct sse_registered_event *reg_evt; > + struct sse_registered_event __percpu *reg_evts; > + > + reg_evts = alloc_percpu(struct sse_registered_event); > + if (!reg_evts) > + return -ENOMEM; > + > + event->local = reg_evts; > + > + for_each_possible_cpu(cpu) { > + reg_evt = per_cpu_ptr(reg_evts, cpu); > + err = sse_event_init_registered(cpu, reg_evt, event); > + if (err) { > + err_cpu = cpu; > + goto err_free_per_cpu; > + } > + } > + > + return 0; > + > +err_free_per_cpu: > + for_each_possible_cpu(cpu) { > + if (cpu == err_cpu) > + break; > + reg_evt = per_cpu_ptr(reg_evts, cpu); > + sse_event_free_registered(reg_evt); > + } > + > + free_percpu(reg_evts); > + > + return err; > +} > + > +static struct sse_event *sse_event_alloc(u32 evt, > + u32 priority, > + sse_event_handler *handler, void *arg) > +{ > + int err; > + struct sse_event *event; > + > + event = kzalloc(sizeof(*event), GFP_KERNEL); > + if (!event) > + return ERR_PTR(-ENOMEM); > + > + event->evt = evt; > + event->priority = priority; > + event->handler_arg = arg; > + event->handler = handler; > + > + if (sse_event_is_global(evt)) { > + err = sse_event_alloc_global(event); > + if (err) > + goto err_alloc_reg_evt; > + } else { > + err = sse_event_alloc_local(event); > + if (err) > + goto err_alloc_reg_evt; > + } > + > + return event; > + > +err_alloc_reg_evt: > + kfree(event); > + > + return ERR_PTR(err); > +} > + > +static int sse_sbi_register_event(struct sse_event *event, > + struct sse_registered_event *reg_evt) > +{ > + int ret; > + > + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PRIO, > + event->priority); > + if (ret) > + return ret; > + > + return arch_sse_register_event(®_evt->arch); > +} > + > +static int sse_event_register_local(struct sse_event *event) > +{ > + int ret; > + struct sse_registered_event *reg_evt = per_cpu_ptr(event->local, > + smp_processor_id()); > + > + ret = sse_sbi_register_event(event, reg_evt); > + if (ret) > + pr_debug("Failed to register event %x: err %d\n", event->evt, > + ret); > + > + return ret; > +} > + > + > +static int sse_sbi_unregister_event(struct sse_event *event) > +{ > + return sse_sbi_event_func(event, SBI_SSE_EVENT_UNREGISTER); > +} > + > +struct sse_per_cpu_evt { > + struct sse_event *event; > + unsigned long func; > + atomic_t error; > +}; > + > +static void sse_event_per_cpu_func(void *info) > +{ > + int ret; > + struct sse_per_cpu_evt *cpu_evt = info; > + > + if (cpu_evt->func == SBI_SSE_EVENT_REGISTER) > + ret = sse_event_register_local(cpu_evt->event); > + else > + ret = sse_sbi_event_func(cpu_evt->event, cpu_evt->func); > + > + if (ret) > + atomic_set(&cpu_evt->error, ret); > +} > + > +static void sse_event_free(struct sse_event *event) > +{ > + unsigned int cpu; > + struct sse_registered_event *reg_evt; > + > + if (sse_event_is_global(event->evt)) { > + sse_event_free_registered(event->global); > + kfree(event->global); > + } else { > + for_each_possible_cpu(cpu) { > + reg_evt = per_cpu_ptr(event->local, cpu); > + sse_event_free_registered(reg_evt); > + } > + free_percpu(event->local); > + } > + > + kfree(event); > +} > + > +int sse_event_enable(struct sse_event *event) > +{ > + int ret = 0; > + struct sse_per_cpu_evt cpu_evt; > + > + scoped_guard(mutex, &sse_mutex) { > + cpus_read_lock(); > + if (sse_event_is_global(event->evt)) { > + ret = sse_sbi_enable_event(event); > + } else { > + cpu_evt.event = event; > + atomic_set(&cpu_evt.error, 0); > + cpu_evt.func = SBI_SSE_EVENT_ENABLE; > + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); > + ret = atomic_read(&cpu_evt.error); > + if (ret) { > + cpu_evt.func = SBI_SSE_EVENT_DISABLE; > + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, > + 1); > + } > + } > + cpus_read_unlock(); > + > + if (ret == 0) > + event->is_enabled = true; > + } > + > + return ret; > +} > + > +static void sse_events_mask(void) > +{ > + sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_MASK, 0, 0, 0, 0, 0, 0); > +} > + > +static void sse_events_unmask(void) > +{ > + sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_UNMASK, 0, 0, 0, 0, 0, 0); > +} > + > +static void sse_event_disable_nolock(struct sse_event *event) > +{ > + struct sse_per_cpu_evt cpu_evt; > + > + if (sse_event_is_global(event->evt)) { > + sse_sbi_disable_event(event); > + } else { > + cpu_evt.event = event; > + cpu_evt.func = SBI_SSE_EVENT_DISABLE; > + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); > + } > +} > + > +void sse_event_disable(struct sse_event *event) > +{ > + scoped_guard(mutex, &sse_mutex) { > + cpus_read_lock(); > + sse_event_disable_nolock(event); > + event->is_enabled = false; > + cpus_read_unlock(); > + } > +} > + > +struct sse_event *sse_event_register(u32 evt, u32 priority, > + sse_event_handler *handler, void *arg) > +{ > + struct sse_per_cpu_evt cpu_evt; > + struct sse_event *event; > + int ret = 0; > + > + if (!sse_available) > + return ERR_PTR(-EOPNOTSUPP); > + > + mutex_lock(&sse_mutex); > + if (sse_event_get(evt)) { > + pr_debug("Event %x already registered\n", evt); > + ret = -EEXIST; > + goto out_unlock; > + } > + > + event = sse_event_alloc(evt, priority, handler, arg); > + if (IS_ERR(event)) { > + ret = PTR_ERR(event); > + goto out_unlock; > + } > + > + cpus_read_lock(); > + if (sse_event_is_global(evt)) { > + unsigned long preferred_hart; > + > + ret = sse_event_attr_get_no_lock(event->global, > + SBI_SSE_ATTR_PREFERRED_HART, > + &preferred_hart); > + if (ret) > + goto err_event_free; > + event->cpu = riscv_hartid_to_cpuid(preferred_hart); > + > + ret = sse_sbi_register_event(event, event->global); > + if (ret) > + goto err_event_free; > + > + } else { > + cpu_evt.event = event; > + atomic_set(&cpu_evt.error, 0); > + cpu_evt.func = SBI_SSE_EVENT_REGISTER; > + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); > + ret = atomic_read(&cpu_evt.error); > + if (ret) { > + cpu_evt.func = SBI_SSE_EVENT_UNREGISTER; > + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); > + goto err_event_free; > + } > + } > + cpus_read_unlock(); > + > + scoped_guard(spinlock, &events_list_lock) > + list_add(&event->list, &events); > + > + mutex_unlock(&sse_mutex); > + > + return event; > + > +err_event_free: > + cpus_read_unlock(); > + sse_event_free(event); > +out_unlock: > + mutex_unlock(&sse_mutex); > + > + return ERR_PTR(ret); > +} > + > +static void sse_event_unregister_nolock(struct sse_event *event) > +{ > + struct sse_per_cpu_evt cpu_evt; > + > + if (sse_event_is_global(event->evt)) { > + sse_sbi_unregister_event(event); > + } else { > + cpu_evt.event = event; > + cpu_evt.func = SBI_SSE_EVENT_UNREGISTER; > + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); > + } > +} > + > +void sse_event_unregister(struct sse_event *event) > +{ > + scoped_guard(mutex, &sse_mutex) { > + cpus_read_lock(); > + sse_event_unregister_nolock(event); > + cpus_read_unlock(); > + > + scoped_guard(spinlock, &events_list_lock) > + list_del(&event->list); > + > + sse_event_free(event); > + } > +} > + > +static int sse_cpu_online(unsigned int cpu) > +{ > + struct sse_event *sse_evt; > + > + scoped_guard(spinlock, &events_list_lock) { > + list_for_each_entry(sse_evt, &events, list) { > + if (sse_event_is_global(sse_evt->evt)) > + continue; > + > + sse_event_register_local(sse_evt); > + if (sse_evt->is_enabled) > + sse_sbi_enable_event(sse_evt); > + } > + } > + > + /* Ready to handle events. Unmask SSE. */ > + sse_events_unmask(); > + > + return 0; > +} > + > +static int sse_cpu_teardown(unsigned int cpu) > +{ > + unsigned int next_cpu; > + struct sse_event *sse_evt; > + > + /* Mask the sse events */ > + sse_events_mask(); > + > + scoped_guard(spinlock, &events_list_lock) { > + list_for_each_entry(sse_evt, &events, list) { > + if (!sse_event_is_global(sse_evt->evt)) { > + > + if (sse_evt->is_enabled) > + sse_sbi_disable_event(sse_evt); > + > + sse_sbi_unregister_event(sse_evt); > + continue; > + } > + > + if (sse_evt->cpu != smp_processor_id()) > + continue; > + > + /* Update destination hart for global event */ > + next_cpu = cpumask_any_but(cpu_online_mask, cpu); > + sse_event_set_target_cpu_nolock(sse_evt, next_cpu); > + } > + } > + > + return 0; > +} > + > +static void sse_reset(void) > +{ > + struct sse_event *event = NULL; > + > + list_for_each_entry(event, &events, list) { > + sse_event_disable_nolock(event); > + sse_event_unregister_nolock(event); > + } > +} > + > +static int sse_pm_notifier(struct notifier_block *nb, unsigned long action, > + void *data) > +{ > + WARN_ON_ONCE(preemptible()); > + > + switch (action) { > + case CPU_PM_ENTER: > + sse_events_mask(); > + break; > + case CPU_PM_EXIT: > + case CPU_PM_ENTER_FAILED: > + sse_events_unmask(); > + break; > + default: > + return NOTIFY_DONE; > + } > + > + return NOTIFY_OK; > +} > + > +static struct notifier_block sse_pm_nb = { > + .notifier_call = sse_pm_notifier, > +}; > + > +/* > + * Mask all CPUs and unregister all events on panic, reboot or kexec. > + */ > +static int sse_reboot_notifier(struct notifier_block *nb, unsigned long action, > + void *data) > +{ > + cpuhp_remove_state(sse_hp_state); > + > + sse_reset(); > + > + return NOTIFY_OK; > +} > + > +static struct notifier_block sse_reboot_nb = { > + .notifier_call = sse_reboot_notifier, > +}; > + > +static int __init sse_init(void) > +{ > + int cpu, ret; > + > + if (sbi_probe_extension(SBI_EXT_SSE) <= 0) { > + pr_err("Missing SBI SSE extension\n"); > + return -EOPNOTSUPP; > + } > + pr_info("SBI SSE extension detected\n"); > + > + for_each_possible_cpu(cpu) > + INIT_LIST_HEAD(&events); > + > + ret = cpu_pm_register_notifier(&sse_pm_nb); > + if (ret) { > + pr_warn("Failed to register CPU PM notifier...\n"); > + return ret; > + } > + > + ret = register_reboot_notifier(&sse_reboot_nb); > + if (ret) { > + pr_warn("Failed to register reboot notifier...\n"); > + goto remove_cpupm; > + } > + > + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/sse:online", > + sse_cpu_online, sse_cpu_teardown); > + if (ret < 0) > + goto remove_reboot; > + > + sse_hp_state = ret; > + sse_available = true; > + > + return 0; > + > +remove_reboot: > + unregister_reboot_notifier(&sse_reboot_nb); > + > +remove_cpupm: > + cpu_pm_unregister_notifier(&sse_pm_nb); > + > + return ret; > +} > +arch_initcall(sse_init); > diff --git a/include/linux/riscv_sse.h b/include/linux/riscv_sse.h > new file mode 100644 > index 000000000000..c73184074b8c > --- /dev/null > +++ b/include/linux/riscv_sse.h > @@ -0,0 +1,56 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +/* > + * Copyright (C) 2024 Rivos Inc. > + */ > + > +#ifndef __LINUX_RISCV_SSE_H > +#define __LINUX_RISCV_SSE_H > + > +#include <linux/types.h> > +#include <linux/linkage.h> > + > +struct sse_event; > +struct pt_regs; > + > +typedef int (sse_event_handler)(u32 event_num, void *arg, struct pt_regs *regs); > + > +#ifdef CONFIG_RISCV_SSE > + > +struct sse_event *sse_event_register(u32 event_num, u32 priority, > + sse_event_handler *handler, void *arg); > + > +void sse_event_unregister(struct sse_event *evt); > + > +int sse_event_set_target_cpu(struct sse_event *sse_evt, unsigned int cpu); > + > +int sse_event_enable(struct sse_event *sse_evt); > + > +void sse_event_disable(struct sse_event *sse_evt); > + > +#else > +static inline struct sse_event *sse_event_register(u32 event_num, u32 priority, > + sse_event_handler *handler, > + void *arg) > +{ > + return ERR_PTR(-EOPNOTSUPP); > +} > + > +static inline void sse_event_unregister(struct sse_event *evt) {} > + > +static inline int sse_event_set_target_cpu(struct sse_event *sse_evt, > + unsigned int cpu) > +{ > + return -EOPNOTSUPP; > +} > + > +static inline int sse_event_enable(struct sse_event *sse_evt) > +{ > + return -EOPNOTSUPP; > +} > + > +static inline void sse_event_disable(struct sse_event *sse_evt) {} > + > + > +#endif > + > +#endif /* __LINUX_RISCV_SSE_H */ > -- > 2.45.2 >
On 13/12/2024 06:03, Himanshu Chauhan wrote: > Hi Clement, > > On Fri, Dec 06, 2024 at 05:30:59PM +0100, Clément Léger wrote: >> Add driver level interface to use RISC-V SSE arch support. This interface >> allows registering SSE handlers, and receive them. This will be used by >> PMU and GHES driver. >> >> Signed-off-by: Himanshu Chauhan <hchauhan@ventanamicro.com> >> Co-developed-by: Himanshu Chauhan <hchauhan@ventanamicro.com> >> Signed-off-by: Clément Léger <cleger@rivosinc.com> >> --- >> MAINTAINERS | 14 + >> drivers/firmware/Kconfig | 1 + >> drivers/firmware/Makefile | 1 + >> drivers/firmware/riscv/Kconfig | 15 + >> drivers/firmware/riscv/Makefile | 3 + >> drivers/firmware/riscv/riscv_sse.c | 691 +++++++++++++++++++++++++++++ >> include/linux/riscv_sse.h | 56 +++ >> 7 files changed, 781 insertions(+) >> create mode 100644 drivers/firmware/riscv/Kconfig >> create mode 100644 drivers/firmware/riscv/Makefile >> create mode 100644 drivers/firmware/riscv/riscv_sse.c >> create mode 100644 include/linux/riscv_sse.h >> >> diff --git a/MAINTAINERS b/MAINTAINERS >> index 686109008d8e..a3ddde7fe9fb 100644 >> --- a/MAINTAINERS >> +++ b/MAINTAINERS >> @@ -20125,6 +20125,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git >> F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml >> F: drivers/iommu/riscv/ >> >> +RISC-V FIRMWARE DRIVERS >> +M: Conor Dooley <conor@kernel.org> >> +L: linux-riscv@lists.infradead.org >> +S: Maintained >> +T: git git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git >> +F: drivers/firmware/riscv/* >> + >> RISC-V MICROCHIP FPGA SUPPORT >> M: Conor Dooley <conor.dooley@microchip.com> >> M: Daire McNamara <daire.mcnamara@microchip.com> >> @@ -20177,6 +20184,13 @@ F: drivers/perf/riscv_pmu.c >> F: drivers/perf/riscv_pmu_legacy.c >> F: drivers/perf/riscv_pmu_sbi.c >> >> +RISC-V SSE DRIVER >> +M: Clément Léger <cleger@rivosinc.com> >> +L: linux-riscv@lists.infradead.org >> +S: Maintained >> +F: drivers/firmware/riscv/riscv_sse.c >> +F: include/linux/riscv_sse.h >> + > > I request you to add me as a reviewer to these SSE files. > Himanshu Chauhan <himanshu@thechauhan.dev> Oh yes sure ! Thanks, Clément > > Thanks > Regards > Himanshu > >> RISC-V THEAD SoC SUPPORT >> M: Drew Fustini <drew@pdp7.com> >> M: Guo Ren <guoren@kernel.org> >> diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig >> index 71d8b26c4103..9e996a1fd511 100644 >> --- a/drivers/firmware/Kconfig >> +++ b/drivers/firmware/Kconfig >> @@ -267,6 +267,7 @@ source "drivers/firmware/meson/Kconfig" >> source "drivers/firmware/microchip/Kconfig" >> source "drivers/firmware/psci/Kconfig" >> source "drivers/firmware/qcom/Kconfig" >> +source "drivers/firmware/riscv/Kconfig" >> source "drivers/firmware/smccc/Kconfig" >> source "drivers/firmware/tegra/Kconfig" >> source "drivers/firmware/xilinx/Kconfig" >> diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile >> index 7a8d486e718f..c0f5009949a8 100644 >> --- a/drivers/firmware/Makefile >> +++ b/drivers/firmware/Makefile >> @@ -33,6 +33,7 @@ obj-y += efi/ >> obj-y += imx/ >> obj-y += psci/ >> obj-y += qcom/ >> +obj-y += riscv/ >> obj-y += smccc/ >> obj-y += tegra/ >> obj-y += xilinx/ >> diff --git a/drivers/firmware/riscv/Kconfig b/drivers/firmware/riscv/Kconfig >> new file mode 100644 >> index 000000000000..8056ed3262d9 >> --- /dev/null >> +++ b/drivers/firmware/riscv/Kconfig >> @@ -0,0 +1,15 @@ >> +# SPDX-License-Identifier: GPL-2.0-only >> +menu "Risc-V Specific firmware drivers" >> +depends on RISCV >> + >> +config RISCV_SSE >> + bool "Enable SBI Supervisor Software Events support" >> + depends on RISCV_SBI >> + default y >> + help >> + The Supervisor Software Events support allow the SBI to deliver >> + NMI-like notifications to the supervisor mode software. When enable, >> + this option provides support to register callbacks on specific SSE >> + events. >> + >> +endmenu >> diff --git a/drivers/firmware/riscv/Makefile b/drivers/firmware/riscv/Makefile >> new file mode 100644 >> index 000000000000..4ccfcbbc28ea >> --- /dev/null >> +++ b/drivers/firmware/riscv/Makefile >> @@ -0,0 +1,3 @@ >> +# SPDX-License-Identifier: GPL-2.0 >> + >> +obj-$(CONFIG_RISCV_SSE) += riscv_sse.o >> diff --git a/drivers/firmware/riscv/riscv_sse.c b/drivers/firmware/riscv/riscv_sse.c >> new file mode 100644 >> index 000000000000..c165e32cc9a5 >> --- /dev/null >> +++ b/drivers/firmware/riscv/riscv_sse.c >> @@ -0,0 +1,691 @@ >> +// SPDX-License-Identifier: GPL-2.0-only >> +/* >> + * Copyright (C) 2024 Rivos Inc. >> + */ >> + >> +#define pr_fmt(fmt) "sse: " fmt >> + >> +#include <linux/cpu.h> >> +#include <linux/cpuhotplug.h> >> +#include <linux/cpu_pm.h> >> +#include <linux/hardirq.h> >> +#include <linux/list.h> >> +#include <linux/percpu-defs.h> >> +#include <linux/reboot.h> >> +#include <linux/riscv_sse.h> >> +#include <linux/slab.h> >> + >> +#include <asm/sbi.h> >> +#include <asm/sse.h> >> + >> +struct sse_event { >> + struct list_head list; >> + u32 evt; >> + u32 priority; >> + sse_event_handler *handler; >> + void *handler_arg; >> + bool is_enabled; >> + /* Only valid for global events */ >> + unsigned int cpu; >> + >> + union { >> + struct sse_registered_event *global; >> + struct sse_registered_event __percpu *local; >> + }; >> +}; >> + >> +static int sse_hp_state; >> +static bool sse_available; >> +static DEFINE_SPINLOCK(events_list_lock); >> +static LIST_HEAD(events); >> +static DEFINE_MUTEX(sse_mutex); >> + >> +struct sse_registered_event { >> + struct sse_event_arch_data arch; >> + struct sse_event *evt; >> + unsigned long attr_buf; >> +}; >> + >> +void sse_handle_event(struct sse_event_arch_data *arch_event, >> + struct pt_regs *regs) >> +{ >> + int ret; >> + struct sse_registered_event *reg_evt = >> + container_of(arch_event, struct sse_registered_event, arch); >> + struct sse_event *evt = reg_evt->evt; >> + >> + ret = evt->handler(evt->evt, evt->handler_arg, regs); >> + if (ret) >> + pr_warn("event %x handler failed with error %d\n", evt->evt, >> + ret); >> +} >> + >> +static bool sse_event_is_global(u32 evt) >> +{ >> + return !!(evt & SBI_SSE_EVENT_GLOBAL); >> +} >> + >> +static >> +struct sse_event *sse_event_get(u32 evt) >> +{ >> + struct sse_event *sse_evt = NULL, *tmp; >> + >> + scoped_guard(spinlock, &events_list_lock) { >> + list_for_each_entry(tmp, &events, list) { >> + if (tmp->evt == evt) { >> + return sse_evt; >> + } >> + } >> + } >> + >> + return NULL; >> +} >> + >> +static phys_addr_t sse_event_get_phys(struct sse_registered_event *reg_evt, >> + void *addr) >> +{ >> + phys_addr_t phys; >> + >> + if (sse_event_is_global(reg_evt->evt->evt)) >> + phys = virt_to_phys(addr); >> + else >> + phys = per_cpu_ptr_to_phys(addr); >> + >> + return phys; >> +} >> + >> +static int sse_sbi_event_func(struct sse_event *event, unsigned long func) >> +{ >> + struct sbiret ret; >> + u32 evt = event->evt; >> + >> + ret = sbi_ecall(SBI_EXT_SSE, func, evt, 0, 0, 0, 0, 0); >> + if (ret.error) >> + pr_debug("Failed to execute func %lx, event %x, error %ld\n", >> + func, evt, ret.error); >> + >> + return sbi_err_map_linux_errno(ret.error); >> +} >> + >> +static int sse_sbi_disable_event(struct sse_event *event) >> +{ >> + return sse_sbi_event_func(event, SBI_SSE_EVENT_DISABLE); >> +} >> + >> +static int sse_sbi_enable_event(struct sse_event *event) >> +{ >> + return sse_sbi_event_func(event, SBI_SSE_EVENT_ENABLE); >> +} >> + >> +static int sse_event_attr_get_no_lock(struct sse_registered_event *reg_evt, >> + unsigned long attr_id, unsigned long *val) >> +{ >> + struct sbiret sret; >> + u32 evt = reg_evt->evt->evt; >> + unsigned long phys; >> + >> + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); >> + >> + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, >> + attr_id, 1, phys, 0, 0); >> + if (sret.error) { >> + pr_debug("Failed to get event %x attr %lx, error %ld\n", evt, >> + attr_id, sret.error); >> + return sbi_err_map_linux_errno(sret.error); >> + } >> + >> + *val = reg_evt->attr_buf; >> + >> + return 0; >> +} >> + >> +static int sse_event_attr_set_nolock(struct sse_registered_event *reg_evt, >> + unsigned long attr_id, unsigned long val) >> +{ >> + struct sbiret sret; >> + u32 evt = reg_evt->evt->evt; >> + unsigned long phys; >> + >> + reg_evt->attr_buf = val; >> + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); >> + >> + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, evt, >> + attr_id, 1, phys, 0, 0); >> + if (sret.error && sret.error != SBI_ERR_INVALID_STATE) { >> + pr_debug("Failed to set event %x attr %lx, error %ld\n", evt, >> + attr_id, sret.error); >> + return sbi_err_map_linux_errno(sret.error); >> + } >> + >> + return 0; >> +} >> + >> +static int sse_event_set_target_cpu_nolock(struct sse_event *event, >> + unsigned int cpu) >> +{ >> + unsigned int hart_id = cpuid_to_hartid_map(cpu); >> + struct sse_registered_event *reg_evt = event->global; >> + u32 evt = event->evt; >> + bool was_enabled; >> + int ret; >> + >> + if (!sse_event_is_global(evt)) >> + return -EINVAL; >> + >> + was_enabled = event->is_enabled; >> + if (was_enabled) >> + sse_sbi_disable_event(event); >> + do { >> + ret = sse_event_attr_set_nolock(reg_evt, >> + SBI_SSE_ATTR_PREFERRED_HART, >> + hart_id); >> + } while (ret == -EINVAL); >> + >> + if (ret == 0) >> + event->cpu = cpu; >> + >> + if (was_enabled) >> + sse_sbi_enable_event(event); >> + >> + return 0; >> +} >> + >> +int sse_event_set_target_cpu(struct sse_event *event, unsigned int cpu) >> +{ >> + int ret; >> + >> + scoped_guard(mutex, &sse_mutex) { >> + cpus_read_lock(); >> + >> + if (!cpu_online(cpu)) >> + return -EINVAL; >> + >> + ret = sse_event_set_target_cpu_nolock(event, cpu); >> + >> + cpus_read_unlock(); >> + } >> + >> + return ret; >> +} >> + >> +static int sse_event_init_registered(unsigned int cpu, >> + struct sse_registered_event *reg_evt, >> + struct sse_event *event) >> +{ >> + reg_evt->evt = event; >> + arch_sse_init_event(®_evt->arch, event->evt, cpu); >> + >> + return 0; >> +} >> + >> +static void sse_event_free_registered(struct sse_registered_event *reg_evt) >> +{ >> + arch_sse_free_event(®_evt->arch); >> +} >> + >> +static int sse_event_alloc_global(struct sse_event *event) >> +{ >> + int err; >> + struct sse_registered_event *reg_evt; >> + >> + reg_evt = kzalloc(sizeof(*reg_evt), GFP_KERNEL); >> + if (!reg_evt) >> + return -ENOMEM; >> + >> + event->global = reg_evt; >> + err = sse_event_init_registered(smp_processor_id(), reg_evt, >> + event); >> + if (err) >> + kfree(reg_evt); >> + >> + return err; >> +} >> + >> +static int sse_event_alloc_local(struct sse_event *event) >> +{ >> + int err; >> + unsigned int cpu, err_cpu; >> + struct sse_registered_event *reg_evt; >> + struct sse_registered_event __percpu *reg_evts; >> + >> + reg_evts = alloc_percpu(struct sse_registered_event); >> + if (!reg_evts) >> + return -ENOMEM; >> + >> + event->local = reg_evts; >> + >> + for_each_possible_cpu(cpu) { >> + reg_evt = per_cpu_ptr(reg_evts, cpu); >> + err = sse_event_init_registered(cpu, reg_evt, event); >> + if (err) { >> + err_cpu = cpu; >> + goto err_free_per_cpu; >> + } >> + } >> + >> + return 0; >> + >> +err_free_per_cpu: >> + for_each_possible_cpu(cpu) { >> + if (cpu == err_cpu) >> + break; >> + reg_evt = per_cpu_ptr(reg_evts, cpu); >> + sse_event_free_registered(reg_evt); >> + } >> + >> + free_percpu(reg_evts); >> + >> + return err; >> +} >> + >> +static struct sse_event *sse_event_alloc(u32 evt, >> + u32 priority, >> + sse_event_handler *handler, void *arg) >> +{ >> + int err; >> + struct sse_event *event; >> + >> + event = kzalloc(sizeof(*event), GFP_KERNEL); >> + if (!event) >> + return ERR_PTR(-ENOMEM); >> + >> + event->evt = evt; >> + event->priority = priority; >> + event->handler_arg = arg; >> + event->handler = handler; >> + >> + if (sse_event_is_global(evt)) { >> + err = sse_event_alloc_global(event); >> + if (err) >> + goto err_alloc_reg_evt; >> + } else { >> + err = sse_event_alloc_local(event); >> + if (err) >> + goto err_alloc_reg_evt; >> + } >> + >> + return event; >> + >> +err_alloc_reg_evt: >> + kfree(event); >> + >> + return ERR_PTR(err); >> +} >> + >> +static int sse_sbi_register_event(struct sse_event *event, >> + struct sse_registered_event *reg_evt) >> +{ >> + int ret; >> + >> + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PRIO, >> + event->priority); >> + if (ret) >> + return ret; >> + >> + return arch_sse_register_event(®_evt->arch); >> +} >> + >> +static int sse_event_register_local(struct sse_event *event) >> +{ >> + int ret; >> + struct sse_registered_event *reg_evt = per_cpu_ptr(event->local, >> + smp_processor_id()); >> + >> + ret = sse_sbi_register_event(event, reg_evt); >> + if (ret) >> + pr_debug("Failed to register event %x: err %d\n", event->evt, >> + ret); >> + >> + return ret; >> +} >> + >> + >> +static int sse_sbi_unregister_event(struct sse_event *event) >> +{ >> + return sse_sbi_event_func(event, SBI_SSE_EVENT_UNREGISTER); >> +} >> + >> +struct sse_per_cpu_evt { >> + struct sse_event *event; >> + unsigned long func; >> + atomic_t error; >> +}; >> + >> +static void sse_event_per_cpu_func(void *info) >> +{ >> + int ret; >> + struct sse_per_cpu_evt *cpu_evt = info; >> + >> + if (cpu_evt->func == SBI_SSE_EVENT_REGISTER) >> + ret = sse_event_register_local(cpu_evt->event); >> + else >> + ret = sse_sbi_event_func(cpu_evt->event, cpu_evt->func); >> + >> + if (ret) >> + atomic_set(&cpu_evt->error, ret); >> +} >> + >> +static void sse_event_free(struct sse_event *event) >> +{ >> + unsigned int cpu; >> + struct sse_registered_event *reg_evt; >> + >> + if (sse_event_is_global(event->evt)) { >> + sse_event_free_registered(event->global); >> + kfree(event->global); >> + } else { >> + for_each_possible_cpu(cpu) { >> + reg_evt = per_cpu_ptr(event->local, cpu); >> + sse_event_free_registered(reg_evt); >> + } >> + free_percpu(event->local); >> + } >> + >> + kfree(event); >> +} >> + >> +int sse_event_enable(struct sse_event *event) >> +{ >> + int ret = 0; >> + struct sse_per_cpu_evt cpu_evt; >> + >> + scoped_guard(mutex, &sse_mutex) { >> + cpus_read_lock(); >> + if (sse_event_is_global(event->evt)) { >> + ret = sse_sbi_enable_event(event); >> + } else { >> + cpu_evt.event = event; >> + atomic_set(&cpu_evt.error, 0); >> + cpu_evt.func = SBI_SSE_EVENT_ENABLE; >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); >> + ret = atomic_read(&cpu_evt.error); >> + if (ret) { >> + cpu_evt.func = SBI_SSE_EVENT_DISABLE; >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, >> + 1); >> + } >> + } >> + cpus_read_unlock(); >> + >> + if (ret == 0) >> + event->is_enabled = true; >> + } >> + >> + return ret; >> +} >> + >> +static void sse_events_mask(void) >> +{ >> + sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_MASK, 0, 0, 0, 0, 0, 0); >> +} >> + >> +static void sse_events_unmask(void) >> +{ >> + sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_UNMASK, 0, 0, 0, 0, 0, 0); >> +} >> + >> +static void sse_event_disable_nolock(struct sse_event *event) >> +{ >> + struct sse_per_cpu_evt cpu_evt; >> + >> + if (sse_event_is_global(event->evt)) { >> + sse_sbi_disable_event(event); >> + } else { >> + cpu_evt.event = event; >> + cpu_evt.func = SBI_SSE_EVENT_DISABLE; >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); >> + } >> +} >> + >> +void sse_event_disable(struct sse_event *event) >> +{ >> + scoped_guard(mutex, &sse_mutex) { >> + cpus_read_lock(); >> + sse_event_disable_nolock(event); >> + event->is_enabled = false; >> + cpus_read_unlock(); >> + } >> +} >> + >> +struct sse_event *sse_event_register(u32 evt, u32 priority, >> + sse_event_handler *handler, void *arg) >> +{ >> + struct sse_per_cpu_evt cpu_evt; >> + struct sse_event *event; >> + int ret = 0; >> + >> + if (!sse_available) >> + return ERR_PTR(-EOPNOTSUPP); >> + >> + mutex_lock(&sse_mutex); >> + if (sse_event_get(evt)) { >> + pr_debug("Event %x already registered\n", evt); >> + ret = -EEXIST; >> + goto out_unlock; >> + } >> + >> + event = sse_event_alloc(evt, priority, handler, arg); >> + if (IS_ERR(event)) { >> + ret = PTR_ERR(event); >> + goto out_unlock; >> + } >> + >> + cpus_read_lock(); >> + if (sse_event_is_global(evt)) { >> + unsigned long preferred_hart; >> + >> + ret = sse_event_attr_get_no_lock(event->global, >> + SBI_SSE_ATTR_PREFERRED_HART, >> + &preferred_hart); >> + if (ret) >> + goto err_event_free; >> + event->cpu = riscv_hartid_to_cpuid(preferred_hart); >> + >> + ret = sse_sbi_register_event(event, event->global); >> + if (ret) >> + goto err_event_free; >> + >> + } else { >> + cpu_evt.event = event; >> + atomic_set(&cpu_evt.error, 0); >> + cpu_evt.func = SBI_SSE_EVENT_REGISTER; >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); >> + ret = atomic_read(&cpu_evt.error); >> + if (ret) { >> + cpu_evt.func = SBI_SSE_EVENT_UNREGISTER; >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); >> + goto err_event_free; >> + } >> + } >> + cpus_read_unlock(); >> + >> + scoped_guard(spinlock, &events_list_lock) >> + list_add(&event->list, &events); >> + >> + mutex_unlock(&sse_mutex); >> + >> + return event; >> + >> +err_event_free: >> + cpus_read_unlock(); >> + sse_event_free(event); >> +out_unlock: >> + mutex_unlock(&sse_mutex); >> + >> + return ERR_PTR(ret); >> +} >> + >> +static void sse_event_unregister_nolock(struct sse_event *event) >> +{ >> + struct sse_per_cpu_evt cpu_evt; >> + >> + if (sse_event_is_global(event->evt)) { >> + sse_sbi_unregister_event(event); >> + } else { >> + cpu_evt.event = event; >> + cpu_evt.func = SBI_SSE_EVENT_UNREGISTER; >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); >> + } >> +} >> + >> +void sse_event_unregister(struct sse_event *event) >> +{ >> + scoped_guard(mutex, &sse_mutex) { >> + cpus_read_lock(); >> + sse_event_unregister_nolock(event); >> + cpus_read_unlock(); >> + >> + scoped_guard(spinlock, &events_list_lock) >> + list_del(&event->list); >> + >> + sse_event_free(event); >> + } >> +} >> + >> +static int sse_cpu_online(unsigned int cpu) >> +{ >> + struct sse_event *sse_evt; >> + >> + scoped_guard(spinlock, &events_list_lock) { >> + list_for_each_entry(sse_evt, &events, list) { >> + if (sse_event_is_global(sse_evt->evt)) >> + continue; >> + >> + sse_event_register_local(sse_evt); >> + if (sse_evt->is_enabled) >> + sse_sbi_enable_event(sse_evt); >> + } >> + } >> + >> + /* Ready to handle events. Unmask SSE. */ >> + sse_events_unmask(); >> + >> + return 0; >> +} >> + >> +static int sse_cpu_teardown(unsigned int cpu) >> +{ >> + unsigned int next_cpu; >> + struct sse_event *sse_evt; >> + >> + /* Mask the sse events */ >> + sse_events_mask(); >> + >> + scoped_guard(spinlock, &events_list_lock) { >> + list_for_each_entry(sse_evt, &events, list) { >> + if (!sse_event_is_global(sse_evt->evt)) { >> + >> + if (sse_evt->is_enabled) >> + sse_sbi_disable_event(sse_evt); >> + >> + sse_sbi_unregister_event(sse_evt); >> + continue; >> + } >> + >> + if (sse_evt->cpu != smp_processor_id()) >> + continue; >> + >> + /* Update destination hart for global event */ >> + next_cpu = cpumask_any_but(cpu_online_mask, cpu); >> + sse_event_set_target_cpu_nolock(sse_evt, next_cpu); >> + } >> + } >> + >> + return 0; >> +} >> + >> +static void sse_reset(void) >> +{ >> + struct sse_event *event = NULL; >> + >> + list_for_each_entry(event, &events, list) { >> + sse_event_disable_nolock(event); >> + sse_event_unregister_nolock(event); >> + } >> +} >> + >> +static int sse_pm_notifier(struct notifier_block *nb, unsigned long action, >> + void *data) >> +{ >> + WARN_ON_ONCE(preemptible()); >> + >> + switch (action) { >> + case CPU_PM_ENTER: >> + sse_events_mask(); >> + break; >> + case CPU_PM_EXIT: >> + case CPU_PM_ENTER_FAILED: >> + sse_events_unmask(); >> + break; >> + default: >> + return NOTIFY_DONE; >> + } >> + >> + return NOTIFY_OK; >> +} >> + >> +static struct notifier_block sse_pm_nb = { >> + .notifier_call = sse_pm_notifier, >> +}; >> + >> +/* >> + * Mask all CPUs and unregister all events on panic, reboot or kexec. >> + */ >> +static int sse_reboot_notifier(struct notifier_block *nb, unsigned long action, >> + void *data) >> +{ >> + cpuhp_remove_state(sse_hp_state); >> + >> + sse_reset(); >> + >> + return NOTIFY_OK; >> +} >> + >> +static struct notifier_block sse_reboot_nb = { >> + .notifier_call = sse_reboot_notifier, >> +}; >> + >> +static int __init sse_init(void) >> +{ >> + int cpu, ret; >> + >> + if (sbi_probe_extension(SBI_EXT_SSE) <= 0) { >> + pr_err("Missing SBI SSE extension\n"); >> + return -EOPNOTSUPP; >> + } >> + pr_info("SBI SSE extension detected\n"); >> + >> + for_each_possible_cpu(cpu) >> + INIT_LIST_HEAD(&events); >> + >> + ret = cpu_pm_register_notifier(&sse_pm_nb); >> + if (ret) { >> + pr_warn("Failed to register CPU PM notifier...\n"); >> + return ret; >> + } >> + >> + ret = register_reboot_notifier(&sse_reboot_nb); >> + if (ret) { >> + pr_warn("Failed to register reboot notifier...\n"); >> + goto remove_cpupm; >> + } >> + >> + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/sse:online", >> + sse_cpu_online, sse_cpu_teardown); >> + if (ret < 0) >> + goto remove_reboot; >> + >> + sse_hp_state = ret; >> + sse_available = true; >> + >> + return 0; >> + >> +remove_reboot: >> + unregister_reboot_notifier(&sse_reboot_nb); >> + >> +remove_cpupm: >> + cpu_pm_unregister_notifier(&sse_pm_nb); >> + >> + return ret; >> +} >> +arch_initcall(sse_init); >> diff --git a/include/linux/riscv_sse.h b/include/linux/riscv_sse.h >> new file mode 100644 >> index 000000000000..c73184074b8c >> --- /dev/null >> +++ b/include/linux/riscv_sse.h >> @@ -0,0 +1,56 @@ >> +/* SPDX-License-Identifier: GPL-2.0 */ >> +/* >> + * Copyright (C) 2024 Rivos Inc. >> + */ >> + >> +#ifndef __LINUX_RISCV_SSE_H >> +#define __LINUX_RISCV_SSE_H >> + >> +#include <linux/types.h> >> +#include <linux/linkage.h> >> + >> +struct sse_event; >> +struct pt_regs; >> + >> +typedef int (sse_event_handler)(u32 event_num, void *arg, struct pt_regs *regs); >> + >> +#ifdef CONFIG_RISCV_SSE >> + >> +struct sse_event *sse_event_register(u32 event_num, u32 priority, >> + sse_event_handler *handler, void *arg); >> + >> +void sse_event_unregister(struct sse_event *evt); >> + >> +int sse_event_set_target_cpu(struct sse_event *sse_evt, unsigned int cpu); >> + >> +int sse_event_enable(struct sse_event *sse_evt); >> + >> +void sse_event_disable(struct sse_event *sse_evt); >> + >> +#else >> +static inline struct sse_event *sse_event_register(u32 event_num, u32 priority, >> + sse_event_handler *handler, >> + void *arg) >> +{ >> + return ERR_PTR(-EOPNOTSUPP); >> +} >> + >> +static inline void sse_event_unregister(struct sse_event *evt) {} >> + >> +static inline int sse_event_set_target_cpu(struct sse_event *sse_evt, >> + unsigned int cpu) >> +{ >> + return -EOPNOTSUPP; >> +} >> + >> +static inline int sse_event_enable(struct sse_event *sse_evt) >> +{ >> + return -EOPNOTSUPP; >> +} >> + >> +static inline void sse_event_disable(struct sse_event *sse_evt) {} >> + >> + >> +#endif >> + >> +#endif /* __LINUX_RISCV_SSE_H */ >> -- >> 2.45.2 >>
On Fri, Dec 06, 2024 at 05:30:59PM +0100, Clément Léger wrote: > Add driver level interface to use RISC-V SSE arch support. This interface > allows registering SSE handlers, and receive them. This will be used by > PMU and GHES driver. > > Signed-off-by: Himanshu Chauhan <hchauhan@ventanamicro.com> > Co-developed-by: Himanshu Chauhan <hchauhan@ventanamicro.com> > Signed-off-by: Clément Léger <cleger@rivosinc.com> > --- > MAINTAINERS | 14 + > drivers/firmware/Kconfig | 1 + > drivers/firmware/Makefile | 1 + > drivers/firmware/riscv/Kconfig | 15 + > drivers/firmware/riscv/Makefile | 3 + > drivers/firmware/riscv/riscv_sse.c | 691 +++++++++++++++++++++++++++++ > include/linux/riscv_sse.h | 56 +++ > 7 files changed, 781 insertions(+) > create mode 100644 drivers/firmware/riscv/Kconfig > create mode 100644 drivers/firmware/riscv/Makefile > create mode 100644 drivers/firmware/riscv/riscv_sse.c > create mode 100644 include/linux/riscv_sse.h > > diff --git a/MAINTAINERS b/MAINTAINERS > index 686109008d8e..a3ddde7fe9fb 100644 > --- a/MAINTAINERS > +++ b/MAINTAINERS > @@ -20125,6 +20125,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git > F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml > F: drivers/iommu/riscv/ > > +RISC-V FIRMWARE DRIVERS > +M: Conor Dooley <conor@kernel.org> > +L: linux-riscv@lists.infradead.org > +S: Maintained > +T: git git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git > +F: drivers/firmware/riscv/* Acked-by: Conor Dooley <conor.dooley@microchip.com> (got some, mostly minor, comments below) > diff --git a/drivers/firmware/riscv/Makefile b/drivers/firmware/riscv/Makefile > new file mode 100644 > index 000000000000..4ccfcbbc28ea > --- /dev/null > +++ b/drivers/firmware/riscv/Makefile > @@ -0,0 +1,3 @@ > +# SPDX-License-Identifier: GPL-2.0 > + > +obj-$(CONFIG_RISCV_SSE) += riscv_sse.o > diff --git a/drivers/firmware/riscv/riscv_sse.c b/drivers/firmware/riscv/riscv_sse.c > new file mode 100644 > index 000000000000..c165e32cc9a5 > --- /dev/null > +++ b/drivers/firmware/riscv/riscv_sse.c > @@ -0,0 +1,691 @@ > +// SPDX-License-Identifier: GPL-2.0-only > +/* > + * Copyright (C) 2024 Rivos Inc. > + */ > + > +#define pr_fmt(fmt) "sse: " fmt > + > +#include <linux/cpu.h> > +#include <linux/cpuhotplug.h> > +#include <linux/cpu_pm.h> > +#include <linux/hardirq.h> > +#include <linux/list.h> > +#include <linux/percpu-defs.h> > +#include <linux/reboot.h> > +#include <linux/riscv_sse.h> > +#include <linux/slab.h> > + > +#include <asm/sbi.h> > +#include <asm/sse.h> > + > +struct sse_event { > + struct list_head list; > + u32 evt; > + u32 priority; > + sse_event_handler *handler; > + void *handler_arg; > + bool is_enabled; > + /* Only valid for global events */ > + unsigned int cpu; > + > + union { > + struct sse_registered_event *global; > + struct sse_registered_event __percpu *local; > + }; > +}; > + > +static int sse_hp_state; > +static bool sse_available; > +static DEFINE_SPINLOCK(events_list_lock); > +static LIST_HEAD(events); > +static DEFINE_MUTEX(sse_mutex); > + > +struct sse_registered_event { > + struct sse_event_arch_data arch; > + struct sse_event *evt; > + unsigned long attr_buf; > +}; > + > +void sse_handle_event(struct sse_event_arch_data *arch_event, > + struct pt_regs *regs) > +{ > + int ret; > + struct sse_registered_event *reg_evt = > + container_of(arch_event, struct sse_registered_event, arch); > + struct sse_event *evt = reg_evt->evt; > + > + ret = evt->handler(evt->evt, evt->handler_arg, regs); Is it possible to get here with a null handler? Or will !registered events not lead to the handler getting called? > + if (ret) > + pr_warn("event %x handler failed with error %d\n", evt->evt, > + ret); > +} > + > +static bool sse_event_is_global(u32 evt) > +{ > + return !!(evt & SBI_SSE_EVENT_GLOBAL); > +} > + > +static > +struct sse_event *sse_event_get(u32 evt) nit: Could you shift this into one line? > +{ > + struct sse_event *sse_evt = NULL, *tmp; > + > + scoped_guard(spinlock, &events_list_lock) { > + list_for_each_entry(tmp, &events, list) { > + if (tmp->evt == evt) { > + return sse_evt; > + } > + } > + } > + > + return NULL; > +} > + > +static phys_addr_t sse_event_get_phys(struct sse_registered_event *reg_evt, > + void *addr) > +{ > + phys_addr_t phys; > + > + if (sse_event_is_global(reg_evt->evt->evt)) > + phys = virt_to_phys(addr); > + else > + phys = per_cpu_ptr_to_phys(addr); > + > + return phys; > +} > + > +static int sse_sbi_event_func(struct sse_event *event, unsigned long func) > +{ > + struct sbiret ret; > + u32 evt = event->evt; > + > + ret = sbi_ecall(SBI_EXT_SSE, func, evt, 0, 0, 0, 0, 0); > + if (ret.error) > + pr_debug("Failed to execute func %lx, event %x, error %ld\n", > + func, evt, ret.error); Why's this only at a debug level? > + > + return sbi_err_map_linux_errno(ret.error); > +} > + > +static int sse_sbi_disable_event(struct sse_event *event) > +{ > + return sse_sbi_event_func(event, SBI_SSE_EVENT_DISABLE); > +} > + > +static int sse_sbi_enable_event(struct sse_event *event) > +{ > + return sse_sbi_event_func(event, SBI_SSE_EVENT_ENABLE); > +} > + > +static int sse_event_attr_get_no_lock(struct sse_registered_event *reg_evt, > + unsigned long attr_id, unsigned long *val) > +{ > + struct sbiret sret; > + u32 evt = reg_evt->evt->evt; > + unsigned long phys; > + > + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); > + > + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, > + attr_id, 1, phys, 0, 0); > + if (sret.error) { > + pr_debug("Failed to get event %x attr %lx, error %ld\n", evt, > + attr_id, sret.error); > + return sbi_err_map_linux_errno(sret.error); > + } > + > + *val = reg_evt->attr_buf; > + > + return 0; > +} > + > +static int sse_event_attr_set_nolock(struct sse_registered_event *reg_evt, > + unsigned long attr_id, unsigned long val) > +{ > + struct sbiret sret; > + u32 evt = reg_evt->evt->evt; > + unsigned long phys; > + > + reg_evt->attr_buf = val; > + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); > + > + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, evt, > + attr_id, 1, phys, 0, 0); > + if (sret.error && sret.error != SBI_ERR_INVALID_STATE) { Why's the invalid state error not treated as an error? > + pr_debug("Failed to set event %x attr %lx, error %ld\n", evt, > + attr_id, sret.error); > + return sbi_err_map_linux_errno(sret.error); > + } > + > + return 0; > +} > + > +static int sse_event_set_target_cpu_nolock(struct sse_event *event, > + unsigned int cpu) > +{ > + unsigned int hart_id = cpuid_to_hartid_map(cpu); > + struct sse_registered_event *reg_evt = event->global; > + u32 evt = event->evt; > + bool was_enabled; > + int ret; > + > + if (!sse_event_is_global(evt)) > + return -EINVAL; > + > + was_enabled = event->is_enabled; > + if (was_enabled) > + sse_sbi_disable_event(event); > + do { > + ret = sse_event_attr_set_nolock(reg_evt, > + SBI_SSE_ATTR_PREFERRED_HART, > + hart_id); > + } while (ret == -EINVAL); > + > + if (ret == 0) > + event->cpu = cpu; > + > + if (was_enabled) > + sse_sbi_enable_event(event); > + > + return 0; > +} > + > +int sse_event_set_target_cpu(struct sse_event *event, unsigned int cpu) > +{ > + int ret; > + > + scoped_guard(mutex, &sse_mutex) { > + cpus_read_lock(); > + > + if (!cpu_online(cpu)) > + return -EINVAL; > + > + ret = sse_event_set_target_cpu_nolock(event, cpu); > + > + cpus_read_unlock(); > + } > + > + return ret; > +} > + > +static int sse_event_init_registered(unsigned int cpu, > + struct sse_registered_event *reg_evt, > + struct sse_event *event) > +{ > + reg_evt->evt = event; > + arch_sse_init_event(®_evt->arch, event->evt, cpu); > + > + return 0; > +} > + > +static void sse_event_free_registered(struct sse_registered_event *reg_evt) > +{ > + arch_sse_free_event(®_evt->arch); > +} > + > +static int sse_event_alloc_global(struct sse_event *event) > +{ > + int err; > + struct sse_registered_event *reg_evt; > + > + reg_evt = kzalloc(sizeof(*reg_evt), GFP_KERNEL); > + if (!reg_evt) > + return -ENOMEM; > + > + event->global = reg_evt; > + err = sse_event_init_registered(smp_processor_id(), reg_evt, > + event); > + if (err) > + kfree(reg_evt); > + > + return err; > +} > + > +static int sse_event_alloc_local(struct sse_event *event) > +{ > + int err; > + unsigned int cpu, err_cpu; > + struct sse_registered_event *reg_evt; > + struct sse_registered_event __percpu *reg_evts; > + > + reg_evts = alloc_percpu(struct sse_registered_event); > + if (!reg_evts) > + return -ENOMEM; > + > + event->local = reg_evts; > + > + for_each_possible_cpu(cpu) { > + reg_evt = per_cpu_ptr(reg_evts, cpu); > + err = sse_event_init_registered(cpu, reg_evt, event); > + if (err) { > + err_cpu = cpu; > + goto err_free_per_cpu; > + } > + } > + > + return 0; > + > +err_free_per_cpu: > + for_each_possible_cpu(cpu) { > + if (cpu == err_cpu) > + break; > + reg_evt = per_cpu_ptr(reg_evts, cpu); > + sse_event_free_registered(reg_evt); > + } > + > + free_percpu(reg_evts); > + > + return err; > +} > + > +static struct sse_event *sse_event_alloc(u32 evt, > + u32 priority, > + sse_event_handler *handler, void *arg) > +{ > + int err; > + struct sse_event *event; > + > + event = kzalloc(sizeof(*event), GFP_KERNEL); > + if (!event) > + return ERR_PTR(-ENOMEM); > + > + event->evt = evt; > + event->priority = priority; > + event->handler_arg = arg; > + event->handler = handler; > + > + if (sse_event_is_global(evt)) { > + err = sse_event_alloc_global(event); > + if (err) > + goto err_alloc_reg_evt; > + } else { > + err = sse_event_alloc_local(event); > + if (err) > + goto err_alloc_reg_evt; > + } > + > + return event; > + > +err_alloc_reg_evt: > + kfree(event); > + > + return ERR_PTR(err); > +} > + > +static int sse_sbi_register_event(struct sse_event *event, > + struct sse_registered_event *reg_evt) > +{ > + int ret; > + > + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PRIO, > + event->priority); > + if (ret) > + return ret; > + > + return arch_sse_register_event(®_evt->arch); > +} > + > +static int sse_event_register_local(struct sse_event *event) > +{ > + int ret; > + struct sse_registered_event *reg_evt = per_cpu_ptr(event->local, > + smp_processor_id()); > + > + ret = sse_sbi_register_event(event, reg_evt); > + if (ret) > + pr_debug("Failed to register event %x: err %d\n", event->evt, > + ret); Same here I guess, why's a registration failure only a debug print? > + > + return ret; > +} > + > + > +static int sse_sbi_unregister_event(struct sse_event *event) > +{ > + return sse_sbi_event_func(event, SBI_SSE_EVENT_UNREGISTER); > +} > + > +struct sse_per_cpu_evt { > + struct sse_event *event; > + unsigned long func; > + atomic_t error; > +}; > + > +static void sse_event_per_cpu_func(void *info) > +{ > + int ret; > + struct sse_per_cpu_evt *cpu_evt = info; > + > + if (cpu_evt->func == SBI_SSE_EVENT_REGISTER) > + ret = sse_event_register_local(cpu_evt->event); > + else > + ret = sse_sbi_event_func(cpu_evt->event, cpu_evt->func); > + > + if (ret) > + atomic_set(&cpu_evt->error, ret); > +} > + > +static void sse_event_free(struct sse_event *event) > +{ > + unsigned int cpu; > + struct sse_registered_event *reg_evt; > + > + if (sse_event_is_global(event->evt)) { > + sse_event_free_registered(event->global); > + kfree(event->global); > + } else { > + for_each_possible_cpu(cpu) { > + reg_evt = per_cpu_ptr(event->local, cpu); > + sse_event_free_registered(reg_evt); > + } > + free_percpu(event->local); > + } > + > + kfree(event); > +} > + > +int sse_event_enable(struct sse_event *event) > +{ > + int ret = 0; > + struct sse_per_cpu_evt cpu_evt; > + > + scoped_guard(mutex, &sse_mutex) { > + cpus_read_lock(); > + if (sse_event_is_global(event->evt)) { > + ret = sse_sbi_enable_event(event); > + } else { > + cpu_evt.event = event; > + atomic_set(&cpu_evt.error, 0); > + cpu_evt.func = SBI_SSE_EVENT_ENABLE; > + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); > + ret = atomic_read(&cpu_evt.error); > + if (ret) { > + cpu_evt.func = SBI_SSE_EVENT_DISABLE; > + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, > + 1); nit: this should fit on one line, no? > + } > + } > + cpus_read_unlock(); > + > + if (ret == 0) > + event->is_enabled = true; > + } > + > + return ret; > +} > 2.45.2 >
On 16/01/2025 14:58, Conor Dooley wrote: > On Fri, Dec 06, 2024 at 05:30:59PM +0100, Clément Léger wrote: >> Add driver level interface to use RISC-V SSE arch support. This interface >> allows registering SSE handlers, and receive them. This will be used by >> PMU and GHES driver. >> >> Signed-off-by: Himanshu Chauhan <hchauhan@ventanamicro.com> >> Co-developed-by: Himanshu Chauhan <hchauhan@ventanamicro.com> >> Signed-off-by: Clément Léger <cleger@rivosinc.com> >> --- >> MAINTAINERS | 14 + >> drivers/firmware/Kconfig | 1 + >> drivers/firmware/Makefile | 1 + >> drivers/firmware/riscv/Kconfig | 15 + >> drivers/firmware/riscv/Makefile | 3 + >> drivers/firmware/riscv/riscv_sse.c | 691 +++++++++++++++++++++++++++++ >> include/linux/riscv_sse.h | 56 +++ >> 7 files changed, 781 insertions(+) >> create mode 100644 drivers/firmware/riscv/Kconfig >> create mode 100644 drivers/firmware/riscv/Makefile >> create mode 100644 drivers/firmware/riscv/riscv_sse.c >> create mode 100644 include/linux/riscv_sse.h >> >> diff --git a/MAINTAINERS b/MAINTAINERS >> index 686109008d8e..a3ddde7fe9fb 100644 >> --- a/MAINTAINERS >> +++ b/MAINTAINERS >> @@ -20125,6 +20125,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git >> F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml >> F: drivers/iommu/riscv/ >> >> +RISC-V FIRMWARE DRIVERS >> +M: Conor Dooley <conor@kernel.org> >> +L: linux-riscv@lists.infradead.org >> +S: Maintained >> +T: git git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git >> +F: drivers/firmware/riscv/* > > Acked-by: Conor Dooley <conor.dooley@microchip.com> > > (got some, mostly minor, comments below) > >> diff --git a/drivers/firmware/riscv/Makefile b/drivers/firmware/riscv/Makefile >> new file mode 100644 >> index 000000000000..4ccfcbbc28ea >> --- /dev/null >> +++ b/drivers/firmware/riscv/Makefile >> @@ -0,0 +1,3 @@ >> +# SPDX-License-Identifier: GPL-2.0 >> + >> +obj-$(CONFIG_RISCV_SSE) += riscv_sse.o >> diff --git a/drivers/firmware/riscv/riscv_sse.c b/drivers/firmware/riscv/riscv_sse.c >> new file mode 100644 >> index 000000000000..c165e32cc9a5 >> --- /dev/null >> +++ b/drivers/firmware/riscv/riscv_sse.c >> @@ -0,0 +1,691 @@ >> +// SPDX-License-Identifier: GPL-2.0-only >> +/* >> + * Copyright (C) 2024 Rivos Inc. >> + */ >> + >> +#define pr_fmt(fmt) "sse: " fmt >> + >> +#include <linux/cpu.h> >> +#include <linux/cpuhotplug.h> >> +#include <linux/cpu_pm.h> >> +#include <linux/hardirq.h> >> +#include <linux/list.h> >> +#include <linux/percpu-defs.h> >> +#include <linux/reboot.h> >> +#include <linux/riscv_sse.h> >> +#include <linux/slab.h> >> + >> +#include <asm/sbi.h> >> +#include <asm/sse.h> >> + >> +struct sse_event { >> + struct list_head list; >> + u32 evt; >> + u32 priority; >> + sse_event_handler *handler; >> + void *handler_arg; >> + bool is_enabled; >> + /* Only valid for global events */ >> + unsigned int cpu; >> + >> + union { >> + struct sse_registered_event *global; >> + struct sse_registered_event __percpu *local; >> + }; >> +}; >> + >> +static int sse_hp_state; >> +static bool sse_available; >> +static DEFINE_SPINLOCK(events_list_lock); >> +static LIST_HEAD(events); >> +static DEFINE_MUTEX(sse_mutex); >> + >> +struct sse_registered_event { >> + struct sse_event_arch_data arch; >> + struct sse_event *evt; >> + unsigned long attr_buf; >> +}; >> + >> +void sse_handle_event(struct sse_event_arch_data *arch_event, >> + struct pt_regs *regs) >> +{ >> + int ret; >> + struct sse_registered_event *reg_evt = >> + container_of(arch_event, struct sse_registered_event, arch); >> + struct sse_event *evt = reg_evt->evt; >> + >> + ret = evt->handler(evt->evt, evt->handler_arg, regs); > > Is it possible to get here with a null handler? Or will !registered > events not lead to the handler getting called? Hi Conor, Basically yes, if we receive an event, it means it was registered and enabled. Since when we register an event, we associate it with an handler, it can not be NULL. > >> + if (ret) >> + pr_warn("event %x handler failed with error %d\n", evt->evt, >> + ret); >> +} >> + >> +static bool sse_event_is_global(u32 evt) >> +{ >> + return !!(evt & SBI_SSE_EVENT_GLOBAL); >> +} >> + >> +static >> +struct sse_event *sse_event_get(u32 evt) > > nit: Could you shift this into one line? Yeah sure. > >> +{ >> + struct sse_event *sse_evt = NULL, *tmp; >> + >> + scoped_guard(spinlock, &events_list_lock) { >> + list_for_each_entry(tmp, &events, list) { >> + if (tmp->evt == evt) { >> + return sse_evt; >> + } >> + } >> + } >> + >> + return NULL; >> +} >> + >> +static phys_addr_t sse_event_get_phys(struct sse_registered_event *reg_evt, >> + void *addr) >> +{ >> + phys_addr_t phys; >> + >> + if (sse_event_is_global(reg_evt->evt->evt)) >> + phys = virt_to_phys(addr); >> + else >> + phys = per_cpu_ptr_to_phys(addr); >> + >> + return phys; >> +} >> + >> +static int sse_sbi_event_func(struct sse_event *event, unsigned long func) >> +{ >> + struct sbiret ret; >> + u32 evt = event->evt; >> + >> + ret = sbi_ecall(SBI_EXT_SSE, func, evt, 0, 0, 0, 0, 0); >> + if (ret.error) >> + pr_debug("Failed to execute func %lx, event %x, error %ld\n", >> + func, evt, ret.error); > > Why's this only at a debug level? That's only really meaningful for debugging, this error is often reported to the upper level and ends up to the final caller. I don't think we should be too verbose for such drivers, but rather propagate the error. If one wants to debug, then, just enable DEBUG. But that's only my opinion, if you'd prefer all pr_debug to be either removed or changed to pr_err(), I'll do it. > >> + >> + return sbi_err_map_linux_errno(ret.error); >> +} >> + >> +static int sse_sbi_disable_event(struct sse_event *event) >> +{ >> + return sse_sbi_event_func(event, SBI_SSE_EVENT_DISABLE); >> +} >> + >> +static int sse_sbi_enable_event(struct sse_event *event) >> +{ >> + return sse_sbi_event_func(event, SBI_SSE_EVENT_ENABLE); >> +} >> + >> +static int sse_event_attr_get_no_lock(struct sse_registered_event *reg_evt, >> + unsigned long attr_id, unsigned long *val) >> +{ >> + struct sbiret sret; >> + u32 evt = reg_evt->evt->evt; >> + unsigned long phys; >> + >> + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); >> + >> + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, >> + attr_id, 1, phys, 0, 0); >> + if (sret.error) { >> + pr_debug("Failed to get event %x attr %lx, error %ld\n", evt, >> + attr_id, sret.error); >> + return sbi_err_map_linux_errno(sret.error); >> + } >> + >> + *val = reg_evt->attr_buf; >> + >> + return 0; >> +} >> + >> +static int sse_event_attr_set_nolock(struct sse_registered_event *reg_evt, >> + unsigned long attr_id, unsigned long val) >> +{ >> + struct sbiret sret; >> + u32 evt = reg_evt->evt->evt; >> + unsigned long phys; >> + >> + reg_evt->attr_buf = val; >> + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); >> + >> + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, evt, >> + attr_id, 1, phys, 0, 0); >> + if (sret.error && sret.error != SBI_ERR_INVALID_STATE) { > > Why's the invalid state error not treated as an error? Nice catch. That's a leftover of a previous implementation. That needs to be removed. > >> + pr_debug("Failed to set event %x attr %lx, error %ld\n", evt, >> + attr_id, sret.error); >> + return sbi_err_map_linux_errno(sret.error); >> + } >> + >> + return 0; >> +} >> + >> +static int sse_event_set_target_cpu_nolock(struct sse_event *event, >> + unsigned int cpu) >> +{ >> + unsigned int hart_id = cpuid_to_hartid_map(cpu); >> + struct sse_registered_event *reg_evt = event->global; >> + u32 evt = event->evt; >> + bool was_enabled; >> + int ret; >> + >> + if (!sse_event_is_global(evt)) >> + return -EINVAL; >> + >> + was_enabled = event->is_enabled; >> + if (was_enabled) >> + sse_sbi_disable_event(event); >> + do { >> + ret = sse_event_attr_set_nolock(reg_evt, >> + SBI_SSE_ATTR_PREFERRED_HART, >> + hart_id); >> + } while (ret == -EINVAL); >> + >> + if (ret == 0) >> + event->cpu = cpu; >> + >> + if (was_enabled) >> + sse_sbi_enable_event(event); >> + >> + return 0; >> +} >> + >> +int sse_event_set_target_cpu(struct sse_event *event, unsigned int cpu) >> +{ >> + int ret; >> + >> + scoped_guard(mutex, &sse_mutex) { >> + cpus_read_lock(); >> + >> + if (!cpu_online(cpu)) >> + return -EINVAL; >> + >> + ret = sse_event_set_target_cpu_nolock(event, cpu); >> + >> + cpus_read_unlock(); >> + } >> + >> + return ret; >> +} >> + >> +static int sse_event_init_registered(unsigned int cpu, >> + struct sse_registered_event *reg_evt, >> + struct sse_event *event) >> +{ >> + reg_evt->evt = event; >> + arch_sse_init_event(®_evt->arch, event->evt, cpu); >> + >> + return 0; >> +} >> + >> +static void sse_event_free_registered(struct sse_registered_event *reg_evt) >> +{ >> + arch_sse_free_event(®_evt->arch); >> +} >> + >> +static int sse_event_alloc_global(struct sse_event *event) >> +{ >> + int err; >> + struct sse_registered_event *reg_evt; >> + >> + reg_evt = kzalloc(sizeof(*reg_evt), GFP_KERNEL); >> + if (!reg_evt) >> + return -ENOMEM; >> + >> + event->global = reg_evt; >> + err = sse_event_init_registered(smp_processor_id(), reg_evt, >> + event); >> + if (err) >> + kfree(reg_evt); >> + >> + return err; >> +} >> + >> +static int sse_event_alloc_local(struct sse_event *event) >> +{ >> + int err; >> + unsigned int cpu, err_cpu; >> + struct sse_registered_event *reg_evt; >> + struct sse_registered_event __percpu *reg_evts; >> + >> + reg_evts = alloc_percpu(struct sse_registered_event); >> + if (!reg_evts) >> + return -ENOMEM; >> + >> + event->local = reg_evts; >> + >> + for_each_possible_cpu(cpu) { >> + reg_evt = per_cpu_ptr(reg_evts, cpu); >> + err = sse_event_init_registered(cpu, reg_evt, event); >> + if (err) { >> + err_cpu = cpu; >> + goto err_free_per_cpu; >> + } >> + } >> + >> + return 0; >> + >> +err_free_per_cpu: >> + for_each_possible_cpu(cpu) { >> + if (cpu == err_cpu) >> + break; >> + reg_evt = per_cpu_ptr(reg_evts, cpu); >> + sse_event_free_registered(reg_evt); >> + } >> + >> + free_percpu(reg_evts); >> + >> + return err; >> +} >> + >> +static struct sse_event *sse_event_alloc(u32 evt, >> + u32 priority, >> + sse_event_handler *handler, void *arg) >> +{ >> + int err; >> + struct sse_event *event; >> + >> + event = kzalloc(sizeof(*event), GFP_KERNEL); >> + if (!event) >> + return ERR_PTR(-ENOMEM); >> + >> + event->evt = evt; >> + event->priority = priority; >> + event->handler_arg = arg; >> + event->handler = handler; >> + >> + if (sse_event_is_global(evt)) { >> + err = sse_event_alloc_global(event); >> + if (err) >> + goto err_alloc_reg_evt; >> + } else { >> + err = sse_event_alloc_local(event); >> + if (err) >> + goto err_alloc_reg_evt; >> + } >> + >> + return event; >> + >> +err_alloc_reg_evt: >> + kfree(event); >> + >> + return ERR_PTR(err); >> +} >> + >> +static int sse_sbi_register_event(struct sse_event *event, >> + struct sse_registered_event *reg_evt) >> +{ >> + int ret; >> + >> + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PRIO, >> + event->priority); >> + if (ret) >> + return ret; >> + >> + return arch_sse_register_event(®_evt->arch); >> +} >> + >> +static int sse_event_register_local(struct sse_event *event) >> +{ >> + int ret; >> + struct sse_registered_event *reg_evt = per_cpu_ptr(event->local, >> + smp_processor_id()); >> + >> + ret = sse_sbi_register_event(event, reg_evt); >> + if (ret) >> + pr_debug("Failed to register event %x: err %d\n", event->evt, >> + ret); > > Same here I guess, why's a registration failure only a debug print? Same reason than before, this should be used for debug purposes. > >> + >> + return ret; >> +} >> + >> + >> +static int sse_sbi_unregister_event(struct sse_event *event) >> +{ >> + return sse_sbi_event_func(event, SBI_SSE_EVENT_UNREGISTER); >> +} >> + >> +struct sse_per_cpu_evt { >> + struct sse_event *event; >> + unsigned long func; >> + atomic_t error; >> +}; >> + >> +static void sse_event_per_cpu_func(void *info) >> +{ >> + int ret; >> + struct sse_per_cpu_evt *cpu_evt = info; >> + >> + if (cpu_evt->func == SBI_SSE_EVENT_REGISTER) >> + ret = sse_event_register_local(cpu_evt->event); >> + else >> + ret = sse_sbi_event_func(cpu_evt->event, cpu_evt->func); >> + >> + if (ret) >> + atomic_set(&cpu_evt->error, ret); >> +} >> + >> +static void sse_event_free(struct sse_event *event) >> +{ >> + unsigned int cpu; >> + struct sse_registered_event *reg_evt; >> + >> + if (sse_event_is_global(event->evt)) { >> + sse_event_free_registered(event->global); >> + kfree(event->global); >> + } else { >> + for_each_possible_cpu(cpu) { >> + reg_evt = per_cpu_ptr(event->local, cpu); >> + sse_event_free_registered(reg_evt); >> + } >> + free_percpu(event->local); >> + } >> + >> + kfree(event); >> +} >> + >> +int sse_event_enable(struct sse_event *event) >> +{ >> + int ret = 0; >> + struct sse_per_cpu_evt cpu_evt; >> + >> + scoped_guard(mutex, &sse_mutex) { >> + cpus_read_lock(); >> + if (sse_event_is_global(event->evt)) { >> + ret = sse_sbi_enable_event(event); >> + } else { >> + cpu_evt.event = event; >> + atomic_set(&cpu_evt.error, 0); >> + cpu_evt.func = SBI_SSE_EVENT_ENABLE; >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); >> + ret = atomic_read(&cpu_evt.error); >> + if (ret) { >> + cpu_evt.func = SBI_SSE_EVENT_DISABLE; >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, >> + 1); > > nit: this should fit on one line, no? the trailing ; is above 80 characters. But if you are ok with 100 char, I can go for it. Thanks ! Clément > >> + } >> + } >> + cpus_read_unlock(); >> + >> + if (ret == 0) >> + event->is_enabled = true; >> + } >> + >> + return ret; >> +} > >> 2.45.2 >>
On Thu, Jan 23, 2025 at 11:52:35AM +0100, Clément Léger wrote: > On 16/01/2025 14:58, Conor Dooley wrote: > >> +static int sse_sbi_event_func(struct sse_event *event, unsigned long func) > >> +{ > >> + struct sbiret ret; > >> + u32 evt = event->evt; > >> + > >> + ret = sbi_ecall(SBI_EXT_SSE, func, evt, 0, 0, 0, 0, 0); > >> + if (ret.error) > >> + pr_debug("Failed to execute func %lx, event %x, error %ld\n", > >> + func, evt, ret.error); > > > > Why's this only at a debug level? > > That's only really meaningful for debugging, this error is often > reported to the upper level and ends up to the final caller. I don't > think we should be too verbose for such drivers, but rather propagate > the error. If one wants to debug, then, just enable DEBUG. > > But that's only my opinion, if you'd prefer all pr_debug to be either > removed or changed to pr_err(), I'll do it. Nah, you can leave it as is, you know better than I about how helpful it would be as an error. > >> +int sse_event_enable(struct sse_event *event) > >> +{ > >> + int ret = 0; > >> + struct sse_per_cpu_evt cpu_evt; > >> + > >> + scoped_guard(mutex, &sse_mutex) { > >> + cpus_read_lock(); > >> + if (sse_event_is_global(event->evt)) { > >> + ret = sse_sbi_enable_event(event); > >> + } else { > >> + cpu_evt.event = event; > >> + atomic_set(&cpu_evt.error, 0); > >> + cpu_evt.func = SBI_SSE_EVENT_ENABLE; > >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); > >> + ret = atomic_read(&cpu_evt.error); > >> + if (ret) { > >> + cpu_evt.func = SBI_SSE_EVENT_DISABLE; > >> + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, > >> + 1); > > > > nit: this should fit on one line, no? > > the trailing ; is above 80 characters. But if you are ok with 100 char, > I can go for it. The slight improvement in readability trumps the slight increase over 80 characters every time for me.
diff --git a/MAINTAINERS b/MAINTAINERS index 686109008d8e..a3ddde7fe9fb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -20125,6 +20125,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml F: drivers/iommu/riscv/ +RISC-V FIRMWARE DRIVERS +M: Conor Dooley <conor@kernel.org> +L: linux-riscv@lists.infradead.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git +F: drivers/firmware/riscv/* + RISC-V MICROCHIP FPGA SUPPORT M: Conor Dooley <conor.dooley@microchip.com> M: Daire McNamara <daire.mcnamara@microchip.com> @@ -20177,6 +20184,13 @@ F: drivers/perf/riscv_pmu.c F: drivers/perf/riscv_pmu_legacy.c F: drivers/perf/riscv_pmu_sbi.c +RISC-V SSE DRIVER +M: Clément Léger <cleger@rivosinc.com> +L: linux-riscv@lists.infradead.org +S: Maintained +F: drivers/firmware/riscv/riscv_sse.c +F: include/linux/riscv_sse.h + RISC-V THEAD SoC SUPPORT M: Drew Fustini <drew@pdp7.com> M: Guo Ren <guoren@kernel.org> diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index 71d8b26c4103..9e996a1fd511 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -267,6 +267,7 @@ source "drivers/firmware/meson/Kconfig" source "drivers/firmware/microchip/Kconfig" source "drivers/firmware/psci/Kconfig" source "drivers/firmware/qcom/Kconfig" +source "drivers/firmware/riscv/Kconfig" source "drivers/firmware/smccc/Kconfig" source "drivers/firmware/tegra/Kconfig" source "drivers/firmware/xilinx/Kconfig" diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 7a8d486e718f..c0f5009949a8 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -33,6 +33,7 @@ obj-y += efi/ obj-y += imx/ obj-y += psci/ obj-y += qcom/ +obj-y += riscv/ obj-y += smccc/ obj-y += tegra/ obj-y += xilinx/ diff --git a/drivers/firmware/riscv/Kconfig b/drivers/firmware/riscv/Kconfig new file mode 100644 index 000000000000..8056ed3262d9 --- /dev/null +++ b/drivers/firmware/riscv/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "Risc-V Specific firmware drivers" +depends on RISCV + +config RISCV_SSE + bool "Enable SBI Supervisor Software Events support" + depends on RISCV_SBI + default y + help + The Supervisor Software Events support allow the SBI to deliver + NMI-like notifications to the supervisor mode software. When enable, + this option provides support to register callbacks on specific SSE + events. + +endmenu diff --git a/drivers/firmware/riscv/Makefile b/drivers/firmware/riscv/Makefile new file mode 100644 index 000000000000..4ccfcbbc28ea --- /dev/null +++ b/drivers/firmware/riscv/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_RISCV_SSE) += riscv_sse.o diff --git a/drivers/firmware/riscv/riscv_sse.c b/drivers/firmware/riscv/riscv_sse.c new file mode 100644 index 000000000000..c165e32cc9a5 --- /dev/null +++ b/drivers/firmware/riscv/riscv_sse.c @@ -0,0 +1,691 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Rivos Inc. + */ + +#define pr_fmt(fmt) "sse: " fmt + +#include <linux/cpu.h> +#include <linux/cpuhotplug.h> +#include <linux/cpu_pm.h> +#include <linux/hardirq.h> +#include <linux/list.h> +#include <linux/percpu-defs.h> +#include <linux/reboot.h> +#include <linux/riscv_sse.h> +#include <linux/slab.h> + +#include <asm/sbi.h> +#include <asm/sse.h> + +struct sse_event { + struct list_head list; + u32 evt; + u32 priority; + sse_event_handler *handler; + void *handler_arg; + bool is_enabled; + /* Only valid for global events */ + unsigned int cpu; + + union { + struct sse_registered_event *global; + struct sse_registered_event __percpu *local; + }; +}; + +static int sse_hp_state; +static bool sse_available; +static DEFINE_SPINLOCK(events_list_lock); +static LIST_HEAD(events); +static DEFINE_MUTEX(sse_mutex); + +struct sse_registered_event { + struct sse_event_arch_data arch; + struct sse_event *evt; + unsigned long attr_buf; +}; + +void sse_handle_event(struct sse_event_arch_data *arch_event, + struct pt_regs *regs) +{ + int ret; + struct sse_registered_event *reg_evt = + container_of(arch_event, struct sse_registered_event, arch); + struct sse_event *evt = reg_evt->evt; + + ret = evt->handler(evt->evt, evt->handler_arg, regs); + if (ret) + pr_warn("event %x handler failed with error %d\n", evt->evt, + ret); +} + +static bool sse_event_is_global(u32 evt) +{ + return !!(evt & SBI_SSE_EVENT_GLOBAL); +} + +static +struct sse_event *sse_event_get(u32 evt) +{ + struct sse_event *sse_evt = NULL, *tmp; + + scoped_guard(spinlock, &events_list_lock) { + list_for_each_entry(tmp, &events, list) { + if (tmp->evt == evt) { + return sse_evt; + } + } + } + + return NULL; +} + +static phys_addr_t sse_event_get_phys(struct sse_registered_event *reg_evt, + void *addr) +{ + phys_addr_t phys; + + if (sse_event_is_global(reg_evt->evt->evt)) + phys = virt_to_phys(addr); + else + phys = per_cpu_ptr_to_phys(addr); + + return phys; +} + +static int sse_sbi_event_func(struct sse_event *event, unsigned long func) +{ + struct sbiret ret; + u32 evt = event->evt; + + ret = sbi_ecall(SBI_EXT_SSE, func, evt, 0, 0, 0, 0, 0); + if (ret.error) + pr_debug("Failed to execute func %lx, event %x, error %ld\n", + func, evt, ret.error); + + return sbi_err_map_linux_errno(ret.error); +} + +static int sse_sbi_disable_event(struct sse_event *event) +{ + return sse_sbi_event_func(event, SBI_SSE_EVENT_DISABLE); +} + +static int sse_sbi_enable_event(struct sse_event *event) +{ + return sse_sbi_event_func(event, SBI_SSE_EVENT_ENABLE); +} + +static int sse_event_attr_get_no_lock(struct sse_registered_event *reg_evt, + unsigned long attr_id, unsigned long *val) +{ + struct sbiret sret; + u32 evt = reg_evt->evt->evt; + unsigned long phys; + + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, + attr_id, 1, phys, 0, 0); + if (sret.error) { + pr_debug("Failed to get event %x attr %lx, error %ld\n", evt, + attr_id, sret.error); + return sbi_err_map_linux_errno(sret.error); + } + + *val = reg_evt->attr_buf; + + return 0; +} + +static int sse_event_attr_set_nolock(struct sse_registered_event *reg_evt, + unsigned long attr_id, unsigned long val) +{ + struct sbiret sret; + u32 evt = reg_evt->evt->evt; + unsigned long phys; + + reg_evt->attr_buf = val; + phys = sse_event_get_phys(reg_evt, ®_evt->attr_buf); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, evt, + attr_id, 1, phys, 0, 0); + if (sret.error && sret.error != SBI_ERR_INVALID_STATE) { + pr_debug("Failed to set event %x attr %lx, error %ld\n", evt, + attr_id, sret.error); + return sbi_err_map_linux_errno(sret.error); + } + + return 0; +} + +static int sse_event_set_target_cpu_nolock(struct sse_event *event, + unsigned int cpu) +{ + unsigned int hart_id = cpuid_to_hartid_map(cpu); + struct sse_registered_event *reg_evt = event->global; + u32 evt = event->evt; + bool was_enabled; + int ret; + + if (!sse_event_is_global(evt)) + return -EINVAL; + + was_enabled = event->is_enabled; + if (was_enabled) + sse_sbi_disable_event(event); + do { + ret = sse_event_attr_set_nolock(reg_evt, + SBI_SSE_ATTR_PREFERRED_HART, + hart_id); + } while (ret == -EINVAL); + + if (ret == 0) + event->cpu = cpu; + + if (was_enabled) + sse_sbi_enable_event(event); + + return 0; +} + +int sse_event_set_target_cpu(struct sse_event *event, unsigned int cpu) +{ + int ret; + + scoped_guard(mutex, &sse_mutex) { + cpus_read_lock(); + + if (!cpu_online(cpu)) + return -EINVAL; + + ret = sse_event_set_target_cpu_nolock(event, cpu); + + cpus_read_unlock(); + } + + return ret; +} + +static int sse_event_init_registered(unsigned int cpu, + struct sse_registered_event *reg_evt, + struct sse_event *event) +{ + reg_evt->evt = event; + arch_sse_init_event(®_evt->arch, event->evt, cpu); + + return 0; +} + +static void sse_event_free_registered(struct sse_registered_event *reg_evt) +{ + arch_sse_free_event(®_evt->arch); +} + +static int sse_event_alloc_global(struct sse_event *event) +{ + int err; + struct sse_registered_event *reg_evt; + + reg_evt = kzalloc(sizeof(*reg_evt), GFP_KERNEL); + if (!reg_evt) + return -ENOMEM; + + event->global = reg_evt; + err = sse_event_init_registered(smp_processor_id(), reg_evt, + event); + if (err) + kfree(reg_evt); + + return err; +} + +static int sse_event_alloc_local(struct sse_event *event) +{ + int err; + unsigned int cpu, err_cpu; + struct sse_registered_event *reg_evt; + struct sse_registered_event __percpu *reg_evts; + + reg_evts = alloc_percpu(struct sse_registered_event); + if (!reg_evts) + return -ENOMEM; + + event->local = reg_evts; + + for_each_possible_cpu(cpu) { + reg_evt = per_cpu_ptr(reg_evts, cpu); + err = sse_event_init_registered(cpu, reg_evt, event); + if (err) { + err_cpu = cpu; + goto err_free_per_cpu; + } + } + + return 0; + +err_free_per_cpu: + for_each_possible_cpu(cpu) { + if (cpu == err_cpu) + break; + reg_evt = per_cpu_ptr(reg_evts, cpu); + sse_event_free_registered(reg_evt); + } + + free_percpu(reg_evts); + + return err; +} + +static struct sse_event *sse_event_alloc(u32 evt, + u32 priority, + sse_event_handler *handler, void *arg) +{ + int err; + struct sse_event *event; + + event = kzalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return ERR_PTR(-ENOMEM); + + event->evt = evt; + event->priority = priority; + event->handler_arg = arg; + event->handler = handler; + + if (sse_event_is_global(evt)) { + err = sse_event_alloc_global(event); + if (err) + goto err_alloc_reg_evt; + } else { + err = sse_event_alloc_local(event); + if (err) + goto err_alloc_reg_evt; + } + + return event; + +err_alloc_reg_evt: + kfree(event); + + return ERR_PTR(err); +} + +static int sse_sbi_register_event(struct sse_event *event, + struct sse_registered_event *reg_evt) +{ + int ret; + + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PRIO, + event->priority); + if (ret) + return ret; + + return arch_sse_register_event(®_evt->arch); +} + +static int sse_event_register_local(struct sse_event *event) +{ + int ret; + struct sse_registered_event *reg_evt = per_cpu_ptr(event->local, + smp_processor_id()); + + ret = sse_sbi_register_event(event, reg_evt); + if (ret) + pr_debug("Failed to register event %x: err %d\n", event->evt, + ret); + + return ret; +} + + +static int sse_sbi_unregister_event(struct sse_event *event) +{ + return sse_sbi_event_func(event, SBI_SSE_EVENT_UNREGISTER); +} + +struct sse_per_cpu_evt { + struct sse_event *event; + unsigned long func; + atomic_t error; +}; + +static void sse_event_per_cpu_func(void *info) +{ + int ret; + struct sse_per_cpu_evt *cpu_evt = info; + + if (cpu_evt->func == SBI_SSE_EVENT_REGISTER) + ret = sse_event_register_local(cpu_evt->event); + else + ret = sse_sbi_event_func(cpu_evt->event, cpu_evt->func); + + if (ret) + atomic_set(&cpu_evt->error, ret); +} + +static void sse_event_free(struct sse_event *event) +{ + unsigned int cpu; + struct sse_registered_event *reg_evt; + + if (sse_event_is_global(event->evt)) { + sse_event_free_registered(event->global); + kfree(event->global); + } else { + for_each_possible_cpu(cpu) { + reg_evt = per_cpu_ptr(event->local, cpu); + sse_event_free_registered(reg_evt); + } + free_percpu(event->local); + } + + kfree(event); +} + +int sse_event_enable(struct sse_event *event) +{ + int ret = 0; + struct sse_per_cpu_evt cpu_evt; + + scoped_guard(mutex, &sse_mutex) { + cpus_read_lock(); + if (sse_event_is_global(event->evt)) { + ret = sse_sbi_enable_event(event); + } else { + cpu_evt.event = event; + atomic_set(&cpu_evt.error, 0); + cpu_evt.func = SBI_SSE_EVENT_ENABLE; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + ret = atomic_read(&cpu_evt.error); + if (ret) { + cpu_evt.func = SBI_SSE_EVENT_DISABLE; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, + 1); + } + } + cpus_read_unlock(); + + if (ret == 0) + event->is_enabled = true; + } + + return ret; +} + +static void sse_events_mask(void) +{ + sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_MASK, 0, 0, 0, 0, 0, 0); +} + +static void sse_events_unmask(void) +{ + sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_HART_UNMASK, 0, 0, 0, 0, 0, 0); +} + +static void sse_event_disable_nolock(struct sse_event *event) +{ + struct sse_per_cpu_evt cpu_evt; + + if (sse_event_is_global(event->evt)) { + sse_sbi_disable_event(event); + } else { + cpu_evt.event = event; + cpu_evt.func = SBI_SSE_EVENT_DISABLE; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + } +} + +void sse_event_disable(struct sse_event *event) +{ + scoped_guard(mutex, &sse_mutex) { + cpus_read_lock(); + sse_event_disable_nolock(event); + event->is_enabled = false; + cpus_read_unlock(); + } +} + +struct sse_event *sse_event_register(u32 evt, u32 priority, + sse_event_handler *handler, void *arg) +{ + struct sse_per_cpu_evt cpu_evt; + struct sse_event *event; + int ret = 0; + + if (!sse_available) + return ERR_PTR(-EOPNOTSUPP); + + mutex_lock(&sse_mutex); + if (sse_event_get(evt)) { + pr_debug("Event %x already registered\n", evt); + ret = -EEXIST; + goto out_unlock; + } + + event = sse_event_alloc(evt, priority, handler, arg); + if (IS_ERR(event)) { + ret = PTR_ERR(event); + goto out_unlock; + } + + cpus_read_lock(); + if (sse_event_is_global(evt)) { + unsigned long preferred_hart; + + ret = sse_event_attr_get_no_lock(event->global, + SBI_SSE_ATTR_PREFERRED_HART, + &preferred_hart); + if (ret) + goto err_event_free; + event->cpu = riscv_hartid_to_cpuid(preferred_hart); + + ret = sse_sbi_register_event(event, event->global); + if (ret) + goto err_event_free; + + } else { + cpu_evt.event = event; + atomic_set(&cpu_evt.error, 0); + cpu_evt.func = SBI_SSE_EVENT_REGISTER; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + ret = atomic_read(&cpu_evt.error); + if (ret) { + cpu_evt.func = SBI_SSE_EVENT_UNREGISTER; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + goto err_event_free; + } + } + cpus_read_unlock(); + + scoped_guard(spinlock, &events_list_lock) + list_add(&event->list, &events); + + mutex_unlock(&sse_mutex); + + return event; + +err_event_free: + cpus_read_unlock(); + sse_event_free(event); +out_unlock: + mutex_unlock(&sse_mutex); + + return ERR_PTR(ret); +} + +static void sse_event_unregister_nolock(struct sse_event *event) +{ + struct sse_per_cpu_evt cpu_evt; + + if (sse_event_is_global(event->evt)) { + sse_sbi_unregister_event(event); + } else { + cpu_evt.event = event; + cpu_evt.func = SBI_SSE_EVENT_UNREGISTER; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + } +} + +void sse_event_unregister(struct sse_event *event) +{ + scoped_guard(mutex, &sse_mutex) { + cpus_read_lock(); + sse_event_unregister_nolock(event); + cpus_read_unlock(); + + scoped_guard(spinlock, &events_list_lock) + list_del(&event->list); + + sse_event_free(event); + } +} + +static int sse_cpu_online(unsigned int cpu) +{ + struct sse_event *sse_evt; + + scoped_guard(spinlock, &events_list_lock) { + list_for_each_entry(sse_evt, &events, list) { + if (sse_event_is_global(sse_evt->evt)) + continue; + + sse_event_register_local(sse_evt); + if (sse_evt->is_enabled) + sse_sbi_enable_event(sse_evt); + } + } + + /* Ready to handle events. Unmask SSE. */ + sse_events_unmask(); + + return 0; +} + +static int sse_cpu_teardown(unsigned int cpu) +{ + unsigned int next_cpu; + struct sse_event *sse_evt; + + /* Mask the sse events */ + sse_events_mask(); + + scoped_guard(spinlock, &events_list_lock) { + list_for_each_entry(sse_evt, &events, list) { + if (!sse_event_is_global(sse_evt->evt)) { + + if (sse_evt->is_enabled) + sse_sbi_disable_event(sse_evt); + + sse_sbi_unregister_event(sse_evt); + continue; + } + + if (sse_evt->cpu != smp_processor_id()) + continue; + + /* Update destination hart for global event */ + next_cpu = cpumask_any_but(cpu_online_mask, cpu); + sse_event_set_target_cpu_nolock(sse_evt, next_cpu); + } + } + + return 0; +} + +static void sse_reset(void) +{ + struct sse_event *event = NULL; + + list_for_each_entry(event, &events, list) { + sse_event_disable_nolock(event); + sse_event_unregister_nolock(event); + } +} + +static int sse_pm_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + WARN_ON_ONCE(preemptible()); + + switch (action) { + case CPU_PM_ENTER: + sse_events_mask(); + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + sse_events_unmask(); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +static struct notifier_block sse_pm_nb = { + .notifier_call = sse_pm_notifier, +}; + +/* + * Mask all CPUs and unregister all events on panic, reboot or kexec. + */ +static int sse_reboot_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + cpuhp_remove_state(sse_hp_state); + + sse_reset(); + + return NOTIFY_OK; +} + +static struct notifier_block sse_reboot_nb = { + .notifier_call = sse_reboot_notifier, +}; + +static int __init sse_init(void) +{ + int cpu, ret; + + if (sbi_probe_extension(SBI_EXT_SSE) <= 0) { + pr_err("Missing SBI SSE extension\n"); + return -EOPNOTSUPP; + } + pr_info("SBI SSE extension detected\n"); + + for_each_possible_cpu(cpu) + INIT_LIST_HEAD(&events); + + ret = cpu_pm_register_notifier(&sse_pm_nb); + if (ret) { + pr_warn("Failed to register CPU PM notifier...\n"); + return ret; + } + + ret = register_reboot_notifier(&sse_reboot_nb); + if (ret) { + pr_warn("Failed to register reboot notifier...\n"); + goto remove_cpupm; + } + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/sse:online", + sse_cpu_online, sse_cpu_teardown); + if (ret < 0) + goto remove_reboot; + + sse_hp_state = ret; + sse_available = true; + + return 0; + +remove_reboot: + unregister_reboot_notifier(&sse_reboot_nb); + +remove_cpupm: + cpu_pm_unregister_notifier(&sse_pm_nb); + + return ret; +} +arch_initcall(sse_init); diff --git a/include/linux/riscv_sse.h b/include/linux/riscv_sse.h new file mode 100644 index 000000000000..c73184074b8c --- /dev/null +++ b/include/linux/riscv_sse.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Rivos Inc. + */ + +#ifndef __LINUX_RISCV_SSE_H +#define __LINUX_RISCV_SSE_H + +#include <linux/types.h> +#include <linux/linkage.h> + +struct sse_event; +struct pt_regs; + +typedef int (sse_event_handler)(u32 event_num, void *arg, struct pt_regs *regs); + +#ifdef CONFIG_RISCV_SSE + +struct sse_event *sse_event_register(u32 event_num, u32 priority, + sse_event_handler *handler, void *arg); + +void sse_event_unregister(struct sse_event *evt); + +int sse_event_set_target_cpu(struct sse_event *sse_evt, unsigned int cpu); + +int sse_event_enable(struct sse_event *sse_evt); + +void sse_event_disable(struct sse_event *sse_evt); + +#else +static inline struct sse_event *sse_event_register(u32 event_num, u32 priority, + sse_event_handler *handler, + void *arg) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline void sse_event_unregister(struct sse_event *evt) {} + +static inline int sse_event_set_target_cpu(struct sse_event *sse_evt, + unsigned int cpu) +{ + return -EOPNOTSUPP; +} + +static inline int sse_event_enable(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} + +static inline void sse_event_disable(struct sse_event *sse_evt) {} + + +#endif + +#endif /* __LINUX_RISCV_SSE_H */