@@ -157,6 +157,19 @@ u64 riscv_pmu_ctr_get_width_mask(struct perf_event *event)
return GENMASK_ULL(cwidth, 0);
}
+static void riscv_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,
+ bool sched_in)
+{
+ struct riscv_pmu *pmu;
+
+ if (!pmu_ctx)
+ return;
+
+ pmu = to_riscv_pmu(pmu_ctx->pmu);
+ if (pmu->sched_task)
+ pmu->sched_task(pmu_ctx, sched_in);
+}
+
u64 riscv_pmu_event_update(struct perf_event *event)
{
struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
@@ -269,6 +282,8 @@ static int riscv_pmu_add(struct perf_event *event, int flags)
cpuc->events[idx] = event;
cpuc->n_events++;
hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
+ if (rvpmu->ctr_add)
+ rvpmu->ctr_add(event, flags);
if (flags & PERF_EF_START)
riscv_pmu_start(event, PERF_EF_RELOAD);
@@ -286,6 +301,9 @@ static void riscv_pmu_del(struct perf_event *event, int flags)
riscv_pmu_stop(event, PERF_EF_UPDATE);
cpuc->events[hwc->idx] = NULL;
+ if (rvpmu->ctr_del)
+ rvpmu->ctr_del(event, flags);
+
/* The firmware need to reset the counter mapping */
if (rvpmu->ctr_stop)
rvpmu->ctr_stop(event, RISCV_PMU_STOP_FLAG_RESET);
@@ -402,6 +420,7 @@ struct riscv_pmu *riscv_pmu_alloc(void)
for_each_possible_cpu(cpuid) {
cpuc = per_cpu_ptr(pmu->hw_events, cpuid);
cpuc->n_events = 0;
+ cpuc->ctr_users = 0;
for (i = 0; i < RISCV_MAX_COUNTERS; i++)
cpuc->events[i] = NULL;
cpuc->snapshot_addr = NULL;
@@ -416,6 +435,7 @@ struct riscv_pmu *riscv_pmu_alloc(void)
.start = riscv_pmu_start,
.stop = riscv_pmu_stop,
.read = riscv_pmu_read,
+ .sched_task = riscv_pmu_sched_task,
};
return pmu;
@@ -1027,6 +1027,12 @@ static void rvpmu_sbi_ctr_stop(struct perf_event *event, unsigned long flag)
}
}
+static void pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,
+ bool sched_in)
+{
+ /* Call CTR specific Sched hook. */
+}
+
static int rvpmu_sbi_find_num_ctrs(void)
{
struct sbiret ret;
@@ -1569,6 +1575,14 @@ static int rvpmu_deleg_ctr_get_idx(struct perf_event *event)
return -ENOENT;
}
+static void rvpmu_ctr_add(struct perf_event *event, int flags)
+{
+}
+
+static void rvpmu_ctr_del(struct perf_event *event, int flags)
+{
+}
+
static void rvpmu_ctr_start(struct perf_event *event, u64 ival)
{
struct hw_perf_event *hwc = &event->hw;
@@ -1984,6 +1998,8 @@ static int rvpmu_device_probe(struct platform_device *pdev)
else
pmu->pmu.attr_groups = riscv_sbi_pmu_attr_groups;
pmu->cmask = cmask;
+ pmu->ctr_add = rvpmu_ctr_add;
+ pmu->ctr_del = rvpmu_ctr_del;
pmu->ctr_start = rvpmu_ctr_start;
pmu->ctr_stop = rvpmu_ctr_stop;
pmu->event_map = rvpmu_event_map;
@@ -1995,6 +2011,7 @@ static int rvpmu_device_probe(struct platform_device *pdev)
pmu->event_mapped = rvpmu_event_mapped;
pmu->event_unmapped = rvpmu_event_unmapped;
pmu->csr_index = rvpmu_csr_index;
+ pmu->sched_task = pmu_sched_task;
ret = riscv_pm_pmu_register(pmu);
if (ret)
@@ -115,6 +115,8 @@ static void pmu_legacy_init(struct riscv_pmu *pmu)
BIT(RISCV_PMU_LEGACY_INSTRET);
pmu->ctr_start = pmu_legacy_ctr_start;
pmu->ctr_stop = NULL;
+ pmu->ctr_add = NULL;
+ pmu->ctr_del = NULL;
pmu->event_map = pmu_legacy_event_map;
pmu->ctr_get_idx = pmu_legacy_ctr_get_idx;
pmu->ctr_get_width = pmu_legacy_ctr_get_width;
@@ -46,6 +46,13 @@
}, \
}
+#define MAX_BRANCH_RECORDS 256
+
+struct branch_records {
+ struct perf_branch_stack branch_stack;
+ struct perf_branch_entry branch_entries[MAX_BRANCH_RECORDS];
+};
+
struct cpu_hw_events {
/* currently enabled events */
int n_events;
@@ -65,6 +72,12 @@ struct cpu_hw_events {
bool snapshot_set_done;
/* A shadow copy of the counter values to avoid clobbering during multiple SBI calls */
u64 snapshot_cval_shcopy[RISCV_MAX_COUNTERS];
+
+ /* Saved branch records. */
+ struct branch_records *branches;
+
+ /* Active events requesting branch records */
+ int ctr_users;
};
struct riscv_pmu {
@@ -78,6 +91,8 @@ struct riscv_pmu {
int (*ctr_get_idx)(struct perf_event *event);
int (*ctr_get_width)(int idx);
void (*ctr_clear_idx)(struct perf_event *event);
+ void (*ctr_add)(struct perf_event *event, int flags);
+ void (*ctr_del)(struct perf_event *event, int flags);
void (*ctr_start)(struct perf_event *event, u64 init_val);
void (*ctr_stop)(struct perf_event *event, unsigned long flag);
int (*event_map)(struct perf_event *event, u64 *config);
@@ -85,10 +100,13 @@ struct riscv_pmu {
void (*event_mapped)(struct perf_event *event, struct mm_struct *mm);
void (*event_unmapped)(struct perf_event *event, struct mm_struct *mm);
uint8_t (*csr_index)(struct perf_event *event);
+ void (*sched_task)(struct perf_event_pmu_context *ctx, bool sched_in);
struct cpu_hw_events __percpu *hw_events;
struct hlist_node node;
struct notifier_block riscv_pm_nb;
+
+ unsigned int ctr_depth;
};
#define to_riscv_pmu(p) (container_of(p, struct riscv_pmu, pmu))
To support Control Transfer Records (CTR) extension, we need to extend the riscv_pmu framework with some basic infrastructure for branch stack sampling. Subsequent patches will use this to add support for CTR in the riscv_pmu_dev driver. With CTR, the branches are stored into a hardware FIFO, which will be sampled by software when perf events overflow. A task may be context-switched between overflows, and to avoid leaking samples we need to clear the last task's records when a task is context-switched in. To do this we will be using the pmu::sched_task() callback added in this patch. Signed-off-by: Rajnesh Kanwal <rkanwal@rivosinc.com> --- drivers/perf/riscv_pmu_common.c | 20 ++++++++++++++++++++ drivers/perf/riscv_pmu_dev.c | 17 +++++++++++++++++ drivers/perf/riscv_pmu_legacy.c | 2 ++ include/linux/perf/riscv_pmu.h | 18 ++++++++++++++++++ 4 files changed, 57 insertions(+)