// SPDX-License-Identifier: GPL-2.0 /* * Performance event support - Processor Activity Instrumentation Extension * Facility * * Copyright IBM Corp. 2022 * Author(s): Thomas Richter */ #define KMSG_COMPONENT "pai_ext" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include #include #include #include #include #include #include #include #include #include #include #define PAIE1_CB_SZ 0x200 /* Size of PAIE1 control block */ #define PAIE1_CTRBLOCK_SZ 0x400 /* Size of PAIE1 counter blocks */ static debug_info_t *paiext_dbg; static unsigned int paiext_cnt; /* Extracted with QPACI instruction */ struct pai_userdata { u16 num; u64 value; } __packed; /* Create the PAI extension 1 control block area. * The PAI extension control block 1 is pointed to by lowcore * address 0x1508 for each CPU. This control block is 512 bytes in size * and requires a 512 byte boundary alignment. */ struct paiext_cb { /* PAI extension 1 control block */ u64 header; /* Not used */ u64 reserved1; u64 acc; /* Addr to analytics counter control block */ u8 reserved2[488]; } __packed; struct paiext_map { unsigned long *area; /* Area for CPU to store counters */ struct pai_userdata *save; /* Area to store non-zero counters */ unsigned int active_events; /* # of PAI Extension users */ refcount_t refcnt; struct perf_event *event; /* Perf event for sampling */ struct paiext_cb *paiext_cb; /* PAI extension control block area */ struct list_head syswide_list; /* List system-wide sampling events */ }; struct paiext_mapptr { struct paiext_map *mapptr; }; static struct paiext_root { /* Anchor to per CPU data */ refcount_t refcnt; /* Overall active events */ struct paiext_mapptr __percpu *mapptr; } paiext_root; /* Free per CPU data when the last event is removed. */ static void paiext_root_free(void) { if (refcount_dec_and_test(&paiext_root.refcnt)) { free_percpu(paiext_root.mapptr); paiext_root.mapptr = NULL; } debug_sprintf_event(paiext_dbg, 5, "%s root.refcount %d\n", __func__, refcount_read(&paiext_root.refcnt)); } /* On initialization of first event also allocate per CPU data dynamically. * Start with an array of pointers, the array size is the maximum number of * CPUs possible, which might be larger than the number of CPUs currently * online. */ static int paiext_root_alloc(void) { if (!refcount_inc_not_zero(&paiext_root.refcnt)) { /* The memory is already zeroed. */ paiext_root.mapptr = alloc_percpu(struct paiext_mapptr); if (!paiext_root.mapptr) { /* Returning without refcnt adjustment is ok. The * error code is handled by paiext_alloc() which * decrements refcnt when an event can not be * created. */ return -ENOMEM; } refcount_set(&paiext_root.refcnt, 1); } return 0; } /* Protects against concurrent increment of sampler and counter member * increments at the same time and prohibits concurrent execution of * counting and sampling events. * Ensures that analytics counter block is deallocated only when the * sampling and counting on that cpu is zero. * For details see paiext_alloc(). */ static DEFINE_MUTEX(paiext_reserve_mutex); /* Free all memory allocated for event counting/sampling setup */ static void paiext_free(struct paiext_mapptr *mp) { kfree(mp->mapptr->area); kfree(mp->mapptr->paiext_cb); kvfree(mp->mapptr->save); kfree(mp->mapptr); mp->mapptr = NULL; } /* Release the PMU if event is the last perf event */ static void paiext_event_destroy_cpu(struct perf_event *event, int cpu) { struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, cpu); struct paiext_map *cpump = mp->mapptr; mutex_lock(&paiext_reserve_mutex); if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */ paiext_free(mp); paiext_root_free(); mutex_unlock(&paiext_reserve_mutex); } static void paiext_event_destroy(struct perf_event *event) { int cpu; free_page(PAI_SAVE_AREA(event)); if (event->cpu == -1) { struct cpumask *mask = PAI_CPU_MASK(event); for_each_cpu(cpu, mask) paiext_event_destroy_cpu(event, cpu); kfree(mask); } else { paiext_event_destroy_cpu(event, event->cpu); } debug_sprintf_event(paiext_dbg, 4, "%s cpu %d\n", __func__, event->cpu); } /* Used to avoid races in checking concurrent access of counting and * sampling for pai_extension events. * * Only one instance of event pai_ext/NNPA_ALL/ for sampling is * allowed and when this event is running, no counting event is allowed. * Several counting events are allowed in parallel, but no sampling event * is allowed while one (or more) counting events are running. * * This function is called in process context and it is safe to block. * When the event initialization functions fails, no other call back will * be invoked. * * Allocate the memory for the event. */ static int paiext_alloc_cpu(struct perf_event *event, int cpu) { struct paiext_mapptr *mp; struct paiext_map *cpump; int rc; mutex_lock(&paiext_reserve_mutex); rc = paiext_root_alloc(); if (rc) goto unlock; mp = per_cpu_ptr(paiext_root.mapptr, cpu); cpump = mp->mapptr; if (!cpump) { /* Paiext_map allocated? */ rc = -ENOMEM; cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); if (!cpump) goto undo; /* Allocate memory for counter area and counter extraction. * These are * - a 512 byte block and requires 512 byte boundary alignment. * - a 1KB byte block and requires 1KB boundary alignment. * Only the first counting event has to allocate the area. * * Note: This works with commit 59bb47985c1d by default. * Backporting this to kernels without this commit might * need adjustment. */ mp->mapptr = cpump; cpump->area = kzalloc(PAIE1_CTRBLOCK_SZ, GFP_KERNEL); cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL); cpump->save = kvmalloc_array(paiext_cnt + 1, sizeof(struct pai_userdata), GFP_KERNEL); if (!cpump->save || !cpump->area || !cpump->paiext_cb) { paiext_free(mp); goto undo; } INIT_LIST_HEAD(&cpump->syswide_list); refcount_set(&cpump->refcnt, 1); rc = 0; } else { refcount_inc(&cpump->refcnt); } undo: if (rc) { /* Error in allocation of event, decrement anchor. Since * the event in not created, its destroy() function is never * invoked. Adjust the reference counter for the anchor. */ paiext_root_free(); } unlock: mutex_unlock(&paiext_reserve_mutex); /* If rc is non-zero, no increment of counter/sampler was done. */ return rc; } static int paiext_alloc(struct perf_event *event) { struct cpumask *maskptr; int cpu, rc = -ENOMEM; maskptr = kzalloc(sizeof(*maskptr), GFP_KERNEL); if (!maskptr) goto out; for_each_online_cpu(cpu) { rc = paiext_alloc_cpu(event, cpu); if (rc) { for_each_cpu(cpu, maskptr) paiext_event_destroy_cpu(event, cpu); kfree(maskptr); goto out; } cpumask_set_cpu(cpu, maskptr); } /* * On error all cpumask are freed and all events have been destroyed. * Save of which CPUs data structures have been allocated for. * Release them in paicrypt_event_destroy call back function * for this event. */ PAI_CPU_MASK(event) = maskptr; rc = 0; out: return rc; } /* The PAI extension 1 control block supports up to 128 entries. Return * the index within PAIE1_CB given the event number. Also validate event * number. */ static int paiext_event_valid(struct perf_event *event) { u64 cfg = event->attr.config; if (cfg >= PAI_NNPA_BASE && cfg <= PAI_NNPA_BASE + paiext_cnt) { /* Offset NNPA in paiext_cb */ event->hw.config_base = offsetof(struct paiext_cb, acc); return 0; } return -EINVAL; } /* Might be called on different CPU than the one the event is intended for. */ static int paiext_event_init(struct perf_event *event) { struct perf_event_attr *a = &event->attr; int rc; /* PMU pai_ext registered as PERF_TYPE_RAW, check event type */ if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) return -ENOENT; /* PAI extension event must be valid and in supported range */ rc = paiext_event_valid(event); if (rc) return rc; /* Allow only event NNPA_ALL for sampling. */ if (a->sample_period && a->config != PAI_NNPA_BASE) return -EINVAL; /* Prohibit exclude_user event selection */ if (a->exclude_user) return -EINVAL; /* Get a page to store last counter values for sampling */ if (a->sample_period) { PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); if (!PAI_SAVE_AREA(event)) return -ENOMEM; } if (event->cpu >= 0) rc = paiext_alloc_cpu(event, event->cpu); else rc = paiext_alloc(event); if (rc) { free_page(PAI_SAVE_AREA(event)); return rc; } event->destroy = paiext_event_destroy; if (a->sample_period) { a->sample_period = 1; a->freq = 0; /* Register for paicrypt_sched_task() to be called */ event->attach_state |= PERF_ATTACH_SCHED_CB; /* Add raw data which are the memory mapped counters */ a->sample_type |= PERF_SAMPLE_RAW; /* Turn off inheritance */ a->inherit = 0; } return 0; } static u64 paiext_getctr(unsigned long *area, int nr) { return area[nr]; } /* Read the counter values. Return value from location in buffer. For event * NNPA_ALL sum up all events. */ static u64 paiext_getdata(struct perf_event *event) { struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); struct paiext_map *cpump = mp->mapptr; u64 sum = 0; int i; if (event->attr.config != PAI_NNPA_BASE) return paiext_getctr(cpump->area, event->attr.config - PAI_NNPA_BASE); for (i = 1; i <= paiext_cnt; i++) sum += paiext_getctr(cpump->area, i); return sum; } static u64 paiext_getall(struct perf_event *event) { return paiext_getdata(event); } static void paiext_read(struct perf_event *event) { u64 prev, new, delta; prev = local64_read(&event->hw.prev_count); new = paiext_getall(event); local64_set(&event->hw.prev_count, new); delta = new - prev; local64_add(delta, &event->count); } static void paiext_start(struct perf_event *event, int flags) { struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); struct paiext_map *cpump = mp->mapptr; u64 sum; if (!event->attr.sample_period) { /* Counting */ sum = paiext_getall(event); /* Get current value */ local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ memcpy((void *)PAI_SAVE_AREA(event), cpump->area, PAIE1_CTRBLOCK_SZ); /* Enable context switch callback for system-wide sampling */ if (!(event->attach_state & PERF_ATTACH_TASK)) { list_add_tail(PAI_SWLIST(event), &cpump->syswide_list); perf_sched_cb_inc(event->pmu); } else { cpump->event = event; } } } static int paiext_add(struct perf_event *event, int flags) { struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); struct paiext_map *cpump = mp->mapptr; struct paiext_cb *pcb = cpump->paiext_cb; if (++cpump->active_events == 1) { get_lowcore()->aicd = virt_to_phys(cpump->paiext_cb); pcb->acc = virt_to_phys(cpump->area) | 0x1; /* Enable CPU instruction lookup for PAIE1 control block */ local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT); } if (flags & PERF_EF_START) paiext_start(event, PERF_EF_RELOAD); event->hw.state = 0; return 0; } static void paiext_have_sample(struct perf_event *, struct paiext_map *); static void paiext_stop(struct perf_event *event, int flags) { struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); struct paiext_map *cpump = mp->mapptr; if (!event->attr.sample_period) { /* Counting */ paiext_read(event); } else { /* Sampling */ if (!(event->attach_state & PERF_ATTACH_TASK)) { list_del(PAI_SWLIST(event)); perf_sched_cb_dec(event->pmu); } else { paiext_have_sample(event, cpump); cpump->event = NULL; } } event->hw.state = PERF_HES_STOPPED; } static void paiext_del(struct perf_event *event, int flags) { struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); struct paiext_map *cpump = mp->mapptr; struct paiext_cb *pcb = cpump->paiext_cb; paiext_stop(event, PERF_EF_UPDATE); if (--cpump->active_events == 0) { /* Disable CPU instruction lookup for PAIE1 control block */ local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT); pcb->acc = 0; get_lowcore()->aicd = 0; } } /* Create raw data and save it in buffer. Returns number of bytes copied. * Saves only positive counter entries of the form * 2 bytes: Number of counter * 8 bytes: Value of counter */ static size_t paiext_copy(struct pai_userdata *userdata, unsigned long *area, unsigned long *area_old) { int i, outidx = 0; for (i = 1; i <= paiext_cnt; i++) { u64 val = paiext_getctr(area, i); u64 val_old = paiext_getctr(area_old, i); if (val >= val_old) val -= val_old; else val = (~0ULL - val_old) + val + 1; if (val) { userdata[outidx].num = i; userdata[outidx].value = val; outidx++; } } return outidx * sizeof(*userdata); } /* Write sample when one or more counters values are nonzero. * * Note: The function paiext_sched_task() and paiext_push_sample() are not * invoked after function paiext_del() has been called because of function * perf_sched_cb_dec(). * The function paiext_sched_task() and paiext_push_sample() are only * called when sampling is active. Function perf_sched_cb_inc() * has been invoked to install function paiext_sched_task() as call back * to run at context switch time (see paiext_add()). * * This causes function perf_event_context_sched_out() and * perf_event_context_sched_in() to check whether the PMU has installed an * sched_task() callback. That callback is not active after paiext_del() * returns and has deleted the event on that CPU. */ static int paiext_push_sample(size_t rawsize, struct paiext_map *cpump, struct perf_event *event) { struct perf_sample_data data; struct perf_raw_record raw; struct pt_regs regs; int overflow; /* Setup perf sample */ memset(®s, 0, sizeof(regs)); memset(&raw, 0, sizeof(raw)); memset(&data, 0, sizeof(data)); perf_sample_data_init(&data, 0, event->hw.last_period); if (event->attr.sample_type & PERF_SAMPLE_TID) { data.tid_entry.pid = task_tgid_nr(current); data.tid_entry.tid = task_pid_nr(current); } if (event->attr.sample_type & PERF_SAMPLE_TIME) data.time = event->clock(); if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) data.id = event->id; if (event->attr.sample_type & PERF_SAMPLE_CPU) data.cpu_entry.cpu = smp_processor_id(); if (event->attr.sample_type & PERF_SAMPLE_RAW) { raw.frag.size = rawsize; raw.frag.data = cpump->save; perf_sample_save_raw_data(&data, &raw); } overflow = perf_event_overflow(event, &data, ®s); perf_event_update_userpage(event); /* Save NNPA lowcore area after read in event */ memcpy((void *)PAI_SAVE_AREA(event), cpump->area, PAIE1_CTRBLOCK_SZ); return overflow; } /* Check if there is data to be saved on schedule out of a task. */ static void paiext_have_sample(struct perf_event *event, struct paiext_map *cpump) { size_t rawsize; if (!event) return; rawsize = paiext_copy(cpump->save, cpump->area, (unsigned long *)PAI_SAVE_AREA(event)); if (rawsize) /* Incremented counters */ paiext_push_sample(rawsize, cpump, event); } /* Check if there is data to be saved on schedule out of a task. */ static void paiext_have_samples(void) { struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); struct paiext_map *cpump = mp->mapptr; struct perf_event *event; list_for_each_entry(event, &cpump->syswide_list, hw.tp_list) paiext_have_sample(event, cpump); } /* Called on schedule-in and schedule-out. No access to event structure, * but for sampling only event NNPA_ALL is allowed. */ static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) { /* We started with a clean page on event installation. So read out * results on schedule_out and if page was dirty, save old values. */ if (!sched_in) paiext_have_samples(); } /* Attribute definitions for pai extension1 interface. As with other CPU * Measurement Facilities, there is one attribute per mapped counter. * The number of mapped counters may vary per machine generation. Use * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction * to determine the number of mapped counters. The instructions returns * a positive number, which is the highest number of supported counters. * All counters less than this number are also supported, there are no * holes. A returned number of zero means no support for mapped counters. * * The identification of the counter is a unique number. The chosen range * is 0x1800 + offset in mapped kernel page. * All CPU Measurement Facility counters identifiers must be unique and * the numbers from 0 to 496 are already used for the CPU Measurement * Counter facility. Number 0x1000 to 0x103e are used for PAI cryptography * counters. * Numbers 0xb0000, 0xbc000 and 0xbd000 are already * used for the CPU Measurement Sampling facility. */ PMU_FORMAT_ATTR(event, "config:0-63"); static struct attribute *paiext_format_attr[] = { &format_attr_event.attr, NULL, }; static struct attribute_group paiext_events_group = { .name = "events", .attrs = NULL, /* Filled in attr_event_init() */ }; static struct attribute_group paiext_format_group = { .name = "format", .attrs = paiext_format_attr, }; static const struct attribute_group *paiext_attr_groups[] = { &paiext_events_group, &paiext_format_group, NULL, }; /* Performance monitoring unit for mapped counters */ static struct pmu paiext = { .task_ctx_nr = perf_hw_context, .event_init = paiext_event_init, .add = paiext_add, .del = paiext_del, .start = paiext_start, .stop = paiext_stop, .read = paiext_read, .sched_task = paiext_sched_task, .attr_groups = paiext_attr_groups, }; /* List of symbolic PAI extension 1 NNPA counter names. */ static const char * const paiext_ctrnames[] = { [0] = "NNPA_ALL", [1] = "NNPA_ADD", [2] = "NNPA_SUB", [3] = "NNPA_MUL", [4] = "NNPA_DIV", [5] = "NNPA_MIN", [6] = "NNPA_MAX", [7] = "NNPA_LOG", [8] = "NNPA_EXP", [9] = "NNPA_IBM_RESERVED_9", [10] = "NNPA_RELU", [11] = "NNPA_TANH", [12] = "NNPA_SIGMOID", [13] = "NNPA_SOFTMAX", [14] = "NNPA_BATCHNORM", [15] = "NNPA_MAXPOOL2D", [16] = "NNPA_AVGPOOL2D", [17] = "NNPA_LSTMACT", [18] = "NNPA_GRUACT", [19] = "NNPA_CONVOLUTION", [20] = "NNPA_MATMUL_OP", [21] = "NNPA_MATMUL_OP_BCAST23", [22] = "NNPA_SMALLBATCH", [23] = "NNPA_LARGEDIM", [24] = "NNPA_SMALLTENSOR", [25] = "NNPA_1MFRAME", [26] = "NNPA_2GFRAME", [27] = "NNPA_ACCESSEXCEPT", [28] = "NNPA_TRANSFORM", [29] = "NNPA_GELU", [30] = "NNPA_MOMENTS", [31] = "NNPA_LAYERNORM", [32] = "NNPA_MATMUL_OP_BCAST1", [33] = "NNPA_SQRT", [34] = "NNPA_INVSQRT", [35] = "NNPA_NORM", [36] = "NNPA_REDUCE", }; static void __init attr_event_free(struct attribute **attrs, int num) { struct perf_pmu_events_attr *pa; struct device_attribute *dap; int i; for (i = 0; i < num; i++) { dap = container_of(attrs[i], struct device_attribute, attr); pa = container_of(dap, struct perf_pmu_events_attr, attr); kfree(pa); } kfree(attrs); } static int __init attr_event_init_one(struct attribute **attrs, int num) { struct perf_pmu_events_attr *pa; /* Index larger than array_size, no counter name available */ if (num >= ARRAY_SIZE(paiext_ctrnames)) { attrs[num] = NULL; return 0; } pa = kzalloc(sizeof(*pa), GFP_KERNEL); if (!pa) return -ENOMEM; sysfs_attr_init(&pa->attr.attr); pa->id = PAI_NNPA_BASE + num; pa->attr.attr.name = paiext_ctrnames[num]; pa->attr.attr.mode = 0444; pa->attr.show = cpumf_events_sysfs_show; pa->attr.store = NULL; attrs[num] = &pa->attr.attr; return 0; } /* Create PMU sysfs event attributes on the fly. */ static int __init attr_event_init(void) { struct attribute **attrs; int ret, i; attrs = kmalloc_array(paiext_cnt + 2, sizeof(*attrs), GFP_KERNEL); if (!attrs) return -ENOMEM; for (i = 0; i <= paiext_cnt; i++) { ret = attr_event_init_one(attrs, i); if (ret) { attr_event_free(attrs, i); return ret; } } attrs[i] = NULL; paiext_events_group.attrs = attrs; return 0; } static int __init paiext_init(void) { struct qpaci_info_block ib; int rc = -ENOMEM; if (!test_facility(197)) return 0; qpaci(&ib); paiext_cnt = ib.num_nnpa; if (paiext_cnt >= PAI_NNPA_MAXCTR) paiext_cnt = PAI_NNPA_MAXCTR; if (!paiext_cnt) return 0; rc = attr_event_init(); if (rc) { pr_err("Creation of PMU " KMSG_COMPONENT " /sysfs failed\n"); return rc; } /* Setup s390dbf facility */ paiext_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128); if (!paiext_dbg) { pr_err("Registration of s390dbf " KMSG_COMPONENT " failed\n"); rc = -ENOMEM; goto out_init; } debug_register_view(paiext_dbg, &debug_sprintf_view); rc = perf_pmu_register(&paiext, KMSG_COMPONENT, -1); if (rc) { pr_err("Registration of " KMSG_COMPONENT " PMU failed with " "rc=%i\n", rc); goto out_pmu; } return 0; out_pmu: debug_unregister_view(paiext_dbg, &debug_sprintf_view); debug_unregister(paiext_dbg); out_init: attr_event_free(paiext_events_group.attrs, ARRAY_SIZE(paiext_ctrnames) + 1); return rc; } device_initcall(paiext_init);