From 82cb8f95ed60352617856f1e110225e3ba4bd21a Mon Sep 17 00:00:00 2001 From: Ken Sato Date: Mon, 18 Apr 2016 13:07:45 +0900 Subject: [PATCH] update PAPI support. --- arch/x86/kernel/perfctr.c | 205 +++++++++++++++++++++++++++++++++--- kernel/syscall.c | 120 ++++++++++----------- lib/include/ihk/perfctr.h | 5 + lib/include/mc_perf_event.h | 33 ++++-- 4 files changed, 274 insertions(+), 89 deletions(-) diff --git a/arch/x86/kernel/perfctr.c b/arch/x86/kernel/perfctr.c index 6b3c9cff..188bb942 100644 --- a/arch/x86/kernel/perfctr.c +++ b/arch/x86/kernel/perfctr.c @@ -14,6 +14,7 @@ #include extern unsigned int *x86_march_perfmap; +static unsigned long pmc_status = 0x0; #define X86_CR4_PCE 0x00000100 @@ -33,20 +34,51 @@ static int set_perfctr_x86_direct(int counter, int mode, unsigned int value) return -EINVAL; } - if (mode & PERFCTR_USER_MODE) { + // clear mode flags + value &= ~(3 << 16); + + // set mode flags + if(mode & PERFCTR_USER_MODE) { value |= 1 << 16; - } - if (mode & PERFCTR_KERNEL_MODE) { + } + if(mode & PERFCTR_KERNEL_MODE) { value |= 1 << 17; - } + } + // wrmsr(MSR_PERF_GLOBAL_CTRL, 0); value |= (1 << 22) | (1 << 18); /* EN */ + value |= (1 << 20); /* Enable overflow interrupt */ wrmsr(MSR_IA32_PERFEVTSEL0 + counter, value); - kprintf("wrmsr: %d <= %x\n", MSR_PERF_GLOBAL_CTRL, 0); + //kprintf("wrmsr: %d <= %x\n", MSR_PERF_GLOBAL_CTRL, 0); kprintf("wrmsr: %d <= %x\n", MSR_IA32_PERFEVTSEL0 + counter, value); + + return 0; +} + +static int set_pmc_x86_direct(int counter, unsigned long val) +{ + unsigned long cnt_bit = 0; + + if (counter < 0) { + return -EINVAL; + } + + cnt_bit = 1UL << counter; + if ( cnt_bit & X86_IA32_PERF_COUNTERS_MASK ) { + // set generic pmc + wrmsr(MSR_IA32_PMC0 + counter, val); + } + else if ( cnt_bit & X86_IA32_FIXED_PERF_COUNTERS_MASK ) { + // set fixed pmc + wrmsr(MSR_IA32_FIXED_CTR0 + counter - X86_IA32_BASE_FIXED_PERF_COUNTERS, val); + } + else { + return -EINVAL; + } + return 0; } @@ -57,6 +89,37 @@ static int set_perfctr_x86(int counter, int event, int mask, int inv, int count, CVAL2(event, mask, inv, count)); } +static int set_fixed_counter(int counter, int mode) +{ + unsigned long value = 0; + unsigned int ctr_mask = 0x7; + int counter_idx = counter - X86_IA32_BASE_FIXED_PERF_COUNTERS ; + unsigned int set_val = 0; + + if (counter_idx < 0 || counter_idx >= X86_IA32_NUM_FIXED_PERF_COUNTERS) { + return -EINVAL; + } + + // clear specified fixed counter info + value = rdmsr(MSR_PERF_FIXED_CTRL); + ctr_mask <<= counter_idx * 4; + value &= ~ctr_mask; + + if (mode & PERFCTR_USER_MODE) { + set_val |= 1 << 1; + } + if (mode & PERFCTR_KERNEL_MODE) { + set_val |= 1; + } + + set_val <<= counter_idx * 4; + value |= set_val; + + wrmsr(MSR_PERF_FIXED_CTRL, value); + + return 0; +} + int ihk_mc_perfctr_init_raw(int counter, unsigned int code, int mode) { if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) { @@ -86,14 +149,15 @@ extern void x86_march_perfctr_start(unsigned long counter_mask); int ihk_mc_perfctr_start(unsigned long counter_mask) { - unsigned int value = 0; + unsigned long value = 0; + unsigned long mask = X86_IA32_PERF_COUNTERS_MASK | X86_IA32_FIXED_PERF_COUNTERS_MASK; #ifdef HAVE_MARCH_PERFCTR_START x86_march_perfctr_start(counter_mask); #endif - counter_mask &= ((1 << X86_IA32_NUM_PERF_COUNTERS) - 1); + counter_mask &= mask; value = rdmsr(MSR_PERF_GLOBAL_CTRL); - value |= counter_mask; + value |= counter_mask; wrmsr(MSR_PERF_GLOBAL_CTRL, value); return 0; @@ -101,9 +165,10 @@ int ihk_mc_perfctr_start(unsigned long counter_mask) int ihk_mc_perfctr_stop(unsigned long counter_mask) { - unsigned int value; + unsigned long value; + unsigned long mask = X86_IA32_PERF_COUNTERS_MASK | X86_IA32_FIXED_PERF_COUNTERS_MASK; - counter_mask &= ((1 << X86_IA32_NUM_PERF_COUNTERS) - 1); + counter_mask &= mask; value = rdmsr(MSR_PERF_GLOBAL_CTRL); value &= ~counter_mask; wrmsr(MSR_PERF_GLOBAL_CTRL, value); @@ -111,17 +176,48 @@ int ihk_mc_perfctr_stop(unsigned long counter_mask) return 0; } -int ihk_mc_perfctr_reset(int counter) +// init for fixed counter +int ihk_mc_perfctr_fixed_init(int counter, int mode) { - if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) { + unsigned long value = 0; + unsigned int ctr_mask = 0x7; + int counter_idx = counter - X86_IA32_BASE_FIXED_PERF_COUNTERS ; + unsigned int set_val = 0; + + if (counter_idx < 0 || counter_idx >= X86_IA32_NUM_FIXED_PERF_COUNTERS) { return -EINVAL; } - wrmsr(MSR_IA32_PMC0 + counter, 0); + // clear specified fixed counter info + value = rdmsr(MSR_PERF_FIXED_CTRL); + ctr_mask <<= counter_idx * 4; + value &= ~ctr_mask; + + if (mode & PERFCTR_USER_MODE) { + set_val |= 1 << 1; + } + if (mode & PERFCTR_KERNEL_MODE) { + set_val |= 1; + } + + set_val <<= counter_idx * 4; + value |= set_val; + + wrmsr(MSR_PERF_FIXED_CTRL, value); return 0; } +int ihk_mc_perfctr_reset(int counter) +{ + return set_pmc_x86_direct(counter, 0); +} + +int ihk_mc_perfctr_set(int counter, unsigned long val) +{ + return set_pmc_x86_direct(counter, val); +} + int ihk_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value) { int i, j; @@ -137,10 +233,89 @@ int ihk_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value) unsigned long ihk_mc_perfctr_read(int counter) { - if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) { + unsigned long retval = 0; + unsigned long cnt_bit = 0; + + if (counter < 0) { return -EINVAL; } - return rdpmc(counter); + cnt_bit = 1UL << counter; + + if ( cnt_bit & X86_IA32_PERF_COUNTERS_MASK ) { + // read generic pmc + retval = rdpmc(counter); + } + else if ( cnt_bit & X86_IA32_FIXED_PERF_COUNTERS_MASK ) { + // read fixed pmc + retval = rdpmc((1 << 30) + (counter - X86_IA32_BASE_FIXED_PERF_COUNTERS)); + } + else { + retval = -EINVAL; + } + + return retval; +} + +// read by rdmsr +unsigned long ihk_mc_perfctr_read_msr(int counter) +{ + unsigned int idx = 0; + unsigned long retval = 0; + unsigned long cnt_bit = 0; + + if (counter < 0) { + return -EINVAL; + } + + cnt_bit = 1UL << counter; + + if ( cnt_bit & X86_IA32_PERF_COUNTERS_MASK ) { + // read generic pmc + idx = MSR_IA32_PMC0 + counter; + retval = (unsigned long) rdmsr(idx); + } + else if ( cnt_bit & X86_IA32_FIXED_PERF_COUNTERS_MASK ) { + // read fixed pmc + idx = MSR_IA32_FIXED_CTR0 + counter; + retval = (unsigned long) rdmsr(idx); + } + else { + retval = -EINVAL; + } + + return retval; +} + +int ihk_mc_perfctr_alloc_counter() +{ + int i = 0; + int ret = -1; + + // find avail generic counter + for(i = 0; i < X86_IA32_NUM_PERF_COUNTERS; i++) { + if(!(pmc_status & (1 << i))) { + ret = i; + pmc_status |= (1 << i); + break; + } + } + + if(ret < 0){ + return ret; + } + + return ret; +} + +void ihk_mc_perfctr_release_counter(int counter) +{ + unsigned long value = 0; + + value = rdmsr(MSR_PERF_GLOBAL_CTRL); + value &= ~(1UL << counter); + pmc_status &= ~(1UL << counter); + + wrmsr(MSR_PERF_GLOBAL_CTRL, 0); } diff --git a/kernel/syscall.c b/kernel/syscall.c index 9054669f..3cf052d5 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -127,9 +127,6 @@ int prepare_process_ranges_args_envs(struct thread *thread, static void do_mod_exit(int status); #endif -// for perf_event -static unsigned int counter_flag[X86_IA32_NUM_PERF_COUNTERS] = {}; - static void send_syscall(struct syscall_request *req, int cpu, int pid) { struct ikc_scd_packet packet; @@ -2389,7 +2386,7 @@ SYSCALL_DECLARE(read) ihk_mc_spinlock_unlock(&proc->mckfd_lock, irqstate); if(fdp && fdp->read_cb){ -kprintf("read: found system fd %d\n", fd); +//kprintf("read: found system fd %d\n", fd); rc = fdp->read_cb(fdp, ctx); } else{ @@ -2414,7 +2411,7 @@ SYSCALL_DECLARE(ioctl) ihk_mc_spinlock_unlock(&proc->mckfd_lock, irqstate); if(fdp && fdp->ioctl_cb){ -kprintf("ioctl: found system fd %d\n", fd); +//kprintf("ioctl: found system fd %d\n", fd); rc = fdp->ioctl_cb(fdp, ctx); } else{ @@ -2439,7 +2436,7 @@ SYSCALL_DECLARE(close) break; if(fdp){ -kprintf("close: found system fd %d pid=%d\n", fd, proc->pid); +//kprintf("close: found system fd %d pid=%d\n", fd, proc->pid); if(fdq) fdq->next = fdp->next; else @@ -2601,47 +2598,12 @@ SYSCALL_DECLARE(signalfd4) return sfd->fd; } -int -release_counter(int cpu_id, int cnt) -{ - int ret = -1; - - if(cnt >= 0 && cnt < X86_IA32_NUM_PERF_COUNTERS) { - counter_flag[cpu_id] &= ~(1 << cnt); - ret = 0; - } - - return ret; -} - -int -get_avail_counter(int cpu_id, int cnt) -{ - int i = 0; - int ret = -1; - // find avail counter - if(cnt < 0) { - for(i = 0; i < X86_IA32_NUM_PERF_COUNTERS; i++) { - if(!(counter_flag[cpu_id] & 1 << i)) { - ret = i; - break; - } - } - // check specified counter is available. - } else if (cnt < X86_IA32_NUM_PERF_COUNTERS) { - if(counter_flag[cpu_id] ^ 1 << cnt) { - ret = cnt; - } - } - - return ret; -} - int -perf_counter_init(struct perf_event_attr *attr, int cpu_id, int counter) +perf_counter_init(struct mc_perf_event *event) { int ret = 0; enum ihk_perfctr_type type; + struct perf_event_attr *attr = &event->attr; int mode = 0x00; if(!attr->exclude_kernel) { @@ -2664,28 +2626,35 @@ perf_counter_init(struct perf_event_attr *attr, int cpu_id, int counter) type = PERFCTR_MAX_TYPE; } - ret = ihk_mc_perfctr_init(counter, type, mode); + event->counter = ihk_mc_perfctr_alloc_counter(); + ret = ihk_mc_perfctr_init(event->counter, type, mode); } else if(attr->type == PERF_TYPE_RAW) { + // PAPI_REF_CYC counted by fixed counter + if((attr->config & 0x0000ffff) == 0x00000300) { + event->counter = 2 + X86_IA32_BASE_FIXED_PERF_COUNTERS; + ret = ihk_mc_perfctr_fixed_init(event->counter, mode); + return ret; + } + // apply MODE to config(event_code) attr->config &= ~(3 << 16); - if(mode &= PERFCTR_USER_MODE) { + if(mode & PERFCTR_USER_MODE) { attr->config |= 1 << 16; } - if(mode &= PERFCTR_KERNEL_MODE) { + if(mode & PERFCTR_KERNEL_MODE) { attr->config |= 1 << 17; } - ret = ihk_mc_perfctr_init_raw(counter, attr->config, mode); + event->counter = ihk_mc_perfctr_alloc_counter(); + ret = ihk_mc_perfctr_init_raw(event->counter, attr->config, mode); } else { // Not supported type. ret = -1; } if(ret >= 0) { - ret = ihk_mc_perfctr_reset(counter); - ret = ihk_mc_perfctr_start(1 << counter); - counter_flag[cpu_id] |= 1 << counter; + ret = ihk_mc_perfctr_reset(event->counter); } return ret; @@ -2770,6 +2739,28 @@ perf_read(struct mckfd *sfd, ihk_mc_user_context_t *ctx) } +static void +perf_start(struct mc_perf_event *event) +{ + int counter = event->counter; + + if((1UL << counter & X86_IA32_PERF_COUNTERS_MASK) | + (1UL << counter & X86_IA32_FIXED_PERF_COUNTERS_MASK)) { + ihk_mc_perfctr_start(1UL << counter); + } +} + +static void +perf_stop(struct mc_perf_event *event) +{ + int counter = event->counter; + + if((1UL << counter & X86_IA32_PERF_COUNTERS_MASK) | + (1UL << counter & X86_IA32_FIXED_PERF_COUNTERS_MASK)) { + ihk_mc_perfctr_stop(1UL << counter); + } +} + static int perf_ioctl(struct mckfd *sfd, ihk_mc_user_context_t *ctx) { @@ -2779,16 +2770,16 @@ perf_ioctl(struct mckfd *sfd, ihk_mc_user_context_t *ctx) switch (cmd) { case PERF_EVENT_IOC_ENABLE: - ihk_mc_perfctr_start(1 << counter); + perf_start(event); break; case PERF_EVENT_IOC_DISABLE: - ihk_mc_perfctr_stop(1 << counter); + perf_stop(event); break; case PERF_EVENT_IOC_RESET: - ihk_mc_perfctr_reset(counter); + ihk_mc_perfctr_set(counter, event->sample_freq * -1); break; case PERF_EVENT_IOC_REFRESH: - ihk_mc_perfctr_reset(counter); + ihk_mc_perfctr_set(counter, event->sample_freq * -1); break; default : return -1; @@ -2801,10 +2792,9 @@ static int perf_close(struct mckfd *sfd, ihk_mc_user_context_t *ctx) { struct mc_perf_event *event = (struct mc_perf_event*)sfd->data; - int cpu_id = event->cpu_id; + ihk_mc_perfctr_release_counter(event->counter); kfree(event); - release_counter(cpu_id, event->counter); return 0; } @@ -2827,11 +2817,12 @@ perf_mmap(struct mckfd *sfd, ihk_mc_user_context_t *ctx) // setup perf_event_mmap_page page = (struct perf_event_mmap_page *)rc; - page->cap_usr_rdpmc = 0; + page->cap_user_rdpmc = 1; return rc; } +extern unsigned long ihk_mc_perfctr_get_info(); SYSCALL_DECLARE(perf_event_open) { struct syscall_request request IHK_DMA_ALIGN; @@ -2872,15 +2863,15 @@ SYSCALL_DECLARE(perf_event_open) if(not_supported_flag) { return -1; } - + // process of perf_event_open event = kmalloc(sizeof(struct mc_perf_event), IHK_MC_AP_NOWAIT); + if(!event) + return -ENOMEM; event->cpu_id = thread->cpu_id; event->attr = (struct perf_event_attr)*attr; - event->counter = get_avail_counter(event->cpu_id, -1); - if(event->counter < 0) { - return -1; - } + + event->sample_freq = attr->sample_freq; event->nr_siblings = 0; INIT_LIST_HEAD(&event->group_entry); INIT_LIST_HEAD(&event->sibling_list); @@ -2897,7 +2888,10 @@ SYSCALL_DECLARE(perf_event_open) } } - perf_counter_init(attr, event->cpu_id, event->counter); + if(perf_counter_init(event) < 0) + return -1; + if(event->counter < 0) + return -1; request.number = __NR_perf_event_open; request.args[0] = 0; diff --git a/lib/include/ihk/perfctr.h b/lib/include/ihk/perfctr.h index 789edd87..e432413d 100644 --- a/lib/include/ihk/perfctr.h +++ b/lib/include/ihk/perfctr.h @@ -52,9 +52,14 @@ int ihk_mc_perfctr_init(int counter, enum ihk_perfctr_type type, int mode); int ihk_mc_perfctr_init_raw(int counter, unsigned int code, int mode); int ihk_mc_perfctr_start(unsigned long counter_mask); int ihk_mc_perfctr_stop(unsigned long counter_mask); +int ihk_mc_perfctr_fixed_init(int counter, int mode); int ihk_mc_perfctr_reset(int counter); +int ihk_mc_perfctr_set(int counter, unsigned long value); int ihk_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value); unsigned long ihk_mc_perfctr_read(int counter); +unsigned long ihk_mc_perfctr_read_msr(int counter); +int ihk_mc_perfctr_alloc_counter(); +void ihk_mc_perfctr_release_counter(int counter); #endif diff --git a/lib/include/mc_perf_event.h b/lib/include/mc_perf_event.h index 545d57ab..670fe081 100644 --- a/lib/include/mc_perf_event.h +++ b/lib/include/mc_perf_event.h @@ -1,6 +1,8 @@ #ifndef MC_PERF_EVNET_H #define MC_PERF_EVENT_H +#include + struct perf_event_attr; /** @@ -217,6 +219,8 @@ struct mc_perf_event { struct perf_event_attr attr; int cpu_id; int counter; + unsigned long sample_freq; + struct mc_perf_event *group_leader; struct list_head sibling_list; int nr_siblings; @@ -224,25 +228,32 @@ struct mc_perf_event { }; struct perf_event_mmap_page { - unsigned int version; - unsigned int compat_version; - unsigned int lock; - unsigned int index; - long offset; - unsigned long time_enabled; - unsigned long time_running; + unsigned int version; // version number of this structure + unsigned int compat_version; // lowest version this is compat with + unsigned int lock; // seqlock for synchronization + unsigned int index; // hardware event identifier + long offset; // add to hardware event value + unsigned long time_enabled; // time evet active + unsigned long time_running; // time event on cpu union { unsigned long capabilities; - unsigned long cap_usr_time : 1, - cap_usr_rdpmc : 1, - cap_____res : 62; + struct { + unsigned long cap_bit0 : 1, // Always 0, deprecated + cap_bit0_is_deprecated : 1, // Always 1, signals that bit 0 is zero + cap_user_rdpmc : 1, // The RDPMC instruction can be used to read counts + cap_user_time : 1, // The time_* fields are used + cap_user_time_zero : 1, // The time_zero field is used + cap_____res : 59; + }; }; unsigned short pmc_width; unsigned short time_shift; unsigned int time_mult; unsigned long time_offset; + unsigned long time_zero; + unsigned int size; - unsigned long __reserved[120]; + unsigned char __reserved[118*8+4]; unsigned long data_head; unsigned long data_tail; };