update PAPI support.

arch/x86/kernel/perfctr.c

@@ -14,6 +14,7 @@
 #include <registers.h>
 
 extern unsigned int *x86_march_perfmap;
+static unsigned long pmc_status = 0x0;
 
 #define X86_CR4_PCE     0x00000100
 
@@ -33,20 +34,51 @@ static int set_perfctr_x86_direct(int counter, int mode, unsigned int value)
 		return -EINVAL;
 	}
 
-	if (mode & PERFCTR_USER_MODE) {
+	// clear mode flags
+	value &= ~(3 << 16);
+
+	// set mode flags
+	if(mode & PERFCTR_USER_MODE) {
 		value |= 1 << 16;
-	}
+        }
-	if (mode & PERFCTR_KERNEL_MODE) {
+        if(mode & PERFCTR_KERNEL_MODE) {
 		value |= 1 << 17;
-	}
+        }
+
     //	wrmsr(MSR_PERF_GLOBAL_CTRL, 0);
 
 	value |= (1 << 22) | (1 << 18); /* EN */
+	value |= (1 << 20); /* Enable overflow interrupt */
 
 	wrmsr(MSR_IA32_PERFEVTSEL0 + counter, value);
 
-	kprintf("wrmsr: %d <= %x\n", MSR_PERF_GLOBAL_CTRL, 0);
+	//kprintf("wrmsr: %d <= %x\n", MSR_PERF_GLOBAL_CTRL, 0);
 	kprintf("wrmsr: %d <= %x\n", MSR_IA32_PERFEVTSEL0 + counter, value);
+
+	return 0;
+}
+
+static int set_pmc_x86_direct(int counter, unsigned long val)
+{
+	unsigned long cnt_bit = 0;
+
+	if (counter < 0) {
+		return -EINVAL;
+	}
+
+	cnt_bit = 1UL << counter;
+	if ( cnt_bit & X86_IA32_PERF_COUNTERS_MASK ) {
+		// set generic pmc
+		wrmsr(MSR_IA32_PMC0 + counter, val);
+	}
+	else if ( cnt_bit & X86_IA32_FIXED_PERF_COUNTERS_MASK ) {
+		// set fixed pmc
+		wrmsr(MSR_IA32_FIXED_CTR0 + counter - X86_IA32_BASE_FIXED_PERF_COUNTERS, val);
+	}
+	else {
+		return -EINVAL;
+	}
+
 	return 0;
 }
 
@@ -57,6 +89,37 @@ static int set_perfctr_x86(int counter, int event, int mask, int inv, int count,
 	                              CVAL2(event, mask, inv, count));
 }
 
+static int set_fixed_counter(int counter, int mode)
+{
+	unsigned long value = 0;
+	unsigned int  ctr_mask = 0x7;
+	int counter_idx = counter - X86_IA32_BASE_FIXED_PERF_COUNTERS ;
+	unsigned int  set_val = 0;
+
+	if (counter_idx < 0 || counter_idx >= X86_IA32_NUM_FIXED_PERF_COUNTERS) {
+		return -EINVAL;
+	}
+
+	// clear specified fixed counter info
+	value = rdmsr(MSR_PERF_FIXED_CTRL);
+	ctr_mask <<= counter_idx * 4;
+	value &= ~ctr_mask;
+	
+	if (mode & PERFCTR_USER_MODE) {
+		set_val |= 1 << 1;
+	}
+	if (mode & PERFCTR_KERNEL_MODE) {
+		set_val |= 1;
+	}
+
+	set_val <<= counter_idx * 4;
+	value |= set_val;
+
+	wrmsr(MSR_PERF_FIXED_CTRL, value);
+
+	return 0;
+}
+
 int ihk_mc_perfctr_init_raw(int counter, unsigned int code, int mode)
 {
 	if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) {
@@ -86,14 +149,15 @@ extern void x86_march_perfctr_start(unsigned long counter_mask);
 
 int ihk_mc_perfctr_start(unsigned long counter_mask)
 {
-	unsigned int value = 0;
+	unsigned long value = 0;
+	unsigned long mask = X86_IA32_PERF_COUNTERS_MASK | X86_IA32_FIXED_PERF_COUNTERS_MASK;
 
 #ifdef HAVE_MARCH_PERFCTR_START
 	x86_march_perfctr_start(counter_mask);
 #endif
-	counter_mask &= ((1 << X86_IA32_NUM_PERF_COUNTERS) - 1);
+	counter_mask &= mask;
 	value = rdmsr(MSR_PERF_GLOBAL_CTRL);
-    value |= counter_mask;
+	value |= counter_mask;
 	wrmsr(MSR_PERF_GLOBAL_CTRL, value);
 
 	return 0;
@@ -101,9 +165,10 @@ int ihk_mc_perfctr_start(unsigned long counter_mask)
 
 int ihk_mc_perfctr_stop(unsigned long counter_mask)
 {
-	unsigned int value;
+	unsigned long value;
+	unsigned long mask = X86_IA32_PERF_COUNTERS_MASK | X86_IA32_FIXED_PERF_COUNTERS_MASK;
 
-	counter_mask &= ((1 << X86_IA32_NUM_PERF_COUNTERS) - 1);
+	counter_mask &= mask;
 	value = rdmsr(MSR_PERF_GLOBAL_CTRL);
 	value &= ~counter_mask;
 	wrmsr(MSR_PERF_GLOBAL_CTRL, value);
@@ -111,17 +176,48 @@ int ihk_mc_perfctr_stop(unsigned long counter_mask)
 	return 0;
 }
 
-int ihk_mc_perfctr_reset(int counter)
+// init for fixed counter
+int ihk_mc_perfctr_fixed_init(int counter, int mode)
 {
-	if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) {
+	unsigned long value = 0;
+	unsigned int  ctr_mask = 0x7;
+	int counter_idx = counter - X86_IA32_BASE_FIXED_PERF_COUNTERS ;
+	unsigned int  set_val = 0;
+
+	if (counter_idx < 0 || counter_idx >= X86_IA32_NUM_FIXED_PERF_COUNTERS) {
 		return -EINVAL;
 	}
 
-	wrmsr(MSR_IA32_PMC0 + counter, 0);
+	// clear specified fixed counter info
+	value = rdmsr(MSR_PERF_FIXED_CTRL);
+	ctr_mask <<= counter_idx * 4;
+	value &= ~ctr_mask;
+	
+	if (mode & PERFCTR_USER_MODE) {
+		set_val |= 1 << 1;
+	}
+	if (mode & PERFCTR_KERNEL_MODE) {
+		set_val |= 1;
+	}
+
+	set_val <<= counter_idx * 4;
+	value |= set_val;
+
+	wrmsr(MSR_PERF_FIXED_CTRL, value);
 
 	return 0;
 }
 
+int ihk_mc_perfctr_reset(int counter)
+{
+	return set_pmc_x86_direct(counter, 0);
+}
+
+int ihk_mc_perfctr_set(int counter, unsigned long val)
+{
+	return set_pmc_x86_direct(counter, val);
+}
+
 int ihk_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value)
 {
 	int i, j;
@@ -137,10 +233,89 @@ int ihk_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value)
 
 unsigned long ihk_mc_perfctr_read(int counter)
 {
-	if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) {
+	unsigned long retval = 0;
+	unsigned long cnt_bit = 0;
+
+	if (counter < 0) {
 		return -EINVAL;
 	}
 
-	return rdpmc(counter);
+	cnt_bit = 1UL << counter;
+
+	if ( cnt_bit & X86_IA32_PERF_COUNTERS_MASK ) {
+		// read generic pmc
+		retval = rdpmc(counter);
+	}
+	else if ( cnt_bit & X86_IA32_FIXED_PERF_COUNTERS_MASK ) {
+		// read fixed pmc
+		retval = rdpmc((1 << 30) + (counter - X86_IA32_BASE_FIXED_PERF_COUNTERS));
+	}
+	else {
+		retval = -EINVAL;
+	}
+
+	return retval;
+}
+
+// read by rdmsr
+unsigned long ihk_mc_perfctr_read_msr(int counter)
+{
+	unsigned int idx = 0;
+	unsigned long retval = 0;
+	unsigned long cnt_bit = 0;
+
+	if (counter < 0) {
+		return -EINVAL;
+	}
+
+	cnt_bit = 1UL << counter;
+
+	if ( cnt_bit & X86_IA32_PERF_COUNTERS_MASK ) {
+		// read generic pmc
+		idx = MSR_IA32_PMC0 + counter;
+		retval = (unsigned long) rdmsr(idx);
+	}
+	else if ( cnt_bit & X86_IA32_FIXED_PERF_COUNTERS_MASK ) {
+		// read fixed pmc
+		idx = MSR_IA32_FIXED_CTR0 + counter;
+		retval = (unsigned long) rdmsr(idx);
+	}
+	else {
+		retval = -EINVAL;
+	}
+
+	return retval;
+}
+
+int ihk_mc_perfctr_alloc_counter()
+{
+	int i = 0;
+        int ret = -1;
+
+        // find avail generic counter
+        for(i = 0; i < X86_IA32_NUM_PERF_COUNTERS; i++) {
+		if(!(pmc_status & (1 << i))) {
+			ret = i;
+			pmc_status |= (1 << i);
+			break;
+		}
+	}
+
+	if(ret < 0){
+		return ret;
+	}
+
+        return ret;
+}
+
+void ihk_mc_perfctr_release_counter(int counter)
+{
+	unsigned long value = 0;
+
+	value = rdmsr(MSR_PERF_GLOBAL_CTRL);
+	value &= ~(1UL << counter);
+	pmc_status &= ~(1UL << counter);
+
+	wrmsr(MSR_PERF_GLOBAL_CTRL, 0);
 }
 

kernel/syscall.c

@@ -127,9 +127,6 @@ int prepare_process_ranges_args_envs(struct thread *thread,
 static void do_mod_exit(int status);
 #endif
 
-// for perf_event
-static unsigned int counter_flag[X86_IA32_NUM_PERF_COUNTERS] = {};
-
 static void send_syscall(struct syscall_request *req, int cpu, int pid)
 {
 	struct ikc_scd_packet packet;
@@ -2389,7 +2386,7 @@ SYSCALL_DECLARE(read)
 	ihk_mc_spinlock_unlock(&proc->mckfd_lock, irqstate);
 
 	if(fdp && fdp->read_cb){
-kprintf("read: found system fd %d\n", fd);
+//kprintf("read: found system fd %d\n", fd);
 		rc = fdp->read_cb(fdp, ctx);
 	}
 	else{
@@ -2414,7 +2411,7 @@ SYSCALL_DECLARE(ioctl)
 	ihk_mc_spinlock_unlock(&proc->mckfd_lock, irqstate);
 
 	if(fdp && fdp->ioctl_cb){
-kprintf("ioctl: found system fd %d\n", fd);
+//kprintf("ioctl: found system fd %d\n", fd);
 		rc = fdp->ioctl_cb(fdp, ctx);
 	}
 	else{
@@ -2439,7 +2436,7 @@ SYSCALL_DECLARE(close)
 			break;
 
 	if(fdp){
-kprintf("close: found system fd %d pid=%d\n", fd, proc->pid);
+//kprintf("close: found system fd %d pid=%d\n", fd, proc->pid);
 		if(fdq)
 			fdq->next = fdp->next;
 		else
@@ -2601,47 +2598,12 @@ SYSCALL_DECLARE(signalfd4)
 	return sfd->fd;
 }
 
-int
-release_counter(int cpu_id, int cnt)
-{
-	int ret = -1;
-
-	if(cnt >= 0 && cnt < X86_IA32_NUM_PERF_COUNTERS) {
-		counter_flag[cpu_id] &= ~(1 << cnt);
-		ret = 0;
-	}
-
-	return ret;
-}
-
-int
-get_avail_counter(int cpu_id, int cnt)
-{
-	int i = 0;
-	int ret = -1;
-	// find avail counter
-	if(cnt < 0) {
-		for(i = 0; i < X86_IA32_NUM_PERF_COUNTERS; i++) {
-			if(!(counter_flag[cpu_id] & 1 << i)) {
-				ret = i;
-				break;
-			}
-		}
-	// check specified counter is available.
-	} else if (cnt < X86_IA32_NUM_PERF_COUNTERS) {
-		if(counter_flag[cpu_id] ^ 1 << cnt) {
-			ret = cnt;
-		}
-	}
-
-	return ret;
-}
-
 int 
-perf_counter_init(struct perf_event_attr *attr, int cpu_id, int counter)
+perf_counter_init(struct mc_perf_event *event)
 {
 	int ret = 0;
 	enum ihk_perfctr_type type;
+	struct perf_event_attr *attr = &event->attr;
 	int mode = 0x00;
 
 	if(!attr->exclude_kernel) {
@@ -2664,28 +2626,35 @@ perf_counter_init(struct perf_event_attr *attr, int cpu_id, int counter)
 			type = PERFCTR_MAX_TYPE;
 		}
 
-		ret = ihk_mc_perfctr_init(counter, type, mode);
+		event->counter = ihk_mc_perfctr_alloc_counter(); 
+		ret = ihk_mc_perfctr_init(event->counter, type, mode);
 	
 	} else if(attr->type == PERF_TYPE_RAW) {
+		// PAPI_REF_CYC counted by fixed counter
+		if((attr->config & 0x0000ffff) == 0x00000300) {
+			event->counter = 2 + X86_IA32_BASE_FIXED_PERF_COUNTERS;
+			ret = ihk_mc_perfctr_fixed_init(event->counter, mode);
+			return ret;
+		}
+
 		// apply MODE to config(event_code)
 		attr->config &= ~(3 << 16);
-		if(mode &= PERFCTR_USER_MODE) {
+		if(mode & PERFCTR_USER_MODE) {
 			attr->config |= 1 << 16;
 		}
-		if(mode &= PERFCTR_KERNEL_MODE) {
+		if(mode & PERFCTR_KERNEL_MODE) {
 			attr->config |= 1 << 17;
 		}
 
-		ret = ihk_mc_perfctr_init_raw(counter, attr->config, mode);
+		event->counter = ihk_mc_perfctr_alloc_counter(); 
+		ret = ihk_mc_perfctr_init_raw(event->counter, attr->config, mode);
 	} else {
 		// Not supported type.
 		ret = -1;
 	}
 
 	if(ret >= 0) {
-	        ret = ihk_mc_perfctr_reset(counter);
+	        ret = ihk_mc_perfctr_reset(event->counter);
-       		ret = ihk_mc_perfctr_start(1 << counter);
-		counter_flag[cpu_id] |= 1 << counter;
 	}
 
 	return ret;
@@ -2770,6 +2739,28 @@ perf_read(struct mckfd *sfd, ihk_mc_user_context_t *ctx)
 
 }
 
+static void 
+perf_start(struct mc_perf_event *event)
+{
+	int counter = event->counter;
+
+	if((1UL << counter & X86_IA32_PERF_COUNTERS_MASK) | 
+	(1UL << counter & X86_IA32_FIXED_PERF_COUNTERS_MASK)) {
+		ihk_mc_perfctr_start(1UL << counter);
+	}
+}
+
+static void
+perf_stop(struct mc_perf_event *event)
+{
+	int counter = event->counter;
+
+	if((1UL << counter & X86_IA32_PERF_COUNTERS_MASK) | 
+	(1UL << counter & X86_IA32_FIXED_PERF_COUNTERS_MASK)) {
+		ihk_mc_perfctr_stop(1UL << counter);
+	}
+}
+
 static int
 perf_ioctl(struct mckfd *sfd, ihk_mc_user_context_t *ctx)
 {
@@ -2779,16 +2770,16 @@ perf_ioctl(struct mckfd *sfd, ihk_mc_user_context_t *ctx)
 
 	switch (cmd) {
         case PERF_EVENT_IOC_ENABLE:
-		ihk_mc_perfctr_start(1 << counter);
+		perf_start(event);
                 break;
         case PERF_EVENT_IOC_DISABLE:
-		ihk_mc_perfctr_stop(1 << counter);
+		perf_stop(event);
                 break;
         case PERF_EVENT_IOC_RESET:
-		ihk_mc_perfctr_reset(counter);
+		ihk_mc_perfctr_set(counter, event->sample_freq * -1);
                 break;
         case PERF_EVENT_IOC_REFRESH:
-		ihk_mc_perfctr_reset(counter);
+		ihk_mc_perfctr_set(counter, event->sample_freq * -1);
 		break;
 	default :
 		return -1;
@@ -2801,10 +2792,9 @@ static int
 perf_close(struct mckfd *sfd, ihk_mc_user_context_t *ctx)
 {
 	struct mc_perf_event *event = (struct mc_perf_event*)sfd->data;
-	int cpu_id = event->cpu_id;
 
+	ihk_mc_perfctr_release_counter(event->counter);
 	kfree(event);
-	release_counter(cpu_id, event->counter);
 
 	return 0;
 }
@@ -2827,11 +2817,12 @@ perf_mmap(struct mckfd *sfd, ihk_mc_user_context_t *ctx)
 
 	// setup perf_event_mmap_page
 	page = (struct perf_event_mmap_page *)rc;
-	page->cap_usr_rdpmc = 0;
+	page->cap_user_rdpmc = 1;
 
 	return rc;
 }
 
+extern unsigned long ihk_mc_perfctr_get_info();
 SYSCALL_DECLARE(perf_event_open)
 {
 	struct syscall_request request IHK_DMA_ALIGN;
@@ -2872,15 +2863,15 @@ SYSCALL_DECLARE(perf_event_open)
 	if(not_supported_flag) {
 		return -1;
 	}
-	
+
 	// process of perf_event_open
 	event = kmalloc(sizeof(struct mc_perf_event), IHK_MC_AP_NOWAIT);
+	if(!event)
+		return -ENOMEM;
 	event->cpu_id = thread->cpu_id;
 	event->attr = (struct perf_event_attr)*attr;
-	event->counter = get_avail_counter(event->cpu_id, -1); 
+
-	if(event->counter < 0) {
+	event->sample_freq = attr->sample_freq;
-		return -1;
-	}
 	event->nr_siblings = 0;
 	INIT_LIST_HEAD(&event->group_entry);
 	INIT_LIST_HEAD(&event->sibling_list);
@@ -2897,7 +2888,10 @@ SYSCALL_DECLARE(perf_event_open)
 		}
 	}
 
-	perf_counter_init(attr, event->cpu_id, event->counter);
+	if(perf_counter_init(event) < 0)
+		return -1;
+	if(event->counter < 0)
+		return -1;
 
 	request.number = __NR_perf_event_open;
 	request.args[0] = 0;

lib/include/ihk/perfctr.h

@@ -52,9 +52,14 @@ int ihk_mc_perfctr_init(int counter, enum ihk_perfctr_type type, int mode);
 int ihk_mc_perfctr_init_raw(int counter, unsigned int code, int mode);
 int ihk_mc_perfctr_start(unsigned long counter_mask);
 int ihk_mc_perfctr_stop(unsigned long counter_mask);
+int ihk_mc_perfctr_fixed_init(int counter, int mode);
 int ihk_mc_perfctr_reset(int counter);
+int ihk_mc_perfctr_set(int counter, unsigned long value);
 int ihk_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value);
 unsigned long ihk_mc_perfctr_read(int counter);
+unsigned long ihk_mc_perfctr_read_msr(int counter);
+int ihk_mc_perfctr_alloc_counter();
+void ihk_mc_perfctr_release_counter(int counter);
 
 #endif
 

lib/include/mc_perf_event.h

@@ -1,6 +1,8 @@
 #ifndef MC_PERF_EVNET_H
 #define MC_PERF_EVENT_H
 
+#include <march.h>
+
 struct perf_event_attr;
 
 /**
@@ -217,6 +219,8 @@ struct mc_perf_event {
 	struct perf_event_attr		attr;
 	int 				cpu_id;
 	int 				counter;
+	unsigned long			sample_freq;
+
 	struct mc_perf_event 		*group_leader;
 	struct list_head		sibling_list;
 	int 				nr_siblings;
@@ -224,25 +228,32 @@ struct mc_perf_event {
 };
 
 struct perf_event_mmap_page {
-	unsigned int version;
+	unsigned int version;		// version number of this structure
-	unsigned int compat_version;
+	unsigned int compat_version;	// lowest version this is compat with
-	unsigned int lock;
+	unsigned int lock;		// seqlock for synchronization
-	unsigned int index;
+	unsigned int index;		// hardware event identifier
-	long	offset;
+	long	offset;			// add to hardware event value
-	unsigned long time_enabled;
+	unsigned long time_enabled;	// time evet active
-	unsigned long time_running;
+	unsigned long time_running;	// time event on cpu
 	union {
 		unsigned long   capabilities;
-		unsigned long   cap_usr_time            : 1, 
+		struct {
-		                cap_usr_rdpmc           : 1, 
+			unsigned long   cap_bit0          : 1, // Always 0, deprecated
-				cap_____res             : 62;
+		                cap_bit0_is_deprecated    : 1, // Always 1, signals that bit 0 is zero
+				cap_user_rdpmc            : 1, // The RDPMC instruction can be used to read counts
+				cap_user_time             : 1, // The time_* fields are used
+				cap_user_time_zero        : 1, // The time_zero field is used
+				cap_____res               : 59;
+		};
 	};
 	unsigned short pmc_width;
 	unsigned short time_shift;
 	unsigned int time_mult;
 	unsigned long time_offset;
+	unsigned long time_zero;
+	unsigned int size;
 
-	unsigned long __reserved[120];
+	unsigned char __reserved[118*8+4];
 	unsigned long data_head;
 	unsigned long data_tail;
 };