perf: update event structure

Change-Id: I5bc0fdd42db509b5d2daca7d97e29ad1f7d11f1a

arch/arm64/kernel/include/ihk/atomic.h

@@ -147,6 +147,8 @@ static inline void ihk_atomic64_add(long i, ihk_atomic64_t *v)
 /* @ref.impl arch/arm64/include/asm/atomic.h::atomic64_inc */
 #define ihk_atomic64_inc(v)			ihk_atomic64_add(1LL, (v))
 
+#define ihk_atomic64_cmpxchg(p, o, n) cmpxchg(&((p)->counter64), o, n)
+
 /***********************************************************************
  * others
  */

arch/arm64/kernel/perfctr.c

@@ -272,3 +272,26 @@ int ihk_mc_event_set_period(struct mc_perf_event *event)
 
 	return ret;
 }
+
+uint64_t ihk_mc_event_update(struct mc_perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	int64_t delta;
+	uint64_t prev_raw_count, new_raw_count;
+	uint64_t max_period = arm_pmu_event_max_period(event);
+
+again:
+	prev_raw_count = ihk_atomic64_read(&hwc->prev_count);
+	new_raw_count = cpu_pmu.read_counter(event->counter_id);
+
+	if (ihk_atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
+			     new_raw_count) != prev_raw_count)
+		goto again;
+
+	delta = (new_raw_count - prev_raw_count) & max_period;
+
+	ihk_atomic64_add(delta, &event->count);
+	ihk_atomic64_add(-delta, &hwc->period_left);
+
+	return new_raw_count;
+}

arch/arm64/kernel/perfctr_armv8pmu.c

@@ -824,6 +824,7 @@ static void armv8pmu_handle_irq(void *priv)
 	}
 
 	if (event) {
+		ihk_mc_event_update(event);
 		ihk_mc_event_set_period(event);
 	}
 	return;

arch/x86_64/kernel/perfctr.c

@@ -513,3 +513,8 @@ int ihk_mc_event_set_period(struct mc_perf_event *event)
 {
 	return 0;
 }
+
+uint64_t ihk_mc_event_update(struct mc_perf_event *event)
+{
+	return 0;
+}

kernel/include/process.h

@@ -725,6 +725,7 @@ struct thread {
 	struct futex_q futex_q;
 
 	// for performance counter
+#define PMC_ALLOC_MAP_BITS BITS_PER_LONG
 	unsigned long pmc_alloc_map;
 	unsigned long extra_reg_alloc_map;
 

kernel/syscall.c

@@ -3914,7 +3914,6 @@ unsigned long perf_event_read_value(struct mc_perf_event *event)
 {
 	unsigned long rtn_count = 0;
 	unsigned long pmc_count = 0;
-	int counter_id = event->counter_id;
 	struct thread *thread = cpu_local_var(current);
 	unsigned long cur_user_tsc, cur_system_tsc;
 
@@ -3954,8 +3953,7 @@ unsigned long perf_event_read_value(struct mc_perf_event *event)
 			}
 		}
 		else {
-			pmc_count = ihk_mc_perfctr_read(counter_id) +
+			ihk_mc_event_update(event);
-				event->attr.sample_freq;
 		}
 	}
 
@@ -4186,6 +4184,10 @@ perf_stop(struct mc_perf_event *event)
 	struct mc_perf_event *leader = event->group_leader, *sub;
 	struct thread *thread = cpu_local_var(current);
 
+	struct mc_perf_event *stop_event[PMC_ALLOC_MAP_BITS + 1];
+	int stop_event_idx = 0;
+
+	stop_event[0] = NULL;
 	counter_id = leader->counter_id;
 	if (ihk_mc_perf_counter_mask_check(1UL << counter_id) &&
 			leader->state == PERF_EVENT_STATE_ACTIVE) {
@@ -4199,6 +4201,8 @@ perf_stop(struct mc_perf_event *event)
 		}
 		else {
 			counter_mask |= 1UL << counter_id;
+			stop_event[stop_event_idx++] = leader;
+			stop_event[stop_event_idx] = NULL;
 		}
 
 		leader->state = PERF_EVENT_STATE_INACTIVE;
@@ -4220,6 +4224,8 @@ perf_stop(struct mc_perf_event *event)
 			}
 			else {
 				counter_mask |= 1UL << counter_id;
+				stop_event[stop_event_idx++] = sub;
+				stop_event[stop_event_idx] = NULL;
 			}
 			sub->state = PERF_EVENT_STATE_INACTIVE;
 		}
@@ -4227,6 +4233,10 @@ perf_stop(struct mc_perf_event *event)
 
 	if (counter_mask) {
 		ihk_mc_perfctr_stop(counter_mask, 0);
+		stop_event_idx = 0;
+		while (stop_event[stop_event_idx]) {
+			ihk_mc_event_update(stop_event[stop_event_idx++]);
+		}
 	}
 	cpu_local_var(current)->proc->monitoring_event = NULL;
 	cpu_local_var(current)->proc->perf_status = PP_NONE;

lib/include/ihk/cpu.h

@@ -92,6 +92,7 @@ unsigned long ihk_mc_raw_event_map(unsigned long raw_event);
 int ihk_mc_validate_event(unsigned long hw_config);
 int hw_perf_event_init(struct mc_perf_event *event);
 int ihk_mc_event_set_period(struct mc_perf_event *event);
+uint64_t ihk_mc_event_update(struct mc_perf_event *event);
 
 static inline int is_sampling_event(struct mc_perf_event *event)
 {