- mcstat is a tool to report McKernel statistics from Linux side.

This is a response to a CEA's request.
	- The tools directory is created under the mckernel directory.
	- Some include files are now installed in the install directory,
	  but we should rethink of it.

Makefile.in

@@ -60,6 +60,7 @@ install:
 		exit 1 \
 		;; \
 	esac
+	@(cd tools/mcstat/; $(MAKE) install)
 
 clean:
 	@(cd executer/kernel/mcctrl; $(MAKE) clean)

configure

@@ -5060,7 +5060,7 @@ ac_config_headers="$ac_config_headers config.h"
 
 # POSTK_DEBUG_ARCH_DEP_37
 # AC_CONFIG_FILES arch dependfiles separate
-ac_config_files="$ac_config_files Makefile executer/user/Makefile executer/user/mcexec.1:executer/user/mcexec.1in executer/user/vmcore2mckdump executer/user/arch/$ARCH/Makefile executer/user/arch/x86_64/Makefile executer/kernel/mcctrl/Makefile executer/kernel/mcctrl/arch/$ARCH/Makefile executer/kernel/mcoverlayfs/Makefile executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile executer/kernel/mcoverlayfs/linux-4.0.9/Makefile executer/kernel/mcoverlayfs/linux-4.6.7/Makefile executer/include/qlmpilib.h kernel/Makefile kernel/Makefile.build kernel/include/swapfmt.h arch/x86_64/tools/mcreboot-attached-mic.sh arch/x86_64/tools/mcshutdown-attached-mic.sh arch/x86_64/tools/mcreboot-builtin-x86.sh arch/x86_64/tools/mcreboot-smp-x86.sh arch/x86_64/tools/mcstop+release-smp-x86.sh arch/x86_64/tools/mcoverlay-destroy-smp-x86.sh arch/x86_64/tools/mcoverlay-create-smp-x86.sh arch/x86_64/tools/eclair-dump-backtrace.exp arch/x86_64/tools/mcshutdown-builtin-x86.sh arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in arch/x86_64/tools/irqbalance_mck.service arch/x86_64/tools/irqbalance_mck.in"
+ac_config_files="$ac_config_files Makefile executer/user/Makefile executer/user/mcexec.1:executer/user/mcexec.1in executer/user/vmcore2mckdump executer/user/arch/$ARCH/Makefile executer/user/arch/x86_64/Makefile executer/kernel/mcctrl/Makefile executer/kernel/mcctrl/arch/$ARCH/Makefile executer/kernel/mcoverlayfs/Makefile executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile executer/kernel/mcoverlayfs/linux-4.0.9/Makefile executer/kernel/mcoverlayfs/linux-4.6.7/Makefile executer/include/qlmpilib.h kernel/Makefile kernel/Makefile.build kernel/include/swapfmt.h arch/x86_64/tools/mcreboot-attached-mic.sh arch/x86_64/tools/mcshutdown-attached-mic.sh arch/x86_64/tools/mcreboot-builtin-x86.sh arch/x86_64/tools/mcreboot-smp-x86.sh arch/x86_64/tools/mcstop+release-smp-x86.sh arch/x86_64/tools/mcoverlay-destroy-smp-x86.sh arch/x86_64/tools/mcoverlay-create-smp-x86.sh arch/x86_64/tools/eclair-dump-backtrace.exp arch/x86_64/tools/mcshutdown-builtin-x86.sh arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in arch/x86_64/tools/irqbalance_mck.service arch/x86_64/tools/irqbalance_mck.in tools/mcstat/Makefile"
 
 
 if test "$TARGET" = "smp-x86"; then
@@ -5797,6 +5797,7 @@ do
     "arch/x86_64/tools/mcreboot.1") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in" ;;
     "arch/x86_64/tools/irqbalance_mck.service") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/irqbalance_mck.service" ;;
     "arch/x86_64/tools/irqbalance_mck.in") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/irqbalance_mck.in" ;;
+    "tools/mcstat/Makefile") CONFIG_FILES="$CONFIG_FILES tools/mcstat/Makefile" ;;
     "arch/x86_64/kernel/Makefile.arch") CONFIG_FILES="$CONFIG_FILES arch/x86_64/kernel/Makefile.arch" ;;
     "kernel/config/config.smp-arm64") CONFIG_FILES="$CONFIG_FILES kernel/config/config.smp-arm64" ;;
     "arch/arm64/kernel/vdso/Makefile") CONFIG_FILES="$CONFIG_FILES arch/arm64/kernel/vdso/Makefile" ;;

configure.ac

@@ -572,6 +572,7 @@ AC_CONFIG_FILES([
 	arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in
 	arch/x86_64/tools/irqbalance_mck.service
 	arch/x86_64/tools/irqbalance_mck.in
+	tools/mcstat/Makefile
 ])
 
 if test "$TARGET" = "smp-x86"; then

tools/mcstat/Makefile.in

@@ -0,0 +1,30 @@
+# Makefile.in COPYRIGHT FUJITSU LIMITED 2015-2016
+CC=@CC@
+MCC=mpicc
+BINDIR=@BINDIR@
+SBINDIR=@SBINDIR@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+LIBDIR=@libdir@
+MANDIR=@MANDIR@
+MCKERNEL_INCDIR=@MCKERNEL_INCDIR@
+KDIR ?= @KDIR@
+ARCH=@ARCH@
+CFLAGS=-Wall -O -I. -I${MCKERNEL_INCDIR}
+LDFLAGS=@LDFLAGS@
+VPATH=@abs_srcdir@
+TARGET=mcstat
+
+all: $(TARGET)
+
+mcexec: mcstat.c libmcexec.a
+	$(CC) -I${KDIR} $(CFLAGS) $(EXTRA_CFLAGS) $(LDFLAGS) -o $@ $^
+
+clean::
+	$(RM) $(TARGET) *.o
+
+.PHONY: all clean install
+
+install::
+	install -m 755 mcstat $(BINDIR)
+

tools/mcstat/mcstat.1in

@@ -0,0 +1,71 @@
+.\" Man page for mcstat
+.\"
+
+.TH MCSTAT 1 "@MCKERNEL_RELEASE_DATE@" "Version @MCKERNEL_VERSION@" MCKERNEL @MCKERNEL_VERSION@"
+.SH NAME
+mcstat \- report McKernel statistics and status
+.\"
+
+.\" ----------------------------  SYNOPSIS ----------------------------
+.SH SYNOPSIS
+.B mcstat \fR [\fIoptions\fR] \fR [\fI<delay>\fR \fI<count>\fR ]
+
+.\" ----------------------------  DESCRIPTION ----------------------------
+.SH DESCRIPTION
+mcstat reports  information about memory, elapsed time of system and users,
+and the number of threads in the McKernel. It also reports McKernel OS status.
+
+An example is shown below.
+
+------- memory (MB) ------- ------- tsc ------ --- thread ---
+    total  current      max    system     user current max
+    2.758    0.461    0.652      2340        3       0   1
+
+It reports that 2.758 MB physical memory has been allocated for
+McKernel, but only 0.461 MB memory is currently used and the maximum
+memory usage is 0.652 MB since rebooted. Elpased times (in 10 msec?)
+in system and user spaces are 2340 and 3, respectively.  No threads
+are currently running, but one thread has run in the past.
+
+.\" ----------------------------  DESCRIPTION ----------------------------
+.SH OPTIONS
+
+.TP
+.I	delay
+The delay between updates in seconds.  If no delay is specified, only
+one report is printed.
+
+.TP
+.I	count
+Number of updates.  In absence of count, when delay is defined,
+default is infinite.
+
+.\" .TP
+.\" .B	-c
+
+.TP
+.B	-n
+Display the header only once. In default, the header is displayed
+every 10 reports.
+
+.TP
+.B	-s
+Reports only McKernel OS status. Without this option, mcstat reports only McKernel statistics.
+
+.TP
+.B	-h
+Display help and exit.
+
+.PP
+.\" ----------------------------  ENVIRONMENT ----------------------------
+.SH ENVIRONMENT
+
+.PP
+.\" ----------------------------  SEE ALSO ----------------------------
+.SH SEE ALSO
+
+.\" ----------------------------  AUTHORS ----------------------------
+.SH AUTHORS
+Copyright (C) 2017 McKernel Development Team, RIKEN AICS, Japan
+
+

tools/mcstat/mcstat.c

@@ -0,0 +1,274 @@
+/*
+ * mcstat -- reports McKernel statistis
+ *	mcstat [-h] 
+ *	mcstat [-n]  [delay [ count]]
+ *	mcstat [-s] 
+ *	mcstat [-c] 
+ */
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <getopt.h>
+
+#include <ihk/ihk_host_user.h>
+#include <ihk/ihklib_private.h>		// mcctrl_ioctl_getrusage_desc is defined here
+#undef IHK_MAX_NUM_NUMA_NODES
+#include <mckernel/ihklib_rusage.h>	// mckernel_rusage is defined here
+
+#define MAX_CPUS	256
+#define MiB100		(100*1024*1024) // 100 MiB
+#define MiB		(1024*1024)
+#define GiB		(1024*1024*1024)
+#define CONV_UNIT(d)	(((float)(d))/scale)
+#define UPDATE_COUNTER(c)	(c = (c + 1) % 10)
+
+struct mckernel_rusage			rbuf;
+
+static void	mcstatistics(int idx, int once, int delay, int count);
+static void	mcstatus(int idx, int delay, int count);
+static void	mcosusage(int idx, int once, int delay, int count);
+
+static void
+usage()
+{
+    fprintf(stderr, "Usage: mcstat [-h|-n|-s] [delay [count]]\n");
+}
+
+int
+main(int argc, char **argv)
+{
+    int		opt;
+    int		idx = 0;	/* index of OS instance */
+    int		sflag = 0;	/* statistic option */
+    int		cflag = 0;	/* cpu info */
+    int		once = 0;	/* header is shown once */
+    int		delay = 0;	/* delay in second */
+    int		count = 1;	/* */
+
+    if (argc > 1) {
+        while ((opt = getopt(argc, argv, "chns")) != -1) {
+            switch (opt) {
+	    case 'c':	/* cpu info */
+		cflag = 1; break;
+	    case 'h':
+		usage(); exit(0);
+	    case 'n':
+		once = 1; break;
+	    case 's':	/* status */
+		sflag = 1; break;
+	    }
+	}
+    }
+    if (optind < argc) { /* interval */
+	delay = atoi(argv[optind]);
+	if (optind + 1 < argc) { /* count */
+	    count = atoi(argv[optind+1]);
+	} else {
+	    count = -1; /* inifi */
+	}
+    }
+
+    if (sflag) {
+	mcstatus(idx, delay, count);
+    } else if (cflag) {
+	mcosusage(idx, once, delay, count);
+    } else {
+	mcstatistics(idx, once, delay, count);
+    }
+    return 0;
+}
+
+static int
+devopen(int idx)
+{
+    int		fd;
+    char	fn[128];
+
+    snprintf(fn, 128, "/dev/mcos%d", idx);
+    fd = open(fn, O_RDONLY);
+    return fd;
+}
+
+static void
+statistics_header(char *unit)
+{
+    printf("------- memory (%s) ------- ------- tsc ------ --- thread ---\n",
+	   unit);
+    printf("    total  current      max    system     user current max\n");
+}
+
+/*
+ * Device should be open in each ioctl time. Otherwise, this command grabs
+ * the device, and cannot be rebooted by others.
+ */
+static int
+mygetrusage(int idx, struct mckernel_rusage *rbp)
+{
+    int		fd, rc;
+    struct mcctrl_ioctl_getrusage_desc	rusage;
+
+    if ((fd = devopen(idx)) < 0) {
+	return -1;
+    }
+    rusage.rusage = rbp;
+    rusage.size_rusage = sizeof(struct mckernel_rusage);
+    memset(rbp, 0, sizeof(struct mckernel_rusage));
+    if ((rc = ioctl(fd, IHK_OS_GETRUSAGE, &rusage)) < 0) {
+	perror("ioctl"); exit(-1);
+    }
+    close(fd);
+    return 0;
+}
+
+static void
+mcstatistics(int idx, int once, int delay, int count)
+{
+    int		i, scale;
+    char	*unit;
+    unsigned char show = 0;
+
+    if (mygetrusage(idx, &rbuf) < 0) {
+	printf("Device has not been created.\n");
+	exit(-1);
+    }
+    if (rbuf.memory_max_usage < MiB100) {
+	scale = MiB; unit = "MB";
+    } else {
+	scale = GiB; unit = "GB";
+    }
+    statistics_header(unit);
+    for (;;) {
+	printf("%9.3f%9.3f%9.3f %9ld%9ld %7d %3d\n",
+	       CONV_UNIT(rbuf.memory_max_usage),
+	       CONV_UNIT(rbuf.memory_kmem_usage),
+	       CONV_UNIT(rbuf.memory_kmem_max_usage),
+	       rbuf.cpuacct_stat_system, rbuf.cpuacct_stat_user,
+	       rbuf.num_threads, rbuf.max_num_threads);
+	if (count > 0 && --count == 0) break;
+	sleep(delay);
+	if (mygetrusage(idx, &rbuf) < 0) {
+	    printf("Device is now invisible.\n");
+	    break;
+	}
+	if (!once) {
+	    if (UPDATE_COUNTER(show) == 0) {
+		statistics_header(unit);
+	    }
+	}
+    }
+/*
+	?? /1000000
+  	rusage->cpuacct_stat_system = st / 10000000;
+	rusage->cpuacct_stat_user = ut / 10000000;
+	rusage->cpuacct_usage = ut;
+	printf("cpuacct_usage = %x\n", rbuf.cpuacct_usage);
+*/
+    for (i = 0; i < rbuf.max_num_threads; i++) {
+	printf("cpuacct_usage_percpu[%d] = %ld\n", i, rbuf.cpuacct_usage_percpu[i]);
+    }
+}
+
+/* ihk_os_status enum is defined in ihk/linux/include/ihk/status.h */
+static char *charstat[] = {
+    "None",	/*IHK_OS_STATUS_NOT_BOOTED*/
+    "Booting",	/*IHK_OS_STATUS_BOOTING*/
+    "Booted",	/*IHK_OS_STATUS_BOOTED, OS booted and acked */
+    "Ready",	/*IHK_OS_STATUS_READY, OS is ready and fully functional */
+    "Freezing",	/*IHK_OS_STATUS_FREEZING, OS is freezing */
+    "Frozen",	/*IHK_OS_STATUS_FROZEN,   OS is frozen */
+    "Shutdown",	/* IHK_OS_STATUS_SHUTDOWN, OS is shutting down */
+    "Stopped",	/* IHK_OS_STATUS_STOPPED, OS stopped successfully */
+    "Panic",	/* IHK_OS_STATUS_FAILED, OS panics or failed to boot */
+    "Hangup",	/* IHK_OS_STATUS_HUNGUP,  OS is hungup */
+};
+
+static void
+mcstatus(int idx, int delay, int count)
+{
+    int		fd, rc;
+
+    for(;;) {
+	if ((fd = devopen(idx)) < 0) {
+	    printf("Devide is not created\n");
+	} else {
+	    rc = ioctl(fd, IHK_OS_STATUS, 0);
+	    close(fd);
+	    printf("McKernel status: ");
+	    if (rc >= IHK_OS_STATUS_NOT_BOOTED && rc <= IHK_OS_STATUS_HUNGUP) {
+		printf("%s\n", charstat[rc]);
+	    } else {
+		printf("ioctl error(IHK_OS_STATUS)\n");
+	    }
+	}
+	if (count > 0 && --count == 0) break;
+	sleep(delay);
+    }
+}
+
+/* status is not contiguous numbers */
+static char *
+monstatus(int status)
+{
+    switch (status) {
+    case IHK_OS_MONITOR_NOT_BOOT:	return "boot";
+    case IHK_OS_MONITOR_IDLE:		return "idle";
+    case IHK_OS_MONITOR_USER:		return "user mode";
+    case IHK_OS_MONITOR_KERNEL:		return "kernel mode";
+    case IHK_OS_MONITOR_KERNEL_HEAVY:   return "kernel mode";
+    case IHK_OS_MONITOR_KERNEL_OFFLOAD:	return "offload";
+    case IHK_OS_MONITOR_KERNEL_FREEZING:return "freezing";
+    case IHK_OS_MONITOR_KERNEL_FROZEN:	return "frozen";
+    case IHK_OS_MONITOR_KERNEL_THAW:	return "thaw";
+    case IHK_OS_MONITOR_PANIC:	return "panic";
+    }
+    return "";
+}
+
+static void
+osusage_header()
+{
+    printf("--cpu-- --status-- --count--\n");
+}
+
+static void
+mcosusage(int idx, int once, int delay, int count)
+{
+    int		fd, i, rc;
+    int		ncpus;
+    unsigned char show = 0;
+    struct ihk_os_cpu_monitor	mon[MAX_CPUS];
+
+    if (mygetrusage(idx, &rbuf) < 0) {
+	printf("Device has not been created.\n");
+    }
+    ncpus = rbuf.max_num_threads;
+    osusage_header();
+    for(;;) {
+	if ((fd = devopen(idx)) < 0) {
+	    printf("Devide is not created\n");
+	} else {
+	    rc = ioctl(fd, IHK_OS_GET_CPU_USAGE, &mon);
+	    close(fd);
+	    if (rc != 0) {
+		printf("ioctl error(IHK_OS_GET_CPU_USAGE)\n");
+		break;
+	    }
+	    for (i = 0; i < ncpus; i++) {
+		printf("%6d: %10s %9ld\n",
+		       i, monstatus(mon[i].status), mon[i].counter);
+	    }
+	}
+	if (count > 0 && --count == 0) break;
+	sleep(delay);
+	if (!once) {
+	    if (UPDATE_COUNTER(show) == 0) {
+		osusage_header();
+	    }
+	}
+    }
+}

tools/mcstat/runtest.c

@@ -0,0 +1,32 @@
+#include <sys/types.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#define SIZE 10000
+double	a[SIZE], b[SIZE], c[SIZE];
+
+int
+main()
+{
+    int		i, j, pid;
+
+    printf("invoked\n");
+    for (i = 0; i < 3; i++) {
+	sleep(1);
+	printf("wakeup %d\n", i);
+    }
+    printf("getpid 1000 times\n");
+    for (i = 0; i < 1000; i++) {
+	pid = getpid();
+    }
+    for (i = 0; i < SIZE; i++) {
+	a[i] = 0; b[i] = 1.0; c[i] = 3.0;
+    }
+    for (j = 0; j < 1000; j++) {
+	for (i = 0; i < SIZE; i++) {
+	    a[i] = b[i] / c[i];
+	}
+    }
+    printf("done\n");
+    return 0;
+}