prerelease: 0.93: investigate smp_ihk_os_panic_notifier

Change-Id: I997b41f80038603261de2e8232b6b8ca200cd8cd

CMakeLists.txt

@@ -10,7 +10,7 @@ project(mckernel C ASM)
 set(MCKERNEL_VERSION "1.7.1")
 
 # See "Fedora Packaging Guidelines -- Versioning"
-set(MCKERNEL_RELEASE "0.7")
+set(MCKERNEL_RELEASE "0.93")
 
 set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
 # for rpmbuild
@@ -65,7 +65,7 @@ if(ENABLE_TOFU)
 endif()
 
 # when compiling on a compute-node
-execute_process(COMMAND bash -c "grep $(hostname) /etc/opt/FJSVfefs/config/fefs_node1.csv 2>/dev/null | cut -d, -f2"
+execute_process(COMMAND bash -c "grep $(hostname) /etc/opt/FJSVfefs/config/fefs_node1.csv 2>/dev/null | cut -d, -f2 | grep -o CN"
   OUTPUT_VARIABLE FUGAKU_NODE_TYPE OUTPUT_STRIP_TRAILING_WHITESPACE)
 if(FUGAKU_NODE_TYPE STREQUAL "CN")
 	option(ENABLE_FUGAKU_HACKS "Fugaku hacks" ON)

docs/install.rst

@@ -129,11 +129,29 @@ Create the tarball and the spec file:
    make dist
    cp mckernel-<version>.tar.gz <rpmbuild>/SOURCES
 
+(optional) Edit the following line in ``scripts/mckernel.spec`` to change
+cmake options. For example:
+
+::
+
+   %cmake -DCMAKE_BUILD_TYPE=Release \
+	-DUNAME_R=%{kernel_version} \
+	-DKERNEL_DIR=%{kernel_dir} \
+	%{?cmake_libdir:-DCMAKE_INSTALL_LIBDIR=%{cmake_libdir}} \
+	%{?build_target:-DBUILD_TARGET=%{build_target}} \
+	%{?toolchain_file:-DCMAKE_TOOLCHAIN_FILE=%{toolchain_file}} \
+	-DENABLE_TOFU=ON -DENABLE_FUGAKU_HACKS=ON \
+	-DENABLE_KRM_WORKAROUND=OFF -DWITH_KRM=ON \
+	-DENABLE_FUGAKU_DEBUG=OFF \
+	.
+
 Create the rpm package:
 
 When not cross-compiling:
 """""""""""""""""""""""""
 
+Then build the rpm:
+
 ::
 
    rpmbuild -ba scripts/mckernel.spec

docs/users.rst

@@ -34,6 +34,19 @@ For example, with Fujitsu Technical Computing Suite (TCS), you need to specify `
 
    #PJM -L jobenv=mck1
 
+(Optional, Fujitsu TCS only) Specify boot parameters
+----------------------------------------------------
+
+You can specify the boot parameters by defining environmental variables and pass them to Fujitsu TCS.
+The parameters include the resource reservation settings, resource reservation amount, kernel arguments and routing of message channels between McKernel CPUs and Linux CPUs.
+See `IHK Specifications - ihk_create_os_str() <spec/ihk.html>`__ for the parameter names and allowed values.
+The example of setting the memory amount is shown below.
+
+.. code-block:: none
+
+   export IHK_MEM="7G@4,7G@5,7G@6,7G@7"
+   pjsub -X run.sh
+
 Insert ``mcexec`` into the command line
 ---------------------------------------
 

docs/uti.rst

@@ -4,24 +4,41 @@ Advanced: Enable Utility Thread offloading Interface (UTI)
 UTI enables a runtime such as MPI runtime to spawn utility threads such
 as MPI asynchronous progress threads to Linux cores.
 
-Install capstone
-~~~~~~~~~~~~~~~~~~~~
+Install ``capstone`` and ``capstone-devel``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-When compute nodes don't have access to repositories
-""""""""""""""""""""""""""""""""""""""""""""""""""""
+When compute nodes don't have access to EPEL repository
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""
 
-Install EPEL capstone-devel:
+Install EPEL ``capstone`` and ``capstone-devel``:
 
 ::
 
    sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-   sudo yum install capstone-devel
+   sudo yum install capstone capstone-devel
 
 
-When compute nodes don't have access to repositories
-""""""""""""""""""""""""""""""""""""""""""""""""""""
+When compute nodes don't have access to EPEL repository
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""
 
-Ask the system administrator to install ``capstone-devel``. Note that it is in the EPEL repository.
+A. Ask the system administrator to install ``capstone`` and ``capstone-devel``. Note that it is in the EPEL repository.
+
+B. Download the rpm with the machine in which you are the administrator:
+
+::
+
+   sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+   sudo yum install yum-utils
+   yumdownloader capstone capstone-devel
+
+and then install it to your home directory of the login node:
+
+::
+
+   cd $HOME/$(uname -p)
+   rpm2cpio capstone-4.0.1-9.el8.aarch64.rpm | cpio -idv
+   rpm2cpio capstone-devel-4.0.1-9.el8.aarch64.rpm | cpio -idv
+   sed -i 's#/usr/#'"$HOME"'/'"$(uname -p)"'/usr/#' $HOME/$(uname -p)/usr/lib64/pkgconfig/capstone.pc
 
 
 Install syscall_intercept
@@ -31,7 +48,24 @@ Install syscall_intercept
 
    git clone https://github.com/RIKEN-SysSoft/syscall_intercept.git
    mkdir build && cd build
-   cmake <syscall_intercept>/arch/aarch64 -DCMAKE_INSTALL_PREFIX=<syscall-intercept-install> -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc -DTREAT_WARNINGS_AS_ERRORS=OFF
+
+When ``capstone`` and ``capstone-devel`` are installed into the system directory:
+
+::
+
+   cmake ../syscall_intercept/arch/aarch64 -DCMAKE_INSTALL_PREFIX=${HOME}/$(uname -p)/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc -DTREAT_WARNINGS_AS_ERRORS=OFF
+
+When ``capstone`` and ``capstone-devel`` are installed into your home directory:
+
+::
+
+   CMAKE_PREFIX_PATH=${HOME}/$(uname -p)/usr cmake ../syscall_intercept/arch/aarch64 -DCMAKE_INSTALL_PREFIX=${HOME}/$(uname -p)/usr -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc -DTREAT_WARNINGS_AS_ERRORS=OFF
+
+Install:
+
+::
+
+   make && make install && make test
 
 Install UTI for McKernel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -48,16 +82,17 @@ Install:
 Install McKernel
 ~~~~~~~~~~~~~~~~~~~~
 
-Add ``-DENABLE_UTI=ON`` option to ``cmake``:
+``cmake`` with the additional options:
 
 ::
 
-   CMAKE_PREFIX_PATH=<syscall-intercept-install> cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel -DENABLE_UTI=ON $HOME/src/ihk+mckernel/mckernel
+   CMAKE_PREFIX_PATH=${HOME}/$(uname -p)/usr cmake -DCMAKE_INSTALL_PREFIX=${HOME}/ihk+mckernel -DENABLE_UTI=ON $HOME/src/ihk+mckernel/mckernel
+   make -j install
 
 Run programs
 ~~~~~~~~~~~~~~~~
 
-Add ``--enable-uti`` option to ``mcexec``:
+``mcexec`` with ``--enable-uti`` option:
 
 ::
 

executer/kernel/mcctrl/control.c

@@ -270,16 +270,17 @@ int mcexec_transfer_image(ihk_os_t os, struct remote_transfer *__user upt)
 		return -EFAULT;
 	}
 
+#ifdef CONFIG_MIC
 	if (pt.size > PAGE_SIZE) {
 		printk("mcexec_transfer_image(): ERROR: size exceeds PAGE_SIZE\n");
 		return -EFAULT;
 	}
-	
+
 	phys = ihk_device_map_memory(ihk_os_to_dev(os), pt.rphys, PAGE_SIZE);
-#ifdef CONFIG_MIC
 	rpm = ioremap_wc(phys, PAGE_SIZE);
 #else
-	rpm = ihk_device_map_virtual(ihk_os_to_dev(os), phys, PAGE_SIZE, NULL, 0);
+	phys = ihk_device_map_memory(ihk_os_to_dev(os), pt.rphys, pt.size);
+	rpm = ihk_device_map_virtual(ihk_os_to_dev(os), phys, pt.size, NULL, 0);
 #endif
 
 	if (!rpm) {
@@ -304,10 +305,11 @@ int mcexec_transfer_image(ihk_os_t os, struct remote_transfer *__user upt)
 
 #ifdef CONFIG_MIC
 	iounmap(rpm);
+	ihk_device_unmap_memory(ihk_os_to_dev(os), phys, PAGE_SIZE);
 #else
-	ihk_device_unmap_virtual(ihk_os_to_dev(os), rpm, PAGE_SIZE);
+	ihk_device_unmap_virtual(ihk_os_to_dev(os), rpm, pt.size);
+	ihk_device_unmap_memory(ihk_os_to_dev(os), phys, pt.size);
 #endif
-	ihk_device_unmap_memory(ihk_os_to_dev(os), phys, PAGE_SIZE);	
 
 	return ret;
 
@@ -3644,7 +3646,8 @@ int __mcctrl_os_read_write_cpu_register(ihk_os_t os, int cpu,
 	isp.op = op;
 	isp.pdesc = virt_to_phys(ldesc);
 
-	ret = mcctrl_ikc_send_wait(os, cpu, &isp, 0, NULL, &do_free, 1, ldesc);
+	/* 1 sec timeout for the case where McKernel can't respond */
+	ret = mcctrl_ikc_send_wait(os, cpu, &isp, 1000, NULL, &do_free, 1, ldesc);
 	if (ret != 0) {
 		printk("%s: ERROR sending IKC msg: %d\n", __FUNCTION__, ret);
 		goto out;

executer/kernel/mcctrl/ikc.c

@@ -536,9 +536,6 @@ int prepare_ikc_channels(ihk_os_t os)
 	usrdata->os = os;
 	ihk_host_os_set_usrdata(os, usrdata);
 
-	ihk_ikc_listen_port(os, &lp_ikc2linux);
-	ihk_ikc_listen_port(os, &lp_ikc2mckernel);
-
 	init_waitqueue_head(&usrdata->wq_procfs);
 	mutex_init(&usrdata->reserve_lock);
 	mutex_init(&usrdata->part_exec_lock);
@@ -555,6 +552,16 @@ int prepare_ikc_channels(ihk_os_t os)
 	INIT_LIST_HEAD(&usrdata->wakeup_descs_list);
 	spin_lock_init(&usrdata->wakeup_descs_lock);
 
+	/* ihk_ikc_listen_port should be performed after
+	 * usrdata->cpu_topology_list is initialized because the
+	 * function enables syscall_packet_handler which accesses
+	 * the list (the call path is sysfsm_packet_handler -->
+	 * sysfsm_work_main --> sysfsm_setup --> setup_sysfs_files
+	 * --> setup_cpus_sysfs_files).
+	 */
+	ihk_ikc_listen_port(os, &lp_ikc2linux);
+	ihk_ikc_listen_port(os, &lp_ikc2mckernel);
+
 	return 0;
 
 error:

executer/user/mcexec.c

@@ -1957,14 +1957,14 @@ opendev()
 			fprintf(stderr, "%s: warning: LD_PRELOAD line is too long\n", __FUNCTION__); \
 			return; \
 		} \
-		strncat(envbuf, elembuf, remainder); \
+		strncat(envbuf, elembuf, remainder - 1); \
 		remainder = PATH_MAX - (strlen(envbuf) + 1); \
 		nelem++; \
 	} while (0)
 
 static ssize_t find_libdir(char *libdir, size_t len)
 {
-	FILE *filep;
+	FILE *filep = NULL;
 	ssize_t rc;
 	size_t linelen = 0;
 	char *line = NULL;
@@ -2020,7 +2020,9 @@ static ssize_t find_libdir(char *libdir, size_t len)
 	}
 
 out:
-	pclose(filep);
+	if (filep) {
+		pclose(filep);
+	}
 	free(line);
 	return rc;
 }
@@ -4121,6 +4123,7 @@ int main_loop(struct thread_data_s *my_thread)
 #endif
 
 		case __NR_gettid:{
+			int rc = 0;
 			/*
 			 * Number of TIDs and the remote physical address where TIDs are
 			 * expected are passed in arg 4 and 5, respectively.
@@ -4132,6 +4135,7 @@ int main_loop(struct thread_data_s *my_thread)
 				int *tids = malloc(sizeof(int) * w.sr.args[4]);
 				if (!tids) {
 					fprintf(stderr, "__NR_gettid(): error allocating TIDs\n");
+					rc = -ENOMEM;
 					goto gettid_out;
 				}
 
@@ -4152,13 +4156,14 @@ int main_loop(struct thread_data_s *my_thread)
 				trans.direction = MCEXEC_UP_TRANSFER_TO_REMOTE;
 
 				if (ioctl(fd, MCEXEC_UP_TRANSFER, &trans) != 0) {
+					rc = -EFAULT;
 					fprintf(stderr, "__NR_gettid(): error transfering TIDs\n");
 				}
 
 				free(tids);
 			}
 gettid_out:
-			do_syscall_return(fd, cpu, 0, 0, 0, 0, 0);
+			do_syscall_return(fd, cpu, rc, 0, 0, 0, 0);
 			break;
 		}
 

kernel/include/cls.h

@@ -106,9 +106,7 @@ struct cpu_local_var {
 	ihk_spinlock_t migq_lock;
 	struct list_head migq;
 	int in_interrupt;
-#ifdef ENABLE_FUGAKU_HACKS
 	int in_page_fault;
-#endif
 	int no_preempt;
 	int timer_enabled;
 	unsigned long nr_ctx_switches;

kernel/mem.c

@@ -1395,7 +1395,6 @@ static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
 			__FUNCTION__, fault_addr, reason, regs);
 
 	preempt_disable();
-#ifdef ENABLE_FUGAKU_HACKS
 	++cpu_local_var(in_page_fault);
 	if (cpu_local_var(in_page_fault) > 1) {
 		kprintf("%s: PF in PF??\n", __func__);
@@ -1408,7 +1407,6 @@ static void page_fault_handler(void *fault_addr, uint64_t reason, void *regs)
 			panic("PANIC");
 		}
 	}
-#endif
 
 	cpu_enable_interrupt();
 
@@ -1475,6 +1473,7 @@ out_linux:
 			__func__, thread ? thread->tid : -1, fault_addr,
 			reason, error);
 		unhandled_page_fault(thread, fault_addr, reason, regs);
+		--cpu_local_var(in_page_fault);
 		preempt_enable();
 
 #ifdef ENABLE_FUGAKU_DEBUG
@@ -1511,9 +1510,7 @@ out_linux:
 out_ok:
 #endif
 	error = 0;
-#ifdef ENABLE_FUGAKU_HACKS
 	--cpu_local_var(in_page_fault);
-#endif
 	preempt_enable();
 out:
 	dkprintf("%s: addr: %p, reason: %lx, regs: %p -> error: %d\n",

scripts/mckernel.spec.in

@@ -25,9 +25,17 @@ Source0: mckernel-%{version}.tar.gz
 
 Requires: systemd-libs numactl-libs libdwarf
 
-# kernel_module_package macro does not handle cross build...
+# don't use kernel_module_package so that one rpm including .ko and binaries are created
 %if "%{?_host_cpu}" == "x86_64" && "%{?_target_cpu}" == "aarch64"
 %define cross_compile 1
+%else
+BuildRequires: systemd-devel numactl-devel binutils-devel kernel-devel libdwarf-devel
+# Friendly reminder of the fact that kernel-rpm-macros is no longer included in kernel-devel
+%if 0%{?rhel} >= 8
+BuildRequires: redhat-rpm-config kernel-rpm-macros elfutils-libelf-devel
+%endif
+%endif
+
 %if 0%{?rhel} >= 8
 Requires: kernel >= %{krequires}
 %else
@@ -35,17 +43,6 @@ Requires: kernel = %{krequires}
 %endif
 Requires(post):   /usr/sbin/depmod
 Requires(postun): /usr/sbin/depmod
-%else
-BuildRequires: systemd-devel numactl-devel binutils-devel kernel-devel libdwarf-devel
-# Friendly reminder of the fact that kernel-rpm-macros is no longer included in kernel-devel
-%if 0%{?rhel} >= 8
-BuildRequires: redhat-rpm-config kernel-rpm-macros elfutils-libelf-devel kmod
-%endif
-%if %{defined kernel_module_package_buildreqs}
-BuildRequires: %kernel_module_package_buildreqs
-%kernel_module_package %{?kmod_flavors}
-%endif
-%endif
 
 %description
 Interface for Heterogeneous Kernels and McKernel.
@@ -78,6 +75,9 @@ This package contains headers and libraries required for build apps using IHK/Mc
 	%{?cmake_libdir:-DCMAKE_INSTALL_LIBDIR=%{cmake_libdir}} \
 	%{?build_target:-DBUILD_TARGET=%{build_target}} \
 	%{?toolchain_file:-DCMAKE_TOOLCHAIN_FILE=%{toolchain_file}} \
+	-DENABLE_TOFU=ON -DENABLE_FUGAKU_HACKS=ON \
+	-DENABLE_KRM_WORKAROUND=OFF -DWITH_KRM=ON \
+	-DENABLE_FUGAKU_DEBUG=OFF \
 	.
 %make_build
 
@@ -113,7 +113,6 @@ This package contains headers and libraries required for build apps using IHK/Mc
 %{_mandir}/man1/ihkosctl.1.gz
 %{_mandir}/man1/mcexec.1.gz
 
-%if 0%{?cross_compile}
 /lib/modules/%{kernel_version}/extra/mckernel/ihk.ko
 /lib/modules/%{kernel_version}/extra/mckernel/mcctrl.ko
 %ifarch x86_64
@@ -122,7 +121,6 @@ This package contains headers and libraries required for build apps using IHK/Mc
 %ifarch aarch64
 /lib/modules/%{kernel_version}/extra/mckernel/ihk-smp-arm64.ko
 %endif
-%endif
 
 %files	devel
 %{_includedir}/ihklib.h
@@ -135,8 +133,7 @@ This package contains headers and libraries required for build apps using IHK/Mc
 %{_includedir}/ihk/ihk_host_driver.h
 /lib/modules/%{kernel_version}/extra/mckernel/ihk/linux/core/Module.symvers
 
-%if 0%{?cross_compile}
-# scripts from /usr/lib/rpm/redhat/kmodtool (kernel_module_package) as well
+# taken from /usr/lib/rpm/redhat/kmodtool (kernel_module_package)
 %post
 if [ -e "/boot/System.map-%{kernel_version}" ]; then
     /usr/sbin/depmod -aeF "/boot/System.map-%{kernel_version}" "%{kernel_version}" > /dev/null || :
@@ -162,7 +159,6 @@ if [ -x "/sbin/weak-modules" ]; then
     printf '%s\n' "${modules[@]}" \
     | /sbin/weak-modules --remove-modules
 fi
-%endif
 
 %changelog
 * Tue Feb 12 2019 Dominique Martinet <dominique.martinet@cea.fr> - 1.6.0-0