sched_yield(): avoid schedule for single thread

<build>/ihk/cokernel/Makefile.common is not found when
<build>/mckernel/kernel/Makfile tries to perform
"Make -C <build>/ihk/{cokernel,ikc}" from mckernel/kernel

arch/x86/kernel/cpu.c

@@ -148,7 +148,7 @@ extern char page_fault[], general_protection_exception[];
 extern char debug_exception[], int3_exception[];
 
 uint64_t boot_pat_state = 0;
-int no_turbo = 0; /* May be updated by early parsing of kargs */
+int no_turbo = 1; /* May be updated by early parsing of kargs */
 
 extern int num_processors; /* kernel/ap.c */
 struct pvclock_vsyscall_time_info *pvti = NULL;
@@ -1079,6 +1079,10 @@ unhandled_page_fault(struct thread *thread, void *fault_addr, void *regs)
 
 	kprintf_unlock(irqflags);
 
+	if (!(error & PF_USER)) {
+		panic("panic: kernel mode PF");
+	}
+
 	/* TODO */
 	ihk_mc_debug_show_interrupt_context(regs);
 

arch/x86/kernel/include/syscall_list.h

@@ -150,5 +150,8 @@ SYSCALL_HANDLED(602, pmc_start)
 SYSCALL_HANDLED(603, pmc_stop)
 SYSCALL_HANDLED(604, pmc_reset)
 SYSCALL_HANDLED(700, get_cpu_id)
+#ifdef TRACK_SYSCALLS
+SYSCALL_HANDLED(701, syscall_offload_clr_cntrs)
+#endif // TRACK_SYSCALLS
 
 /**** End of File ****/

arch/x86/kernel/memory.c

@@ -2367,8 +2367,18 @@ int write_process_vm(struct process_vm *vm, void *udst, const void *ksrc, size_t
 			return error;
 		}
 
-		va = phys_to_virt(pa);
-		memcpy(va, from, cpsize);
+		if (pa < ihk_mc_get_memory_address(IHK_MC_GMA_MAP_START, 0) ||
+			pa >= ihk_mc_get_memory_address(IHK_MC_GMA_MAP_END, 0)) {
+			dkprintf("%s: pa is outside of LWK memory, from: %p,"
+				"pa: %p, cpsize: %d\n", __FUNCTION__, from, pa, cpsize);
+			va = ihk_mc_map_virtual(pa, 1, PTATTR_ACTIVE);
+			memcpy(va, from, cpsize);
+			ihk_mc_unmap_virtual(va, 1, 1);
+		}
+		else {
+			va = phys_to_virt(pa);
+			memcpy(va, from, cpsize);
+		}
 
 		from += cpsize;
 		to += cpsize;

arch/x86/kernel/syscall.c

@@ -544,14 +544,14 @@ void ptrace_report_signal(struct thread *thread, int sig)
 	int parent_pid;
 	struct siginfo info;
 
-	dkprintf("ptrace_report_signal,pid=%d\n", thread->proc->pid);
+	dkprintf("ptrace_report_signal, tid=%d, pid=%d\n", thread->tid, thread->proc->pid);
 
 	mcs_rwlock_writer_lock(&proc->update_lock, &lock);	
 	if(!(proc->ptrace & PT_TRACED)){
 		mcs_rwlock_writer_unlock(&proc->update_lock, &lock);
 		return;
 	}
-	proc->exit_status = sig;
+	thread->exit_status = sig;
 	/* Transition thread state */
 	proc->status = PS_TRACED;
 	thread->status = PS_TRACED;
@@ -569,8 +569,8 @@ void ptrace_report_signal(struct thread *thread, int sig)
 	memset(&info, '\0', sizeof info);
 	info.si_signo = SIGCHLD;
 	info.si_code = CLD_TRAPPED;
-	info._sifields._sigchld.si_pid = thread->proc->pid;
-	info._sifields._sigchld.si_status = thread->proc->exit_status;
+	info._sifields._sigchld.si_pid = thread->tid;
+	info._sifields._sigchld.si_status = thread->exit_status;
 	do_kill(cpu_local_var(current), parent_pid, -1, SIGCHLD, &info, 0);
 	/* Wake parent (if sleeping in wait4()) */
 	waitq_wakeup(&proc->parent->waitpid_q);
@@ -695,10 +695,10 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 	int	orgsig;
 	int	ptraceflag = 0;
 	struct mcs_rwlock_node_irqsave lock;
-	unsigned long irqstate;
+	struct mcs_rwlock_node_irqsave mcs_rw_node;
 
 	for(w = pending->sigmask.__val[0], sig = 0; w; sig++, w >>= 1);
-	dkprintf("do_signal,pid=%d,sig=%d\n", proc->pid, sig);
+	dkprintf("do_signal(): tid=%d, pid=%d, sig=%d\n", thread->tid, proc->pid, sig);
 	orgsig = sig;
 
 	if((proc->ptrace & PT_TRACED) &&
@@ -718,12 +718,12 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 		rc = regs->gpr.rax;
 	}
 
-	irqstate = ihk_mc_spinlock_lock(&thread->sigcommon->lock);
+	mcs_rwlock_writer_lock(&thread->sigcommon->lock, &mcs_rw_node);
 	k = thread->sigcommon->action + sig - 1;
 
 	if(k->sa.sa_handler == SIG_IGN){
 		kfree(pending);
-		ihk_mc_spinlock_unlock(&thread->sigcommon->lock, irqstate);
+		mcs_rwlock_writer_unlock(&thread->sigcommon->lock, &mcs_rw_node);
 		return;
 	}
 	else if(k->sa.sa_handler){
@@ -808,7 +808,7 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 
 		if(copy_to_user(sigsp, &ksigsp, sizeof ksigsp)){
 			kfree(pending);
-			ihk_mc_spinlock_unlock(&thread->sigcommon->lock, irqstate);
+			mcs_rwlock_writer_unlock(&thread->sigcommon->lock, &mcs_rw_node);
 			kprintf("do_signal,write_process_vm failed\n");
 			terminate(0, sig);
 			return;
@@ -827,7 +827,7 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 		if(!(k->sa.sa_flags & SA_NODEFER))
 			thread->sigmask.__val[0] |= pending->sigmask.__val[0];
 		kfree(pending);
-		ihk_mc_spinlock_unlock(&thread->sigcommon->lock, irqstate);
+		mcs_rwlock_writer_unlock(&thread->sigcommon->lock, &mcs_rw_node);
 		if(regs->gpr.rflags & RFLAGS_TF){
 			struct siginfo info;
 
@@ -853,7 +853,7 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 		}
 		else
 			kfree(pending);
-		ihk_mc_spinlock_unlock(&thread->sigcommon->lock, irqstate);
+		mcs_rwlock_writer_unlock(&thread->sigcommon->lock, &mcs_rw_node);
 		switch (sig) {
 		case SIGSTOP:
 		case SIGTSTP:
@@ -885,7 +885,8 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 				/* Wake up the parent who tried wait4 and sleeping */
 				waitq_wakeup(&proc->parent->waitpid_q);
 
-				dkprintf("do_signal,SIGSTOP,sleeping\n");
+				dkprintf("do_signal(): pid: %d, tid: %d SIGSTOP, sleeping\n", 
+					proc->pid, thread->tid);
 				/* Sleep */
 				schedule();
 				dkprintf("SIGSTOP(): woken up\n");
@@ -899,7 +900,7 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 
 			/* Update thread state in fork tree */
 			mcs_rwlock_writer_lock(&proc->update_lock, &lock);	
-			proc->exit_status = SIGTRAP;
+			thread->exit_status = SIGTRAP;
 			proc->status = PS_TRACED;
 			thread->status = PS_TRACED;
 			mcs_rwlock_writer_unlock(&proc->update_lock, &lock);	
@@ -953,11 +954,11 @@ do_signal(unsigned long rc, void *regs0, struct thread *thread, struct sig_pendi
 static struct sig_pending *
 getsigpending(struct thread *thread, int delflag){
 	struct list_head *head;
-	ihk_spinlock_t *lock;
+	mcs_rwlock_lock_t *lock;
+	struct mcs_rwlock_node_irqsave mcs_rw_node;
 	struct sig_pending *next;
 	struct sig_pending *pending;
 	__sigset_t w;
-	int	irqstate;
 	__sigset_t x;
 	int sig;
 	struct k_sigaction *k;
@@ -966,8 +967,12 @@ getsigpending(struct thread *thread, int delflag){
 
 	lock = &thread->sigcommon->lock;
 	head = &thread->sigcommon->sigpending;
-	for(;;){
-		irqstate = ihk_mc_spinlock_lock(lock);
+	for(;;) {
+		if (delflag)
+			mcs_rwlock_writer_lock(lock, &mcs_rw_node);
+		else 
+			mcs_rwlock_reader_lock(lock, &mcs_rw_node);
+
 		list_for_each_entry_safe(pending, next, head, list){
 			for(x = pending->sigmask.__val[0], sig = 0; x; sig++, x >>= 1);
 			k = thread->sigcommon->action + sig - 1;
@@ -976,17 +981,26 @@ getsigpending(struct thread *thread, int delflag){
 			   (k->sa.sa_handler != (void *)1 &&
 			    k->sa.sa_handler != NULL)){
 				if(!(pending->sigmask.__val[0] & w)){
-					if(delflag)
+					if(delflag) 
 						list_del(&pending->list);
-					ihk_mc_spinlock_unlock(lock, irqstate);
+
+					if (delflag)
+						mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
+					else 
+						mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
 					return pending;
 				}
 			}
 		}
-		ihk_mc_spinlock_unlock(lock, irqstate);
+
+		if (delflag)
+			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
+		else 
+			mcs_rwlock_reader_unlock(lock, &mcs_rw_node);
 
 		if(lock == &thread->sigpendinglock)
 			return NULL;
+
 		lock = &thread->sigpendinglock;
 		head = &thread->sigpending;
 	}
@@ -1034,22 +1048,25 @@ check_signal(unsigned long rc, void *regs0, int num)
 			}
 		}
 		ihk_mc_spinlock_unlock(&(cpu_local_var(runq_lock)), irqstate);
-		return;
+		goto out;
 	}
 
 	if(regs != NULL && !interrupt_from_user(regs)) {
-		return;
+		goto out;
 	}
 
 	for(;;){
 		pending = getsigpending(thread, 1);
 		if(!pending) {
 			dkprintf("check_signal,queue is empty\n");
-			return;
+			goto out;
 		}
 
 		do_signal(rc, regs, thread, pending, num);
 	}
+
+out:
+	return;
 }
 
 unsigned long
@@ -1063,7 +1080,8 @@ do_kill(struct thread *thread, int pid, int tid, int sig, siginfo_t *info,
 	struct thread *tthread = NULL;
 	int i;
 	__sigset_t mask;
-	ihk_spinlock_t *savelock = NULL;
+	mcs_rwlock_lock_t *savelock = NULL;
+	struct mcs_rwlock_node mcs_rw_node;
 	struct list_head *head = NULL;
 	int rc;
 	unsigned long irqstate = 0;
@@ -1247,7 +1265,7 @@ done:
 
 	doint = 0;
 
-	ihk_mc_spinlock_lock_noirq(savelock);
+	mcs_rwlock_writer_lock_noirq(savelock, &mcs_rw_node);
 
 	/* Put signal event even when handler is SIG_IGN or SIG_DFL
 	   because target ptraced thread must call ptrace_report_signal 
@@ -1286,7 +1304,7 @@ done:
 			}
 		}
 	}
-	ihk_mc_spinlock_unlock_noirq(savelock);
+	mcs_rwlock_writer_unlock_noirq(savelock, &mcs_rw_node);
 	cpu_restore_interrupt(irqstate);
 
 	if (doint && !(mask & tthread->sigmask.__val[0])) {

arch/x86/tools/mcreboot-smp-x86.sh.in

@@ -39,7 +39,9 @@ else
 	irqbalance_used="no"
 fi
 
-while getopts :i:k:c:m:o:f: OPT
+turbo=""
+
+while getopts :ti:k:c:m:o:f: OPT
 do
 	case ${OPT} in
 	f)	facility=${OPTARG}
@@ -76,6 +78,8 @@ do
 		;;
 	m) mem=${OPTARG}
 		;;
+	t) turbo="turbo"
+		;;
 	*)  echo "invalid option -${OPT}" >&2
 		exit 1
 	esac
@@ -340,7 +344,7 @@ if ! ${SBINDIR}/ihkosctl 0 load ${KERNDIR}/mckernel.img; then
 fi
 
 # Set kernel arguments
-if ! ${SBINDIR}/ihkosctl 0 kargs "hidos ksyslogd=${LOGMODE}"; then
+if ! ${SBINDIR}/ihkosctl 0 kargs "hidos ksyslogd=${LOGMODE} $turbo"; then
 	echo "error: setting kernel arguments" >&2
 	error_exit "os_created"
 fi
@@ -374,7 +378,7 @@ if [ "$enable_mcoverlay" == "yes" ]; then
 	fi
 	while [ ! -e /proc/mcos0 ]
 	do
-		sleep 1
+		sleep 0.1
 	done
 	if [ ! -e /tmp/mcos/mcos0_proc ]; then mkdir -p /tmp/mcos/mcos0_proc; fi
 	if [ ! -e /tmp/mcos/mcos0_proc_upper ]; then mkdir -p /tmp/mcos/mcos0_proc_upper; fi

configure

@@ -2922,6 +2922,7 @@ ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $
 ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 	XCC=$CC
+	CFLAGS="$CFLAGS -ffreestanding -fno-tree-loop-distribute-patterns"
 	;;
     builtin-mic)
 	ARCH=k1om
@@ -3912,6 +3913,7 @@ fi
 
 
 
+
 
 
 ac_config_headers="$ac_config_headers executer/config.h"

configure.ac

@@ -70,6 +70,7 @@ case $WITH_TARGET in
 	ARCH=`uname -m`
 	AC_PROG_CC
 	XCC=$CC
+	CFLAGS="$CFLAGS -ffreestanding -fno-tree-loop-distribute-patterns"
 	;;
     builtin-mic)
 	ARCH=k1om
@@ -286,6 +287,7 @@ AC_SUBST(ETCDIR)
 AC_SUBST(KMODDIR)
 AC_SUBST(KERNDIR)
 AC_SUBST(MANDIR)
+AC_SUBST(CFLAGS)
 AC_SUBST(ENABLE_MCOVERLAYFS)
 
 AC_SUBST(IHK_VERSION)

executer/user/mcexec.c

@@ -1550,7 +1550,7 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
-	n_threads = ncpu + 1;
+	n_threads = ncpu;
 
 	/* 
 	 * XXX: keep thread_data ncpu sized despite that there are only
@@ -1561,6 +1561,10 @@ int main(int argc, char **argv)
 	 * TODO: implement dynaic thread pool resizing.
 	 */
 	thread_data = (struct thread_data_s *)malloc(sizeof(struct thread_data_s) * (ncpu + 1));
+	if (!thread_data) {
+		fprintf(stderr, "error: allocating thread pool data\n");
+		return 1;
+	}
 	memset(thread_data, '\0', sizeof(struct thread_data_s) * (ncpu + 1));
 
 #if 0	

kernel/Makefile.build.in

@@ -6,12 +6,12 @@ OBJS += process.o copy.o waitq.o futex.o timer.o plist.o fileobj.o shmobj.o
 OBJS += zeroobj.o procfs.o devobj.o sysfs.o
 DEPSRCS=$(wildcard $(SRC)/*.c)
 
-CFLAGS += -I$(SRC)/include -D__KERNEL__ -g
+CFLAGS += -I$(SRC)/include -D__KERNEL__ -g -fno-omit-frame-pointer -fno-inline -fno-inline-small-functions
 LDFLAGS += -e arch_start
 IHKOBJ = ihk/ihk.o
 
 include $(SRC)/config/config.$(TARGET)
-include $(IHKBASE)/Makefile.common
+include @abs_builddir@/../../ihk/cokernel/Makefile.common
 
 # CFLAGS += -I$(SRC)/../arch/$(IHKARCH)/kernel/include -I$(SRC)/../lib/include
 

kernel/Makefile.in

@@ -9,7 +9,7 @@ V ?= $(VERBOSE)
 KERNEL = kernel.img
 KERNELS = $(addsuffix /$(KERNEL),$(addprefix $(O)/,$(BUILD_TARGET)))
 
-SUBCMD_OPTS = V='$(V)'
+SUBCMD_OPTS = V='$(V)' BUILD_IHK_COKERNEL=@abs_builddir@/../../ihk/cokernel
 
 $(if $(O),,$(error Specify the compilation target directory))
 #$(if $(shell ls $(IHKBASE)/Makefile),,\

kernel/include/init.h

@@ -32,4 +32,6 @@ extern void cpu_sysfs_setup(void);
 
 extern char *find_command_line(char *name);
 
+extern int num_processors;
+
 #endif

kernel/include/process.h

@@ -232,6 +232,8 @@ enum mpol_rebind_step {
 #include <waitq.h>
 #include <futex.h>
 
+//#define TRACK_SYSCALLS
+
 struct resource_set;
 struct process_hash;
 struct thread_hash;
@@ -405,7 +407,7 @@ struct mckfd {
 #define SFD_NONBLOCK 04000
 
 struct sig_common {
-	ihk_spinlock_t	lock;
+	mcs_rwlock_lock_t lock;
 	ihk_atomic_t use;
 	struct k_sigaction action[_NSIG];
 	struct list_head sigpending;
@@ -466,7 +468,7 @@ struct process {
 			// V       +----   |
 			// PS_STOPPED -----+
 			// (PS_TRACED)
-	int exit_status;
+	int exit_status; // only for zombie
 
 	/* Store exit_status for a group of threads when stopped by SIGSTOP.
 	   exit_status can't be used because values of exit_status of threads
@@ -578,6 +580,7 @@ struct thread {
 			// PS_TRACED
 			// PS_INTERRPUTIBLE
 			// PS_UNINTERRUPTIBLE
+	int exit_status;
 
 	// process vm
 	struct process_vm *vm;
@@ -608,12 +611,20 @@ struct thread {
 	fp_regs_struct *fp_regs;
 	int in_syscall_offload;
 
+#ifdef TRACK_SYSCALLS
+	int socc_enabled;
+	uint64_t *syscall_times;
+	uint32_t *syscall_cnts;
+	uint64_t *offload_times;
+	uint32_t *offload_cnts;
+#endif // TRACK_SYSCALLS
+
 	// signal
 	struct sig_common *sigcommon;
 	sigset_t sigmask;
 	stack_t sigstack;
 	struct list_head sigpending;
-	ihk_spinlock_t	sigpendinglock;
+	mcs_rwlock_lock_t sigpendinglock;
 	volatile int sigevent;
 
 	// gpio

kernel/mem.c

@@ -754,6 +754,8 @@ void remote_flush_tlb_cpumask(struct process_vm *vm,
 			flush_tlb();
 		}
 
+		/* Flush on this core */
+		flush_tlb_single(addr & PAGE_MASK);
 		/* Wait for all cores */
 		while (ihk_atomic_read(&flush_entry->pending) != 0) {
 			cpu_pause();

kernel/process.c

@@ -74,7 +74,6 @@ init_process(struct process *proc, struct process *parent)
 {
 	/* These will be filled out when changing status */
 	proc->pid = -1;
-	proc->exit_status = -1;
 	proc->status = PS_RUNNING;
 
 	if(parent){
@@ -278,10 +277,10 @@ create_thread(unsigned long user_pc)
 	dkprintf("fork(): sigshared\n");
 
 	ihk_atomic_set(&thread->sigcommon->use, 1);
-	ihk_mc_spinlock_init(&thread->sigcommon->lock);
+	mcs_rwlock_init(&thread->sigcommon->lock);
 	INIT_LIST_HEAD(&thread->sigcommon->sigpending);
 
-	ihk_mc_spinlock_init(&thread->sigpendinglock);
+	mcs_rwlock_init(&thread->sigpendinglock);
 	INIT_LIST_HEAD(&thread->sigpending);
 
 	thread->sigstack.ss_sp = NULL;
@@ -298,6 +297,7 @@ create_thread(unsigned long user_pc)
 	if(init_process_vm(proc, asp, vm) != 0){
 		goto err;
 	}
+	thread->exit_status = -1;
 
 	cpu_set(ihk_mc_get_processor_id(), &thread->vm->address_space->cpu_set,
 			&thread->vm->address_space->cpu_set_lock);
@@ -441,11 +441,11 @@ clone_thread(struct thread *org, unsigned long pc, unsigned long sp,
 		memcpy(thread->sigcommon->action, org->sigcommon->action,
 		       sizeof(struct k_sigaction) * _NSIG);
 		ihk_atomic_set(&thread->sigcommon->use, 1);
-		ihk_mc_spinlock_init(&thread->sigcommon->lock);
+		mcs_rwlock_init(&thread->sigcommon->lock);
 		INIT_LIST_HEAD(&thread->sigcommon->sigpending);
 		// TODO: copy signalfd
 	}
-	ihk_mc_spinlock_init(&thread->sigpendinglock);
+	mcs_rwlock_init(&thread->sigpendinglock);
 	INIT_LIST_HEAD(&thread->sigpending);
 	thread->sigmask = org->sigmask;
 
@@ -1690,10 +1690,9 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui
 	range = lookup_process_memory_range(vm, fault_addr, fault_addr+1);
 	if (range == NULL) {
 		error = -EFAULT;
-		dkprintf("[%d]do_page_fault_process_vm(%p,%lx,%lx):"
-				"out of range. %d\n",
-				ihk_mc_get_processor_id(), vm,
-				fault_addr0, reason, error);
+		dkprintf("do_page_fault_process_vm(): vm: %p, addr: %p, reason: %lx):"
+				"out of range: %d\n",
+				vm, fault_addr0, reason, error);
 		goto out;
 	}
 
@@ -1723,10 +1722,18 @@ static int do_page_fault_process_vm(struct process_vm *vm, void *fault_addr0, ui
 			kprintf("if (((range->flag & VR_PROT_MASK) == VR_PROT_NONE))\n");
 		if (((reason & PF_WRITE) && !(reason & PF_PATCH)))
 			kprintf("if (((reason & PF_WRITE) && !(reason & PF_PATCH)))\n");
-		if (!(range->flag & VR_PROT_WRITE))
+		if (!(range->flag & VR_PROT_WRITE)) {
 			kprintf("if (!(range->flag & VR_PROT_WRITE))\n");
-		if ((reason & PF_INSTR) && !(range->flag & VR_PROT_EXEC))
+			//kprintf("setting VR_PROT_WRITE\n");
+			//range->flag |= VR_PROT_WRITE;
+			//goto cont;
+		}
+		if ((reason & PF_INSTR) && !(range->flag & VR_PROT_EXEC)) {
 			kprintf("if ((reason & PF_INSTR) && !(range->flag & VR_PROT_EXEC))\n");
+			//kprintf("setting VR_PROT_EXEC\n");
+			//range->flag |= VR_PROT_EXEC;
+			//goto cont;
+		}
 		goto out;
 	}
 
@@ -2717,7 +2724,9 @@ redo:
 			restore_fp_regs(next);
 		}
 
-		ihk_mc_load_page_table(next->vm->address_space->page_table);
+		if (prev && prev->vm->address_space->page_table !=
+				next->vm->address_space->page_table)
+			ihk_mc_load_page_table(next->vm->address_space->page_table);
 
 		dkprintf("[%d] schedule: tlsblock_base: 0x%lX\n",
 		         ihk_mc_get_processor_id(), next->tlsblock_base);
@@ -2949,6 +2958,7 @@ find_thread(int pid, int tid, struct mcs_rwlock_node_irqsave *lock)
 	if(tid <= 0)
 		return NULL;
 	mcs_rwlock_reader_lock(&thash->lock[hash], lock);
+retry:
 	list_for_each_entry(thread, &thash->list[hash], hash_list){
 		if(thread->tid == tid){
 			if(pid <= 0)
@@ -2957,6 +2967,13 @@ find_thread(int pid, int tid, struct mcs_rwlock_node_irqsave *lock)
 				return thread;
 		}
 	}
+	/* If no thread with pid == tid was found, then we may be looking for a
+	 * specific thread (not the main thread of the process), try to find it
+	 * based on tid only */
+	if (pid > 0 && pid == tid) {
+		pid = 0;
+		goto retry;
+	}
 	mcs_rwlock_reader_unlock(&thash->lock[hash], lock);
 	return NULL;
 }

kernel/procfs.c

@@ -24,6 +24,7 @@
 #include <page.h>
 #include <mman.h>
 #include <bitmap.h>
+#include <init.h>
 
 //#define DEBUG_PRINT_PROCFS
 

kernel/syscall.c

@@ -129,6 +129,95 @@ int prepare_process_ranges_args_envs(struct thread *thread,
 static void do_mod_exit(int status);
 #endif
 
+#ifdef TRACK_SYSCALLS
+
+#define SOCC_CLEAR  1
+#define SOCC_ON     2
+#define SOCC_OFF    4
+#define SOCC_PRINT  8
+
+void print_syscall_stats(struct thread *thread)
+{
+	int i;
+	unsigned long flags;
+
+	flags = kprintf_lock();
+
+	for (i = 0; i < 300; ++i) {
+		if (!thread->syscall_cnts[i] &&
+				!thread->offload_cnts[i]) continue;
+
+		//__kprintf("(%20s): sys.cnt: %3lu (%15lukC)\n",
+		__kprintf("(%3d,%20s): sys.cnt: %5lu (%10lukC), offl.cnt: %5lu (%10lukC)\n",
+				i,
+				syscall_name[i],
+				thread->syscall_cnts[i],
+				(thread->syscall_times[i] /
+				 (thread->syscall_cnts[i] ? thread->syscall_cnts[i] : 1))
+				/ 1000,
+				thread->offload_cnts[i],
+				(thread->offload_times[i] /
+				 (thread->offload_cnts[i] ? thread->offload_cnts[i] : 1))
+				/ 1000
+				);
+	}
+
+	kprintf_unlock(flags);
+}
+
+void alloc_syscall_counters(struct thread *thread)
+{
+	thread->syscall_times = kmalloc(sizeof(*thread->syscall_times) * 300, IHK_MC_AP_NOWAIT);
+	thread->syscall_cnts = kmalloc(sizeof(*thread->syscall_cnts) * 300, IHK_MC_AP_NOWAIT);
+	thread->offload_times = kmalloc(sizeof(*thread->offload_times) * 300, IHK_MC_AP_NOWAIT);
+	thread->offload_cnts = kmalloc(sizeof(*thread->offload_cnts) * 300, IHK_MC_AP_NOWAIT);
+
+	if (!thread->syscall_times ||
+			!thread->syscall_cnts ||
+			!thread->offload_times ||
+			!thread->offload_cnts) {
+		kprintf("ERROR: allocating counters\n");
+		panic("");
+	}
+
+	memset(thread->syscall_times, 0, sizeof(*thread->syscall_times) * 300);
+	memset(thread->syscall_cnts, 0, sizeof(*thread->syscall_cnts) * 300);
+	memset(thread->offload_times, 0, sizeof(*thread->offload_times) * 300);
+	memset(thread->offload_cnts, 0, sizeof(*thread->offload_cnts) * 300);
+}
+
+SYSCALL_DECLARE(syscall_offload_clr_cntrs)
+{
+	int flag = (int)ihk_mc_syscall_arg0(ctx);
+	struct thread *thread = cpu_local_var(current);
+	int i;
+
+	if (flag & SOCC_PRINT)
+		print_syscall_stats(thread);
+
+	if (flag & SOCC_CLEAR) {
+		for (i = 0; i < 300; ++i) {
+			if (!thread->syscall_cnts[i] &&
+					!thread->offload_cnts[i]) continue;
+
+			thread->syscall_cnts[i] = 0;
+			thread->syscall_times[i] = 0;
+			thread->offload_cnts[i] = 0;
+			thread->offload_times[i] = 0;
+		}
+	}
+
+	if (flag & SOCC_ON) {
+		thread->socc_enabled = 1;
+	}
+	else if (flag & SOCC_OFF) {
+		thread->socc_enabled = 0;
+	}
+
+	return 0;
+}
+#endif // TRACK_SYSCALLS
+
 static void send_syscall(struct syscall_request *req, int cpu, int pid, struct syscall_response *res)
 {
 	struct ikc_scd_packet packet IHK_DMA_ALIGN;
@@ -189,6 +278,10 @@ long do_syscall(struct syscall_request *req, int cpu, int pid)
 	unsigned long irqstate;
 	struct thread *thread = cpu_local_var(current);
 	struct process *proc = thread->proc;
+#ifdef TRACK_SYSCALLS
+	uint64_t t_s;
+	t_s = rdtsc();
+#endif // TRACK_SYSCALLS
 
 	dkprintf("SC(%d)[%3d] sending syscall\n",
 		ihk_mc_get_processor_id(),
@@ -307,6 +400,23 @@ long do_syscall(struct syscall_request *req, int cpu, int pid)
 	if(req->number != __NR_exit_group){
 		--thread->in_syscall_offload;
 	}
+
+#ifdef TRACK_SYSCALLS
+	if (req->number < 300) {
+		if (!cpu_local_var(current)->offload_cnts) {
+			alloc_syscall_counters(cpu_local_var(current));
+		}
+		if (cpu_local_var(current)->socc_enabled) {
+			cpu_local_var(current)->offload_times[req->number] += 
+				(rdtsc() - t_s);
+			cpu_local_var(current)->offload_cnts[req->number]++;
+		}
+	}
+	else {
+		dkprintf("offload syscall > 300?? : %d\n", req->number);
+	}
+#endif // TRACK_SYSCALLS
+
 	return rc;
 }
 
@@ -347,16 +457,16 @@ static int wait_zombie(struct thread *thread, struct process *child, int *status
     return ret;
 }
 
-static int wait_stopped(struct thread *thread, struct process *child, int *status, int options)
+static int wait_stopped(struct thread *thread, struct process *child, struct thread *c_thread, int *status, int options)
 {
 	dkprintf("wait_stopped,proc->pid=%d,child->pid=%d,options=%08x\n",
 			 thread->proc->pid, child->pid, options);
 	int ret;
 
 	/* Copy exit_status created in do_signal */
-	int *exit_status = child->status == PS_STOPPED ? 
+	int *exit_status = (child->status == PS_STOPPED || !c_thread) ? 
 		&child->group_exit_status :
-		&child->exit_status;
+		&c_thread->exit_status;
 
 	/* Skip this process because exit_status has been reaped. */
 	if (!*exit_status) {
@@ -400,6 +510,26 @@ static int wait_continued(struct thread *thread, struct process *child, int *sta
 	return ret;
 }
 
+struct thread *find_thread_of_process(struct process *child, int pid)
+{
+	int c_found = 0;
+	struct mcs_rwlock_node c_lock;
+	struct thread *c_thread = NULL;
+
+	mcs_rwlock_reader_lock_noirq(&child->threads_lock, &c_lock);
+	list_for_each_entry(c_thread, &child->threads_list, siblings_list) {
+		if (c_thread->tid == pid) {
+			c_found = 1;
+			break;
+		}
+	}
+	mcs_rwlock_reader_unlock_noirq(&child->threads_lock, &c_lock);
+	if (!c_found) c_thread = NULL;
+
+	return c_thread;
+}
+
+
 /* 
  * From glibc: INLINE_SYSCALL (wait4, 4, pid, stat_loc, options, NULL);
  */
@@ -415,22 +545,30 @@ do_wait(int pid, int *status, int options, void *rusage)
 	int empty = 1;
 	int orgpid = pid;
 	struct mcs_rwlock_node lock;
+	struct thread *c_thread = NULL;
 
-	dkprintf("wait4,thread->pid=%d,pid=%d\n", thread->proc->pid, pid);
+	dkprintf("wait4(): current->proc->pid: %d, pid: %d\n", thread->proc->pid, pid);
 
  rescan:
 	pid = orgpid;
 
 	mcs_rwlock_writer_lock_noirq(&thread->proc->children_lock, &lock);
 	list_for_each_entry_safe(child, next, &proc->children_list, siblings_list) {	
+		/*
 		if (!(!!(options & __WCLONE) ^ (child->termsig == SIGCHLD))) {
 			continue;
 		}
+		*/
+
+		/* Find thread with pid == tid, this will be either the main thread
+		 * or the one we are looking for specifically when __WCLONE is passed */
+		//if (options & __WCLONE) 
+		c_thread = find_thread_of_process(child, pid);
 
 		if ((pid < 0 && -pid == child->pgid) ||
 			pid == -1 ||
 			(pid == 0 && pgid == child->pgid) ||
-			(pid > 0 && pid == child->pid)) {
+			(pid > 0 && pid == child->pid) || c_thread != NULL) {
 
 			empty = 0;
 
@@ -478,8 +616,11 @@ do_wait(int pid, int *status, int options, void *rusage)
 			if(!(child->ptrace & PT_TRACED) &&
 			   (child->signal_flags & SIGNAL_STOP_STOPPED) &&
 			   (options & WUNTRACED)) {
+				/* Find main thread of process if pid == -1 */
+				if (pid == -1)
+					c_thread = find_thread_of_process(child, child->pid);
 				/* Not ptraced and in stopped state and WUNTRACED is specified */
-				ret = wait_stopped(thread, child, status, options);
+				ret = wait_stopped(thread, child, c_thread, status, options);
 				if(!(options & WNOWAIT)){
 					child->signal_flags &= ~SIGNAL_STOP_STOPPED;
 				}
@@ -489,8 +630,15 @@ do_wait(int pid, int *status, int options, void *rusage)
 
 			if((child->ptrace & PT_TRACED) &&
 			   (child->status & (PS_STOPPED | PS_TRACED))) {
-				ret = wait_stopped(thread, child, status, options);
-				if(ret == child->pid){
+				/* Find main thread of process if pid == -1 */
+				if (pid == -1)
+					c_thread = find_thread_of_process(child, child->pid);
+				ret = wait_stopped(thread, child, c_thread, status, options);
+				if(c_thread && ret == child->pid){
+					/* Are we looking for a specific thread? */
+					if (pid == c_thread->tid) {
+						ret = c_thread->tid;
+					}
 					if(!(options & WNOWAIT)){
 						child->signal_flags &= ~SIGNAL_STOP_STOPPED;
 					}
@@ -639,6 +787,7 @@ terminate(int rc, int sig)
 	int n;
 	int *ids = NULL;
 	struct syscall_request request IHK_DMA_ALIGN;
+	int exit_status;
 
 	// sync perf info
 	if(proc->monitoring_event)
@@ -655,7 +804,7 @@ terminate(int rc, int sig)
 		// no return
 		return;
 	}
-	proc->exit_status = ((rc & 0x00ff) << 8) | (sig & 0xff);
+	exit_status = mythread->exit_status = ((rc & 0x00ff) << 8) | (sig & 0xff);
 	proc->status = PS_EXITED;
 	mcs_rwlock_writer_unlock_noirq(&proc->update_lock, &updatelock);
 	mcs_rwlock_reader_unlock(&proc->threads_lock, &lock);
@@ -791,7 +940,7 @@ terminate(int rc, int sig)
 	// clean up memory
 	if(!proc->nohost){
 		request.number = __NR_exit_group;
-		request.args[0] = proc->exit_status;
+		request.args[0] = exit_status;
 		do_syscall(&request, ihk_mc_get_processor_id(), proc->pid);
 		proc->nohost = 1;
 	}
@@ -803,6 +952,7 @@ terminate(int rc, int sig)
 	}
 	else {
 		proc->status = PS_ZOMBIE;
+		proc->exit_status = exit_status;
 
 		dkprintf("terminate,wakeup\n");
 
@@ -813,11 +963,11 @@ terminate(int rc, int sig)
 
 			memset(&info, '\0', sizeof info);
 			info.si_signo = SIGCHLD;
-			info.si_code = (proc->exit_status & 0x7f)?
-			               ((proc->exit_status & 0x80)?
+			info.si_code = (exit_status & 0x7f)?
+			               ((exit_status & 0x80)?
 			                CLD_DUMPED: CLD_KILLED): CLD_EXITED;
 			info._sifields._sigchld.si_pid = proc->pid;
-			info._sifields._sigchld.si_status = proc->exit_status;
+			info._sifields._sigchld.si_status = exit_status;
 			error = do_kill(NULL, proc->parent->pid, -1, SIGCHLD, &info, 0);
 			dkprintf("terminate,klll %d,error=%d\n",
 					proc->termsig, error);
@@ -855,7 +1005,7 @@ interrupt_syscall(int pid, int tid)
 	ihk_mc_user_context_t ctx;
 	long lerror;
 
-kprintf("interrupt_syscall pid=%d tid=%d\n", pid, tid);
+	dkprintf("interrupt_syscall pid=%d tid=%d\n", pid, tid);
 	ihk_mc_syscall_arg0(&ctx) = pid;
 	ihk_mc_syscall_arg1(&ctx) = tid;
 
@@ -1065,7 +1215,8 @@ do_mmap(const intptr_t addr0, const size_t len0, const int prot,
 	}
 	else {
 		/* choose mapping address */
-		error = search_free_space(len, region->map_end,
+		error = search_free_space(len, region->map_end +
+				(fd > 0) ? PTL4_SIZE : 0,
 				PAGE_SHIFT+p2align, &addr);
 		if (error) {
 			ekprintf("do_mmap:search_free_space(%lx,%lx,%d) failed. %d\n",
@@ -1222,7 +1373,7 @@ out:
 	}
 	ihk_mc_spinlock_unlock_noirq(&thread->vm->memory_range_lock);
 
-	if (!error && populated_mapping) {
+	if (!error && populated_mapping && !((vrflags & VR_PROT_MASK) == VR_PROT_NONE)) {
 		error = populate_process_memory(thread->vm, (void *)addr, len);
 		if (error) {
 			ekprintf("%s: error :populate_process_memory"
@@ -1595,7 +1746,7 @@ static int ptrace_report_clone(struct thread *thread, struct thread *new, int ev
 
 	/* Save reason why stopped and process state for wait4() to reap */
 	mcs_rwlock_writer_lock_noirq(&thread->proc->update_lock, &lock);
-	thread->proc->exit_status = (SIGTRAP | (event << 8));
+	thread->exit_status = (SIGTRAP | (event << 8));
 	/* Transition process state */
 	thread->proc->status = PS_TRACED;
 	thread->status = PS_TRACED;
@@ -1610,24 +1761,26 @@ static int ptrace_report_clone(struct thread *thread, struct thread *new, int ev
 		mcs_rwlock_writer_lock_noirq(&new->proc->update_lock, &updatelock);
 		/* set ptrace features to new process */
 		new->proc->ptrace = thread->proc->ptrace;
-		new->proc->ppid_parent = new->proc->parent; /* maybe proc */
+		if (event != PTRACE_EVENT_CLONE) {
+			new->proc->ppid_parent = new->proc->parent; /* maybe proc */
+		}
 
 		if ((new->proc->ptrace & PT_TRACED) && new->ptrace_debugreg == NULL) {
 			alloc_debugreg(new);
 		}
 
-		mcs_rwlock_writer_lock_noirq(&new->proc->parent->children_lock, &lock);
-		list_del(&new->proc->siblings_list);
-		list_add_tail(&new->proc->ptraced_siblings_list, &new->proc->parent->ptraced_children_list);
-		mcs_rwlock_writer_unlock_noirq(&new->proc->parent->children_lock, &lock);
-
-		new->proc->parent = thread->proc->parent; /* new ptracing parent */
-		mcs_rwlock_writer_lock_noirq(&new->proc->parent->children_lock, &lock);
-		list_add_tail(&new->proc->siblings_list, &new->proc->parent->children_list);
-		mcs_rwlock_writer_unlock_noirq(&new->proc->parent->children_lock, &lock);
-
+		if (event != PTRACE_EVENT_CLONE) {
+			mcs_rwlock_writer_lock_noirq(&new->proc->parent->children_lock, &lock);
+			list_del(&new->proc->siblings_list);
+			list_add_tail(&new->proc->ptraced_siblings_list, &new->proc->parent->ptraced_children_list);
+			mcs_rwlock_writer_unlock_noirq(&new->proc->parent->children_lock, &lock);
+			new->proc->parent = thread->proc->parent; /* new ptracing parent */
+			mcs_rwlock_writer_lock_noirq(&new->proc->parent->children_lock, &lock);
+			list_add_tail(&new->proc->siblings_list, &new->proc->parent->children_list);
+			mcs_rwlock_writer_unlock_noirq(&new->proc->parent->children_lock, &lock);
+		}
 		/* trace and SIGSTOP */
-		new->proc->exit_status = SIGSTOP;
+		new->exit_status = SIGSTOP;
 		new->proc->status = PS_TRACED;
 		new->status = PS_TRACED;
 
@@ -1639,7 +1792,7 @@ static int ptrace_report_clone(struct thread *thread, struct thread *new, int ev
 	info.si_signo = SIGCHLD;
 	info.si_code = CLD_TRAPPED;
 	info._sifields._sigchld.si_pid = thread->proc->pid;
-	info._sifields._sigchld.si_status = thread->proc->exit_status;
+	info._sifields._sigchld.si_status = thread->exit_status;
 	rc = do_kill(cpu_local_var(current), parent_pid, -1, SIGCHLD, &info, 0);
 	if(rc < 0) {
 		dkprintf("ptrace_report_clone,do_kill failed\n");
@@ -2083,8 +2236,8 @@ retry_tid:
 		}
 	}
 
-	dkprintf("clone: kicking scheduler!,cpuid=%d pid=%d tid %d -> tid=%d\n", 
-		cpuid, newproc->pid, 
+	dkprintf("clone: kicking scheduler!,cpuid=%d pid=%d tid %d -> tid=%d\n",
+		cpuid, newproc->pid,
 		old->tid,
 		new->tid);
 
@@ -2473,16 +2626,16 @@ do_sigaction(int sig, struct k_sigaction *act, struct k_sigaction *oact)
 {
 	struct thread *thread = cpu_local_var(current);
 	struct k_sigaction *k;
-	long	irqstate;
+	struct mcs_rwlock_node_irqsave mcs_rw_node;
 	ihk_mc_user_context_t ctx0;
 
-	irqstate = ihk_mc_spinlock_lock(&thread->sigcommon->lock);
+	mcs_rwlock_writer_lock(&thread->sigcommon->lock, &mcs_rw_node);
 	k = thread->sigcommon->action + sig - 1;
 	if(oact)
 		memcpy(oact, k, sizeof(struct k_sigaction));
 	if(act)
 		memcpy(k, act, sizeof(struct k_sigaction));
-	ihk_mc_spinlock_unlock(&thread->sigcommon->lock, irqstate);
+	mcs_rwlock_writer_unlock(&thread->sigcommon->lock, &mcs_rw_node);
 
 	if(act){
 		ihk_mc_syscall_arg0(&ctx0) = sig;
@@ -2654,10 +2807,10 @@ fault:
 
 SYSCALL_DECLARE(rt_sigpending)
 {
-	int flag;
 	struct sig_pending *pending;
 	struct list_head *head;
-	ihk_spinlock_t *lock;
+	mcs_rwlock_lock_t *lock;
+	struct mcs_rwlock_node_irqsave mcs_rw_node;
 	__sigset_t w = 0;
 	struct thread *thread = cpu_local_var(current);
 	sigset_t *set = (sigset_t *)ihk_mc_syscall_arg0(ctx);
@@ -2668,19 +2821,19 @@ SYSCALL_DECLARE(rt_sigpending)
 
 	lock = &thread->sigcommon->lock;
 	head = &thread->sigcommon->sigpending;
-	flag = ihk_mc_spinlock_lock(lock);
+	mcs_rwlock_writer_lock(lock, &mcs_rw_node);
 	list_for_each_entry(pending, head, list){
 		w |= pending->sigmask.__val[0];
 	}
-	ihk_mc_spinlock_unlock(lock, flag);
+	mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 
 	lock = &thread->sigpendinglock;
 	head = &thread->sigpending;
-	flag = ihk_mc_spinlock_lock(lock);
+	mcs_rwlock_writer_lock(lock, &mcs_rw_node);
 	list_for_each_entry(pending, head, list){
 		w |= pending->sigmask.__val[0];
 	}
-	ihk_mc_spinlock_unlock(lock, flag);
+	mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 
 	if(copy_to_user(set->__val, &w, sizeof w))
 		return -EFAULT;
@@ -3215,10 +3368,10 @@ SYSCALL_DECLARE(rt_sigtimedwait)
 	__sigset_t wset;
 	__sigset_t nset;
 	struct timespec wtimeout;
-	unsigned long flag;
 	struct sig_pending *pending;
 	struct list_head *head;
-	ihk_spinlock_t *lock;
+	mcs_rwlock_lock_t *lock;
+	struct mcs_rwlock_node_irqsave mcs_rw_node;
 	int w;
 	int sig;
         struct timespec ats;
@@ -3284,18 +3437,18 @@ SYSCALL_DECLARE(rt_sigtimedwait)
 
 		lock = &thread->sigcommon->lock;
 		head = &thread->sigcommon->sigpending;
-		flag = ihk_mc_spinlock_lock(lock);
+		mcs_rwlock_writer_lock(lock, &mcs_rw_node);
 		list_for_each_entry(pending, head, list){
 			if(pending->sigmask.__val[0] & wset)
 				break;
 		}
 
 		if(&pending->list == head){
-			ihk_mc_spinlock_unlock(lock, flag);
+			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 
 			lock = &thread->sigpendinglock;
 			head = &thread->sigpending;
-			flag = ihk_mc_spinlock_lock(lock);
+			mcs_rwlock_writer_lock(lock, &mcs_rw_node);
 			list_for_each_entry(pending, head, list){
 				if(pending->sigmask.__val[0] & wset)
 					break;
@@ -3305,25 +3458,25 @@ SYSCALL_DECLARE(rt_sigtimedwait)
 		if(&pending->list != head){
 			list_del(&pending->list);
 			thread->sigmask.__val[0] = bset;
-			ihk_mc_spinlock_unlock(lock, flag);
+			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 			break;
 		}
-		ihk_mc_spinlock_unlock(lock, flag);
+		mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 
 		lock = &thread->sigcommon->lock;
 		head = &thread->sigcommon->sigpending;
-		flag = ihk_mc_spinlock_lock(lock);
+		mcs_rwlock_writer_lock(lock, &mcs_rw_node);
 		list_for_each_entry(pending, head, list){
 			if(pending->sigmask.__val[0] & nset)
 				break;
 		}
 
 		if(&pending->list == head){
-			ihk_mc_spinlock_unlock(lock, flag);
+			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 
 			lock = &thread->sigpendinglock;
 			head = &thread->sigpending;
-			flag = ihk_mc_spinlock_lock(lock);
+			mcs_rwlock_writer_lock(lock, &mcs_rw_node);
 			list_for_each_entry(pending, head, list){
 				if(pending->sigmask.__val[0] & nset)
 					break;
@@ -3333,11 +3486,11 @@ SYSCALL_DECLARE(rt_sigtimedwait)
 		if(&pending->list != head){
 			list_del(&pending->list);
 			thread->sigmask.__val[0] = bset;
-			ihk_mc_spinlock_unlock(lock, flag);
+			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 			do_signal(-EINTR, NULL, thread, pending, 0);
 			return -EINTR;
 		}
-		ihk_mc_spinlock_unlock(lock, flag);
+		mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 		thread->sigevent = 0;
 	}
 
@@ -3374,10 +3527,10 @@ do_sigsuspend(struct thread *thread, const sigset_t *set)
 {
 	__sigset_t wset;
 	__sigset_t bset;
-	unsigned long flag;
 	struct sig_pending *pending;
 	struct list_head *head;
-	ihk_spinlock_t *lock;
+	mcs_rwlock_lock_t *lock;
+	struct mcs_rwlock_node_irqsave mcs_rw_node;
 
 	wset = set->__val[0];
 	wset &= ~__sigmask(SIGKILL);
@@ -3392,31 +3545,31 @@ do_sigsuspend(struct thread *thread, const sigset_t *set)
 
 		lock = &thread->sigcommon->lock;
 		head = &thread->sigcommon->sigpending;
-		flag = ihk_mc_spinlock_lock(lock);
+		mcs_rwlock_writer_lock(lock, &mcs_rw_node);
 		list_for_each_entry(pending, head, list){
 			if(!(pending->sigmask.__val[0] & wset))
 				break;
 		}
 
 		if(&pending->list == head){
-			ihk_mc_spinlock_unlock(lock, flag);
+			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 
 			lock = &thread->sigpendinglock;
 			head = &thread->sigpending;
-			flag = ihk_mc_spinlock_lock(lock);
+			mcs_rwlock_writer_lock(lock, &mcs_rw_node);
 			list_for_each_entry(pending, head, list){
 				if(!(pending->sigmask.__val[0] & wset))
 					break;
 			}
 		}
 		if(&pending->list == head){
-			ihk_mc_spinlock_unlock(lock, flag);
+			mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 			thread->sigevent = 0;
 			continue;
 		}
 
 		list_del(&pending->list);
-		ihk_mc_spinlock_unlock(lock, flag);
+		mcs_rwlock_writer_unlock(lock, &mcs_rw_node);
 		thread->sigmask.__val[0] = bset;
 		do_signal(-EINTR, NULL, thread, pending, 0);
 		break;
@@ -5183,29 +5336,37 @@ static int ptrace_attach(int pid)
 		goto out;
 	}
 	child = thread->proc;
-	dkprintf("ptrace_attach,pid=%d,thread->proc->parent=%p\n", thread->proc->pid, thread->proc->parent);
+	dkprintf("ptrace_attach(): pid requested:%d, thread->tid:%d, thread->proc->pid=%d, thread->proc->parent=%p\n", pid, thread->tid, thread->proc->pid, thread->proc->parent);
 
 	mcs_rwlock_writer_lock_noirq(&child->update_lock, &updatelock);
-	if (thread->proc->ptrace & PT_TRACED) {
-		mcs_rwlock_writer_unlock_noirq(&child->update_lock, &updatelock);
-		thread_unlock(thread, &lock);
-		error = -EPERM;
-		goto out;
+
+	/* Only for the first thread of a process XXX: fix this */
+	if (thread->tid == child->pid) {
+		if (thread->proc->ptrace & PT_TRACED) {
+			mcs_rwlock_writer_unlock_noirq(&child->update_lock, &updatelock);
+			thread_unlock(thread, &lock);
+			dkprintf("ptrace_attach: -EPERM\n");
+			error = -EPERM;
+			goto out;
+		}
 	}
 
 	parent = child->parent;
+	/* XXX: tmp */
+	if (parent != proc) {
 
-	dkprintf("ptrace_attach,parent->pid=%d\n", parent->pid);
+		dkprintf("ptrace_attach() parent->pid=%d\n", parent->pid);
 
-	mcs_rwlock_writer_lock_noirq(&parent->children_lock, &childlock);
-	list_del(&child->siblings_list);
-	list_add_tail(&child->ptraced_siblings_list, &parent->ptraced_children_list);
-	mcs_rwlock_writer_unlock_noirq(&parent->children_lock, &childlock);
+		mcs_rwlock_writer_lock_noirq(&parent->children_lock, &childlock);
+		list_del(&child->siblings_list);
+		list_add_tail(&child->ptraced_siblings_list, &parent->ptraced_children_list);
+		mcs_rwlock_writer_unlock_noirq(&parent->children_lock, &childlock);
 
-	mcs_rwlock_writer_lock_noirq(&proc->children_lock, &childlock);
-	list_add_tail(&child->siblings_list, &proc->children_list);
-	child->parent = proc;
-	mcs_rwlock_writer_unlock_noirq(&proc->children_lock, &childlock);
+		mcs_rwlock_writer_lock_noirq(&proc->children_lock, &childlock);
+		list_add_tail(&child->siblings_list, &proc->children_list);
+		child->parent = proc;
+		mcs_rwlock_writer_unlock_noirq(&proc->children_lock, &childlock);
+	}
 
 	child->ptrace = PT_TRACED | PT_TRACE_EXEC;
 
@@ -5227,7 +5388,7 @@ static int ptrace_attach(int pid)
 	info.si_signo = SIGSTOP;
 	info.si_code = SI_USER;
 	info._sifields._kill.si_pid = proc->pid;
-	error = do_kill(mythread, pid, -1, SIGSTOP, &info, 2);
+	error = do_kill(mythread, -1, pid, SIGSTOP, &info, 2);
 	if (error < 0) {
 		goto out;
 	}
@@ -5539,8 +5700,8 @@ SYSCALL_DECLARE(sched_setparam)
 	struct sched_param param;
 	struct thread *thread = cpu_local_var(current);
 	struct mcs_rwlock_node_irqsave lock;
-	
 	struct syscall_request request1 IHK_DMA_ALIGN;
+	int other_thread = 0;
 
 	dkprintf("sched_setparam: pid: %d, uparam: 0x%lx\n", pid, uparam);
 
@@ -5552,12 +5713,11 @@ SYSCALL_DECLARE(sched_setparam)
 		pid = thread->proc->pid;
 
 	if (thread->proc->pid != pid) {
+		other_thread = 1;
 		thread = find_thread(pid, pid, &lock);
 		if (!thread) {
 			return -ESRCH;
 		}
-		// TODO: unlock 場所のチェック
-		// 何をしようとしているのか理解
 		thread_unlock(thread, &lock);
 		
 		/* Ask Linux about ownership.. */
@@ -5576,7 +5736,17 @@ SYSCALL_DECLARE(sched_setparam)
 		return -EFAULT;
 	}
 
-	return setscheduler(thread, thread->sched_policy, &param);
+	if (other_thread) {
+		thread = find_thread(pid, pid, &lock);
+		if (!thread) {
+			return -ESRCH;
+		}
+	}
+	retval = setscheduler(thread, thread->sched_policy, &param);
+	if (other_thread) {
+		thread_unlock(thread, &lock);
+	}
+	return retval;
 }
 
 SYSCALL_DECLARE(sched_getparam)
@@ -5856,7 +6026,7 @@ SYSCALL_DECLARE(sched_getaffinity)
 	int ret;
 
 	dkprintf("%s() len: %d, mask: %p\n", __FUNCTION__, len, u_cpu_set);
-	if (!len)
+	if (!len || u_cpu_set == (cpu_set_t *)-1)
 		return -EINVAL;
 
 	if ((len * BITS_PER_BYTE) < __CPU_SETSIZE)
@@ -6279,7 +6449,23 @@ SYSCALL_DECLARE(nanosleep)
 
 SYSCALL_DECLARE(sched_yield)
 {
-	schedule();
+	struct cpu_local_var *v;
+	int do_schedule = 0;
+	long runq_irqstate;
+
+	runq_irqstate =
+		ihk_mc_spinlock_lock(&(get_this_cpu_local_var()->runq_lock));
+	v = get_this_cpu_local_var();
+
+	if (v->flags & CPU_FLAG_NEED_RESCHED || v->runq_len > 1) {
+		do_schedule = 1;
+	}
+
+	ihk_mc_spinlock_unlock(&v->runq_lock, runq_irqstate);
+
+	if (do_schedule) {
+		schedule();
+	}
 
 	return 0;
 }
@@ -7066,7 +7252,7 @@ SYSCALL_DECLARE(getcpu)
 	const uintptr_t cpup = ihk_mc_syscall_arg0(ctx);
 	const uintptr_t nodep = ihk_mc_syscall_arg1(ctx);
 	const int cpu = ihk_mc_get_processor_id();
-	const int node = 0;
+	const int node = ihk_mc_get_numa_id();
 	int error;
 
 	if (cpup) {
@@ -8224,6 +8410,9 @@ set_cputime(int mode)
 long syscall(int num, ihk_mc_user_context_t *ctx)
 {
 	long l;
+#ifdef TRACK_SYSCALLS
+	uint64_t t_s;
+#endif // TRACK_SYSCALLS
 
 	set_cputime(1);
 	if(cpu_local_var(current)->proc->status == PS_EXITED &&
@@ -8265,6 +8454,9 @@ long syscall(int num, ihk_mc_user_context_t *ctx)
 #endif
     dkprintf("\n");
 
+#ifdef TRACK_SYSCALLS
+	t_s = rdtsc();
+#endif // TRACK_SYSCALLS
 
 	if ((0 <= num) && (num < (sizeof(syscall_table) / sizeof(syscall_table[0])))
 			&& (syscall_table[num] != NULL)) {
@@ -8281,8 +8473,31 @@ long syscall(int num, ihk_mc_user_context_t *ctx)
 		l = syscall_generic_forwarding(num, ctx);
 	}
 
-	check_signal(l, NULL, num);
-	check_need_resched();
+	if (num != __NR_sched_yield &&
+			num != __NR_futex) {
+		check_signal(l, NULL, num);
+	}
+
+#ifdef TRACK_SYSCALLS
+	if (num < 300) {
+		if (!cpu_local_var(current)->syscall_cnts) {
+			alloc_syscall_counters(cpu_local_var(current));
+		}
+		if (cpu_local_var(current)->socc_enabled) {
+			cpu_local_var(current)->syscall_times[num] += (rdtsc() - t_s);
+			cpu_local_var(current)->syscall_cnts[num]++;
+		}
+	}
+	else {
+		if (num != 701)
+			kprintf("syscall > 300?? : %d\n", num);
+	}
+#endif // TRACK_SYSCALLS
+
+	if (num != __NR_sched_yield &&
+			num != __NR_futex) {
+		check_need_resched();
+	}
 
 	if (cpu_local_var(current)->proc->ptrace) {
 		ptrace_syscall_exit(cpu_local_var(current));