diff --git a/arch/arm64/kernel/include/arch-lock.h b/arch/arm64/kernel/include/arch-lock.h index 0086671f..a9f7c339 100644 --- a/arch/arm64/kernel/include/arch-lock.h +++ b/arch/arm64/kernel/include/arch-lock.h @@ -659,5 +659,102 @@ static inline int irqflags_can_interrupt(unsigned long flags) } #endif /* CONFIG_HAS_NMI */ +struct ihk_rwlock { + unsigned int lock; +}; +static inline void ihk_mc_rwlock_init(struct ihk_rwlock *rw) +{ + rw->lock = 0; +} + +static inline void ihk_mc_read_lock(struct ihk_rwlock *rw) +{ + unsigned int tmp, tmp2; + + asm volatile( + " sevl\n" + "1: wfe\n" + "2: ldaxr %w0, %2\n" + " add %w0, %w0, #1\n" + " tbnz %w0, #31, 1b\n" + " stxr %w1, %w0, %2\n" + " cbnz %w1, 2b\n" + : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) + : + : "cc", "memory"); +} + +static inline int ihk_mc_read_trylock(struct ihk_rwlock *rw) +{ + unsigned int tmp, tmp2 = 1; + + asm volatile( + " ldaxr %w0, %2\n" + " add %w0, %w0, #1\n" + " tbnz %w0, #31, 1f\n" + " stxr %w1, %w0, %2\n" + "1:\n" + : "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock) + : + : "cc", "memory"); + + return !tmp2; +} + +static inline void ihk_mc_read_unlock(struct ihk_rwlock *rw) +{ + unsigned int tmp, tmp2; + + asm volatile( + "1: ldxr %w0, %2\n" + " sub %w0, %w0, #1\n" + " stlxr %w1, %w0, %2\n" + " cbnz %w1, 1b\n" + : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) + : + : "cc", "memory"); +} + +static inline void ihk_mc_write_lock(struct ihk_rwlock *rw) +{ + unsigned int tmp; + + asm volatile( + " sevl\n" + "1: wfe\n" + "2: ldaxr %w0, %1\n" + " cbnz %w0, 1b\n" + " stxr %w0, %w2, %1\n" + " cbnz %w0, 2b\n" + : "=&r" (tmp), "+Q" (rw->lock) + : "r" (0x80000000) + : "cc", "memory"); +} + +static inline int ihk_mc_write_trylock(struct ihk_rwlock *rw) +{ + unsigned int tmp; + + asm volatile( + " ldaxr %w0, %1\n" + " cbnz %w0, 1f\n" + " stxr %w0, %w2, %1\n" + "1:\n" + : "=&r" (tmp), "+Q" (rw->lock) + : "r" (0x80000000) + : "cc", "memory"); + + return !tmp; +} + +static inline void ihk_mc_write_unlock(struct ihk_rwlock *rw) +{ + asm volatile( + " stlr %w1, %0\n" + : "=Q" (rw->lock) : "r" (0) : "memory"); +} + +#define ihk_mc_read_can_lock(rw) ((rw)->lock < 0x80000000) +#define ihk_mc_write_can_lock(rw) ((rw)->lock == 0) #endif /* !__HEADER_ARM64_COMMON_ARCH_LOCK_H */ diff --git a/arch/x86_64/kernel/include/arch-lock.h b/arch/x86_64/kernel/include/arch-lock.h index dbc93ab0..6159c7de 100644 --- a/arch/x86_64/kernel/include/arch-lock.h +++ b/arch/x86_64/kernel/include/arch-lock.h @@ -592,4 +592,90 @@ static inline int irqflags_can_interrupt(unsigned long flags) return !!(flags & 0x200); } +struct ihk_rwlock { + union { + long lock; + struct { + unsigned int read; + int write; + }; + } lock; +}; + +static inline void ihk_mc_rwlock_init(struct ihk_rwlock *rw) +{ + rw->lock.read = 0; + rw->lock.write = 1; +} + +static inline void ihk_mc_read_lock(struct ihk_rwlock *rw) +{ + asm volatile("1:\t" + "lock; decq %0\n\t" + "jns 3f\n\t" + "lock incq %0\n\t" + "2:\t" + "pause\n\t" + "cmpq $0x1, %0\n\t" + "jns 1b\n\t" + "jmp 2b\n\t" + "3:" + : "+m" (rw->lock.lock) : : "memory"); +} + +static inline void ihk_mc_write_lock(struct ihk_rwlock *rw) +{ + asm volatile("1:\t" + "lock; decl %0\n\t" + "je 3f\n\t" + "lock; incl %0\n\t" + "2:\t" + "pause\n\t" + "cmpl $0x1,%0\n\t" + "je 1b\n\t" + "jmp 2b\n\t" + "3:" + : "+m" (rw->lock.write) : "i" (((1L) << 32)) : "memory"); +} + +static inline int ihk_mc_read_trylock(struct ihk_rwlock *rw) +{ + ihk_atomic64_t *count = (ihk_atomic64_t *)rw; + + if (ihk_atomic64_sub_return(1, count) >= 0) + return 1; + ihk_atomic64_inc(count); + return 0; +} + +static inline int ihk_mc_write_trylock(struct ihk_rwlock *rw) +{ + ihk_atomic_t *count = (ihk_atomic_t *)&rw->lock.write; + + if (ihk_atomic_dec_and_test(count)) + return 1; + ihk_atomic_inc(count); + return 0; +} + +static inline void ihk_mc_read_unlock(struct ihk_rwlock *rw) +{ + asm volatile("lock; incq %0" : "+m" (rw->lock.lock) : : "memory"); +} + +static inline void ihk_mc_write_unlock(struct ihk_rwlock *rw) +{ + asm volatile("lock; incl %0" + : "+m" (rw->lock.write) : "i" (((1L) << 32)) : "memory"); +} + +static inline int ihk_mc_write_can_lock(struct ihk_rwlock *rw) +{ + return rw->lock.write == 1; +} + +static inline int ihk_mc_read_can_lock(struct ihk_rwlock *rw) +{ + return rw->lock.lock > 0; +} #endif diff --git a/arch/x86_64/kernel/include/ihk/atomic.h b/arch/x86_64/kernel/include/ihk/atomic.h index 14f7a52e..4fa2773d 100644 --- a/arch/x86_64/kernel/include/ihk/atomic.h +++ b/arch/x86_64/kernel/include/ihk/atomic.h @@ -126,6 +126,22 @@ static inline void ihk_atomic64_inc(ihk_atomic64_t *v) asm volatile ("lock incq %0" : "+m"(v->counter64)); } +static inline long ihk_atomic64_add_return(long i, ihk_atomic64_t *v) +{ + long __i; + + __i = i; + asm volatile("lock xaddq %0, %1" + : "+r" (i), "+m" (v->counter64) + : : "memory"); + return i + __i; +} + +static inline long ihk_atomic64_sub_return(long i, ihk_atomic64_t *v) +{ + return ihk_atomic64_add_return(-i, v); +} + /*********************************************************************** * others */ diff --git a/test/issues/1323/Makefile b/test/issues/1323/Makefile new file mode 100644 index 00000000..312d4ff0 --- /dev/null +++ b/test/issues/1323/Makefile @@ -0,0 +1,13 @@ +CC = gcc +TARGET = rwlock + +all:: $(TARGET) + +rwlock: rwlock.c + $(CC) -g -Wall -o $@ $^ -lpthread + +test:: all + sh ./rwlock.sh + +clean:: + rm -f $(TARGET) *.o diff --git a/test/issues/1323/README b/test/issues/1323/README new file mode 100644 index 00000000..54cce8e1 --- /dev/null +++ b/test/issues/1323/README @@ -0,0 +1,19 @@ +【read/write lock 動作確認】 +□ テスト内容 +rwlock test 1: write_lock のテストを行う +rwlock test 2: write_trylock のテストを行う +rwlock test 3: write_lock と read_lock のテストを行う +rwlock test 4: write_trylock と read_trylock のテストを行う + +□ 実行手順 +$ make test + +McKernelのインストール先は、$HOME/.mck_test_config を +参照する。.mck_test_config は、McKernel をビルドした際に生成される +mck_test_config.sample ファイルを $HOME にコピーし、適宜編集すること。 + +尚、テスト実行には rwlock.patch を適用した McKernel を使用すること。 + +□ 実行結果 +rwlock.txt 参照。 +全ての項目が PASS していることを確認。 diff --git a/test/issues/1323/rwlock.c b/test/issues/1323/rwlock.c new file mode 100644 index 00000000..1f95cace --- /dev/null +++ b/test/issues/1323/rwlock.c @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include +#include +#include + +int cmd; +int procs; +int rc; + +void * +rwtest(void *arg) +{ + int r = syscall(750, cmd, procs); + + if (r) + rc = 1; + + return NULL; +} + +int +main(int argc, char **argv) +{ + pthread_t *threads; + int i; + + if (!argv[1]) { + long val = syscall(750, 10); + int running = syscall(750, 11); + long lockval = syscall(750, 12); + + fprintf(stderr, "%ld %d %016lx\n", val, running, lockval); + exit(0); + } + cmd = atoi(argv[1]); + if (cmd < 1 || cmd > 4) { + fprintf(stderr, "invalid test ID (%s)\n", argv[1]); + exit(1); + } + if (!argv[2]) { + fprintf(stderr, "no procs present\n"); + exit(1); + } + procs = atoi(argv[2]); + if (procs < 1) { + fprintf(stderr, "invalid procs (%s)\n", argv[2]); + exit(1); + } + if (syscall(750, 0) == -1) { + fprintf(stderr, "invalid test environment\n"); + exit(1); + } + threads = malloc(sizeof(pthread_t) * procs); + for (i = 0; i < procs; i++) { + if (pthread_create(threads + i, NULL, rwtest, NULL)) { + fprintf(stderr, "pthread_create: %s\n", + strerror(errno)); + exit(1); + } + } + rc = 0; + for (i = 0; i < procs; i++) { + pthread_join(threads[i], NULL); + } + if (rc) { + fprintf(stderr, "rwlock test %d FAIL\n", cmd); + exit(1); + } + fprintf(stderr, "rwlock test %d PASS\n", cmd); + exit(0); +} diff --git a/test/issues/1323/rwlock.patch b/test/issues/1323/rwlock.patch new file mode 100644 index 00000000..b0ff9097 --- /dev/null +++ b/test/issues/1323/rwlock.patch @@ -0,0 +1,239 @@ +diff --git a/arch/arm64/kernel/include/syscall_list.h b/arch/arm64/kernel/include/syscall_list.h +index f911674..fe089fc 100644 +--- a/arch/arm64/kernel/include/syscall_list.h ++++ b/arch/arm64/kernel/include/syscall_list.h +@@ -134,6 +134,8 @@ SYSCALL_HANDLED(731, util_indicate_clone) + SYSCALL_HANDLED(732, get_system) + SYSCALL_HANDLED(733, util_register_desc) + ++SYSCALL_HANDLED(750, rwlock_test) ++ + /* McKernel Specific */ + SYSCALL_HANDLED(801, swapout) + SYSCALL_HANDLED(802, linux_mlock) +diff --git a/arch/x86_64/kernel/include/syscall_list.h b/arch/x86_64/kernel/include/syscall_list.h +index 79eda7f..4fac75c 100644 +--- a/arch/x86_64/kernel/include/syscall_list.h ++++ b/arch/x86_64/kernel/include/syscall_list.h +@@ -174,6 +174,8 @@ SYSCALL_HANDLED(731, util_indicate_clone) + SYSCALL_HANDLED(732, get_system) + SYSCALL_HANDLED(733, util_register_desc) + ++SYSCALL_HANDLED(750, rwlock_test) ++ + /* McKernel Specific */ + SYSCALL_HANDLED(801, swapout) + SYSCALL_HANDLED(802, linux_mlock) +diff --git a/kernel/syscall.c b/kernel/syscall.c +index 06d3d48..acdd702 100644 +--- a/kernel/syscall.c ++++ b/kernel/syscall.c +@@ -9482,6 +9482,208 @@ SYSCALL_DECLARE(util_register_desc) + return 0; + } + ++SYSCALL_DECLARE(rwlock_test) ++{ ++ int cmd = (int)ihk_mc_syscall_arg0(ctx); ++ int procs = (int)ihk_mc_syscall_arg1(ctx); ++ static ihk_atomic_t barrier; ++ static int counter; ++ static struct ihk_rwlock lock; ++ int i; ++ int bsp; ++ union { ++ int rcint; ++ long rclong; ++ } retval; ++ ++ switch(cmd) { ++ case 0: ++ ihk_mc_rwlock_init(&lock); ++ return 0; ++ ++ case 1: ++ bsp = ihk_atomic_inc_return(&barrier) == 1; ++ if (bsp) { ++ kprintf("rwlock_test 1 start\n"); ++ counter = 0; ++ } ++ while (ihk_atomic_read(&barrier) != procs) { ++ cpu_pause(); ++ } ++ ++ for (i = 0; i < 100000; i++) { ++ ihk_mc_write_lock(&lock); ++ counter++; ++ ihk_mc_write_unlock(&lock); ++ } ++ ihk_atomic_dec(&barrier); ++ while (ihk_atomic_read(&barrier) != 0) { ++ cpu_pause(); ++ } ++ ++ if (bsp) { ++ if (counter == 100000 * procs) { ++ kprintf("rwlock_test 1 OK\n"); ++ } ++ else { ++ kprintf("rwlock_test 1 NG %d != %d\n", ++ ihk_atomic_read(&barrier), ++ 100000 * procs); ++ } ++ } ++ break; ++ ++ case 2: ++ bsp = ihk_atomic_inc_return(&barrier) == 1; ++ if (bsp) { ++ kprintf("rwlock_test 2 start\n"); ++ counter = 0; ++ } ++ while (ihk_atomic_read(&barrier) != procs) { ++ cpu_pause(); ++ } ++ ++ for (i = 0; i < 100000; i++) { ++ while (!ihk_mc_write_trylock(&lock)) { ++ while (!ihk_mc_write_can_lock(&lock)) { ++ cpu_pause(); ++ } ++ } ++ counter++; ++ ihk_mc_write_unlock(&lock); ++ } ++ ihk_atomic_dec(&barrier); ++ while (ihk_atomic_read(&barrier) != 0) { ++ cpu_pause(); ++ } ++ ++ if (bsp) { ++ if (counter == 100000 * procs) { ++ kprintf("rwlock_test 2 OK\n"); ++ } ++ else { ++ kprintf("rwlock_test 2 NG %d != %d\n", ++ ihk_atomic_read(&barrier), ++ 100000 * procs); ++ } ++ } ++ break; ++ ++ case 3: ++ bsp = ihk_atomic_inc_return(&barrier) == 1; ++ if (bsp) { ++ kprintf("rwlock_test 3 start\n"); ++ counter = 0; ++ } ++ while (ihk_atomic_read(&barrier) != procs) { ++ cpu_pause(); ++ } ++ ++ for (i = 0; i < 100000; i++) { ++ int tmp; ++ ihk_mc_write_lock(&lock); ++ counter++; ++ ihk_mc_write_unlock(&lock); ++ ihk_mc_read_lock(&lock); ++ tmp = counter; ++ ihk_mc_read_unlock(&lock); ++ if (tmp >= 100000 * procs) { ++ kprintf("rwlock_test 3 break OK\n"); ++ break; ++ } ++ } ++ ihk_atomic_dec(&barrier); ++ while (ihk_atomic_read(&barrier) != 0) { ++ cpu_pause(); ++ } ++ ++ if (bsp) { ++ if (counter == 100000 * procs) { ++ kprintf("rwlock_test 3 OK\n"); ++ } ++ else { ++ kprintf("rwlock_test 3 NG %d != %d\n", ++ ihk_atomic_read(&barrier), ++ 100000 * procs); ++ } ++ } ++ break; ++ ++ case 4: ++ bsp = ihk_atomic_inc_return(&barrier) == 1; ++ if (bsp) { ++ kprintf("rwlock_test 4 start\n"); ++ counter = 0; ++ } ++ while (ihk_atomic_read(&barrier) != procs) { ++ cpu_pause(); ++ } ++ ++ for (i = 0; i < 100000; i++) { ++ int brk = 0; ++ ++ for (;;) { ++ int tmp; ++ ++ if (ihk_mc_write_trylock(&lock)) { ++ counter++; ++ ihk_mc_write_unlock(&lock); ++ brk = 1; ++ } ++ if (ihk_mc_read_trylock(&lock)) { ++ tmp = counter; ++ ihk_mc_read_unlock(&lock); ++ if (tmp >= 100000 * procs) { ++ kprintf("rwlock_test 4 break OK\n"); ++ break; ++ } ++ } ++ if (brk) { ++ break; ++ } ++ while (!ihk_mc_write_can_lock(&lock) && ++ !ihk_mc_read_can_lock(&lock)) { ++ cpu_pause(); ++ } ++ } ++ } ++ ihk_atomic_dec(&barrier); ++ while (ihk_atomic_read(&barrier) != 0) { ++ cpu_pause(); ++ } ++ ++ if (bsp) { ++ if (counter == 100000 * procs) { ++ kprintf("rwlock_test 4 OK\n"); ++ } ++ else { ++ kprintf("rwlock_test 4 NG %d != %d\n", ++ ihk_atomic_read(&barrier), ++ 100000 * procs); ++ } ++ } ++ break; ++ ++ case 10: ++ return counter; ++ ++ case 11: ++ return ihk_atomic_read(&barrier); ++ ++ case 12: ++ memcpy(&retval, &lock, sizeof(struct ihk_rwlock)); ++ if (sizeof(struct ihk_rwlock) == 8) { ++ return retval.rclong; ++ } ++ return retval.rcint; ++ ++ default: ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ + void + reset_cputime() + { diff --git a/test/issues/1323/rwlock.sh b/test/issues/1323/rwlock.sh new file mode 100644 index 00000000..6bd242ce --- /dev/null +++ b/test/issues/1323/rwlock.sh @@ -0,0 +1,12 @@ +#!/bin/sh +USELTP=0 +USEOSTEST=0 +BOOTPARAM="-c 2-7,9-15 -m 1G@0" + +. ../../common.sh + +################################################################################ +$MCEXEC ./rwlock 1 10 +$MCEXEC ./rwlock 2 10 +$MCEXEC ./rwlock 3 10 +$MCEXEC ./rwlock 4 10 diff --git a/test/issues/1323/rwlock.txt b/test/issues/1323/rwlock.txt new file mode 100644 index 00000000..1101c1b1 --- /dev/null +++ b/test/issues/1323/rwlock.txt @@ -0,0 +1,14 @@ +Script started on Wed Jul 24 10:55:01 2019 +bash-4.2$ make test +gcc -g -Wall -o rwlock rwlock.c -lpthread +sh ./rwlock.sh +mcstop+release.sh ... done +mcreboot.sh -c 2-7,9-15 -m 1G@0 ... done +rwlock test 1 PASS +rwlock test 2 PASS +rwlock test 3 PASS +rwlock test 4 PASS +bash-4.2$ exit +exit + +Script done on Wed Jul 24 10:55:20 2019