gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update test programs for qlmpi (do swap with using shared memory, ib_pingpong)

Browse Source
This commit is contained in:
Ken Sato
2017-09-25 16:56:52 +09:00
parent dae9a5ff13
commit 58c1fd4512
16 changed files with 569 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
test/qlmpi/rusage/Makefile
View File

@@ -29,7 +29,6 @@ rusage010: rusage010.o
rusage010.o: rusage010.c rusage010.o: rusage010.c
$(CC) $(CCFLAGSMCK) $(CPPFLAGSMCK) -c $< $(CC) $(CCFLAGSMCK) $(CPPFLAGSMCK) -c $<
rusage008: rusage008.o rusage008: rusage008.o
$(CC) -o $@ $^ $(LDFLAGSMCK) $(CC) -o $@ $^ $(LDFLAGSMCK)
@@ -48,6 +47,17 @@ rusage011: rusage011.o
rusage011.o: rusage011.c rusage011.o: rusage011.c
$(CC) $(CCFLAGSMCK) $(CPPFLAGSMCK) -c $< $(CC) $(CCFLAGSMCK) $(CPPFLAGSMCK) -c $<
rusage002: rusage002.o
$(CC) -o $@ $^ $(LDFLAGSMCK) -lrt
rusage002.o: rusage002.c
$(CC) $(CCFLAGSMCK) $(CPPFLAGSMCK) -c $<
rusage003: rusage003.o
$(CC) -o $@ $^ $(LDFLAGSMCK) -lrt
rusage003.o: rusage003.c
$(CC) $(CCFLAGSMCK) $(CPPFLAGSMCK) -c $<
clean: clean:
rm -f core $(EXES) $(OBJS) $(EXESMCK) $(OBJSMCK) rm -f core $(EXES) $(OBJS) $(EXESMCK) $(OBJSMCK)

16
test/qlmpi/rusage/README
View File

@@ -2,14 +2,15 @@
Test matrix Test matrix
=========== ===========
rusage002:
do_mmap(),/dev/shm with --mpol_shm_premap,pre-page->get_page(),st->set_pte()->munmap()->clear_range() [OK]
rusage003:
do_mmap(),/dev/shm without --mpol_shm_premap,pre-page->get_page(),st->set_pte()->munmap()->clear_range() [OK]
rusage010: rusage010:
app->mmap() 2M,anon,pre-page ->set_range()->munmap()->free_process_memory_range()->clear_range()[OK] app->mmap() 2M,anon,pre-page ->set_range()->munmap()->free_process_memory_range()->clear_range()[OK]
rusage005: device file (ib ping-pong)
devobj()->get_page()->pf->munmap()->clear_range() [OK]
remote page fault->cow->clear_range() [OK]
ld-linux.so->mmap private->cow->clear_range() [OK]
rusage008: sharing file-map page rusage008: sharing file-map page
fork()->filemap->pf->clear_range() [OK] fork()->filemap->pf->clear_range() [OK]
@@ -19,3 +20,8 @@ fork()->shmat()->pf->clear_range() [OK]
rusage011: sharing xpmem page rusage011: sharing xpmem page
fork()->xpmem_attach()->pf->clear_range() [OK] fork()->xpmem_attach()->pf->clear_range() [OK]
device file (ib ping-pong) in verbs directory
devobj()->get_page()->pf->munmap()->clear_range() [OK]
remote page fault->cow->clear_range() [OK]
ld-linux.so->mmap private->cow->clear_range() [OK]

14
test/qlmpi/rusage/enable_swap.patch Normal file
View File

@@ -0,0 +1,14 @@
diff --git a/executer/user/qlmpilib.c b/executer/user/qlmpilib.c
index 7fcbcb5..9058ec8 100644
--- a/executer/user/qlmpilib.c
+++ b/executer/user/qlmpilib.c
@@ -158,7 +158,8 @@ int ql_client(int *argc,char ***argv)
syscall(803);
rc = PMI_Barrier();
- rc = swapout(swap_file, buffer, BUF_SIZE, 1);
+ //rc = swapout(swap_file, buffer, BUF_SIZE, 1);
+ rc = swapout(swap_file, buffer, BUF_SIZE, 0);
#ifdef QL_DEBUG
printf(" swapout rc=%d\n",rc);

5
test/qlmpi/rusage/run.sh
View File

@@ -31,6 +31,11 @@ if [ "${pid}" != "" ]; then
fi fi
case ${testname} in case ${testname} in
rusage002)
mcexecopt="--mpol-shm-premap"
;;
rusage003)
;;
rusage010) rusage010)
testopt="1" testopt="1"
;; ;;

99
test/qlmpi/rusage/rusage002.c Normal file
View File

@@ -0,0 +1,99 @@
#include <unistd.h>
#include <stdio.h>
#include <sys/mman.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include "qltest.h"
#define DEBUG
#ifdef DEBUG
#define dprintf(...) \
do { \
char msg[1024]; \
sprintf(msg, __VA_ARGS__); \
fprintf(stderr, "%s,%s", __FUNCTION__, msg); \
} while (0);
#define eprintf(...) \
do { \
char msg[1024]; \
sprintf(msg, __VA_ARGS__); \
fprintf(stderr, "%s,%s", __FUNCTION__, msg); \
} while (0);
#else
#define dprintf(...) do { } while (0)
#define eprintf(...) do { } while (0)
#endif
#define CHKANDJUMP(cond, err, ...) \
do { \
if(cond) { \
eprintf(__VA_ARGS__); \
ret = err; \
goto fn_fail; \
} \
} while(0)
int sz_mem[] = {
4 * (1ULL<<10),
2 * (1ULL<<20),
1 * (1ULL<<30),
134217728};
#define SZ_INDEX 0
int main(int argc, char** argv) {
void* mem;
int ret = 0;
int fd;
char fn[256] = "/dev/shm/Intel_MPI";
#define TEST_VAL 0x1234
int swap_rc = 0;
char buffer[BUF_SIZE];
fd = open(fn, O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR);
CHKANDJUMP(fd == -1, 255, "shm_open failed,str=%s\n", strerror(errno));
ret = ftruncate(fd, sz_mem[SZ_INDEX]);
CHKANDJUMP(ret != 0, 255, "ftruncate failed\n");
mem = mmap(0, sz_mem[SZ_INDEX], PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
CHKANDJUMP(mem == MAP_FAILED, 255, "mmap failed\n");
memset(mem, 0, sz_mem[SZ_INDEX]);
// before swap
*((unsigned long*)mem) = TEST_VAL;
unsigned long val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] before swap, val:0x%lx\n", val);
} else {
printf("[NG] before swap, val is not correct, val:0x%lx\n", val);
}
swap_rc = do_swap("/tmp/rusage002.swp", buffer);
if (swap_rc < 0) {
printf("[NG] swap is failed\n");
}
// after swap
val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] after swap, val:0x%lx\n", val);
} else {
printf("[NG] after swap, val is not correct, val:0x%lx\n", val);
}
munmap(mem, sz_mem[SZ_INDEX]);
ret = close(fd);
CHKANDJUMP(ret != 0, 255, "close failed\n");
ret = unlink(fn);
CHKANDJUMP(ret != 0, 255, "shm_unlink failed\n");
fn_exit:
return ret;
fn_fail:
goto fn_exit;
}

1
test/qlmpi/rusage/rusage003.c Symbolic link
View File

@@ -0,0 +1 @@
rusage002.c

28
test/qlmpi/rusage/rusage008.c
View File

@@ -54,6 +54,7 @@ int main(int argc, char** argv) {
int status; int status;
int fd; int fd;
// for swap_test // for swap_test
#define TEST_VAL 0x1234
int swap_rc = 0; int swap_rc = 0;
char buffer[BUF_SIZE]; char buffer[BUF_SIZE];
@@ -65,14 +66,16 @@ int main(int argc, char** argv) {
mem = mmap(0, sz_mem[SZ_INDEX], PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); mem = mmap(0, sz_mem[SZ_INDEX], PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
CHKANDJUMP(mem == MAP_FAILED, 255, "mmap failed\n"); CHKANDJUMP(mem == MAP_FAILED, 255, "mmap failed\n");
memset(mem, 0, sz_mem[SZ_INDEX]);
unsigned long val = *((unsigned long*)mem);
// for swap_test // for swap_test
swap_rc = do_swap("/tmp/rusage008_c.swp", buffer); swap_rc = do_swap("/tmp/rusage008_c.swp", buffer);
if (swap_rc < 0) { if (swap_rc < 0) {
printf("[NG] swap in child is failed\n"); printf("[NG] swap in child is failed\n");
} }
*((unsigned long*)mem) = TEST_VAL;
_exit(123); _exit(123);
} else { } else {
fd = open("./file", O_RDWR); fd = open("./file", O_RDWR);
@@ -81,16 +84,33 @@ int main(int argc, char** argv) {
mem = mmap(0, sz_mem[SZ_INDEX], PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); mem = mmap(0, sz_mem[SZ_INDEX], PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
CHKANDJUMP(mem == MAP_FAILED, 255, "mmap failed\n"); CHKANDJUMP(mem == MAP_FAILED, 255, "mmap failed\n");
unsigned long val = *((unsigned long*)mem);
ret = waitpid(pid, &status, 0); ret = waitpid(pid, &status, 0);
CHKANDJUMP(ret == -1, 255, "waitpid failed\n"); CHKANDJUMP(ret == -1, 255, "waitpid failed\n");
printf("child exit status=%d\n", WEXITSTATUS(status));
// for swap_test // for swap_test
// before swap
unsigned long val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] before swap, val:0x%lx\n", val);
} else {
printf("[NG] before swap, val is not 0x%lx, val is 0x%lx\n", TEST_VAL, val);
}
swap_rc = do_swap("/tmp/rusage008_p.swp", buffer); swap_rc = do_swap("/tmp/rusage008_p.swp", buffer);
if (swap_rc < 0) { if (swap_rc < 0) {
printf("[NG] swap in parent is failed\n"); printf("[NG] swap in parent is failed\n");
} }
printf("exit status=%d\n", WEXITSTATUS(status));
// after swap
val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] after swap, val:0x%lx\n", val);
} else {
printf("[NG] after swap, val is not 0x%lx, val is 0x%lx\n", TEST_VAL, val);
}
} }
fn_exit: fn_exit:

20
test/qlmpi/rusage/rusage009.c
View File

@@ -56,6 +56,7 @@ int main(int argc, char** argv) {
key_t key = ftok(argv[0], 0); key_t key = ftok(argv[0], 0);
int shmid; int shmid;
// for swap_test // for swap_test
#define TEST_VAL 0x1234
int swap_rc = 0; int swap_rc = 0;
char buffer[BUF_SIZE]; char buffer[BUF_SIZE];
@@ -67,13 +68,14 @@ int main(int argc, char** argv) {
if(pid == 0) { if(pid == 0) {
mem = shmat(shmid, NULL, 0); mem = shmat(shmid, NULL, 0);
CHKANDJUMP(mem == (void*)-1, 255, "shmat failed: %s\n", strerror(errno)); CHKANDJUMP(mem == (void*)-1, 255, "shmat failed: %s\n", strerror(errno));
memset(mem, 0, sz_mem[SZ_INDEX]);
// for swap_test // for swap_test
swap_rc = do_swap("/tmp/rusage009_c.swp", buffer); swap_rc = do_swap("/tmp/rusage009_c.swp", buffer);
if (swap_rc < 0) { if (swap_rc < 0) {
printf("[NG] swap in child is failed\n"); printf("[NG] swap in child is failed\n");
} }
*((unsigned long*)mem) = 0x1234; *((unsigned long*)mem) = TEST_VAL;
ret = shmdt(mem); ret = shmdt(mem);
CHKANDJUMP(ret == -1, 255, "shmdt failed\n"); CHKANDJUMP(ret == -1, 255, "shmdt failed\n");
@@ -87,12 +89,26 @@ int main(int argc, char** argv) {
CHKANDJUMP(ret == -1, 255, "waitpid failed\n"); CHKANDJUMP(ret == -1, 255, "waitpid failed\n");
// for swap_test // for swap_test
// before swap
unsigned long val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] before swap, val:0x%lx\n", val);
} else {
printf("[NG] before swap, val is not 0x%lx, val is 0x%lx\n", TEST_VAL, val);
}
swap_rc = do_swap("/tmp/rusage009_p.swp", buffer); swap_rc = do_swap("/tmp/rusage009_p.swp", buffer);
if (swap_rc < 0) { if (swap_rc < 0) {
printf("[NG] swap in parent is failed\n"); printf("[NG] swap in parent is failed\n");
} }
printf("%lx\n", *((unsigned long*)mem)); // after swap
val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] after swap, val:0x%lx\n", val);
} else {
printf("[NG] after swap, val is not 0x%lx, val is 0x%lx\n", TEST_VAL, val);
}
#if 0 #if 0
struct shmid_ds buf; struct shmid_ds buf;

22
test/qlmpi/rusage/rusage010.c
View File

@@ -49,6 +49,8 @@ int main(int argc, char** argv) {
void* anon[NUM_AREAS]; void* anon[NUM_AREAS];
int ret = 0; int ret = 0;
// for qlmpi test // for qlmpi test
#define TEST_VAL 0x1234
void* mem;
int swap_rc = 0; int swap_rc = 0;
char buffer[BUF_SIZE]; char buffer[BUF_SIZE];
@@ -61,9 +63,27 @@ int main(int argc, char** argv) {
memset(anon[i], 0, sz_anon[sz_index]); memset(anon[i], 0, sz_anon[sz_index]);
} }
// for qlmpi test // for qlmpi test
// before swap
mem = anon[0];
*((unsigned long*)mem) = TEST_VAL;
unsigned long val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] before swap, val:0x%lx\n", val);
} else {
printf("[NG] before swap, val is not correct, val:0x%lx\n", val);
}
swap_rc = do_swap("/tmp/rusage010.swp", buffer); swap_rc = do_swap("/tmp/rusage010.swp", buffer);
if (swap_rc < 0) { if (swap_rc < 0) {
printf("[NG] swap is failed.\n"); printf("[NG] swap in parent is failed\n");
}
// after swap
val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] after swap, val:0x%lx\n", val);
} else {
printf("[NG] after swap, val is not correct, val:0x%lx\n", val);
} }
for(i = 0; i < NUM_AREAS; i++) { for(i = 0; i < NUM_AREAS; i++) {

22
test/qlmpi/rusage/rusage011.c
View File

@@ -58,6 +58,7 @@ int main(int argc, char** argv) {
int shmid; int shmid;
xpmem_segid_t segid; xpmem_segid_t segid;
// for swap_test // for swap_test
#define TEST_VAL 0x1234
int swap_rc = 0; int swap_rc = 0;
char buffer[BUF_SIZE]; char buffer[BUF_SIZE];
@@ -88,13 +89,14 @@ int main(int argc, char** argv) {
struct xpmem_addr addr = { .apid = apid, .offset = 0 }; struct xpmem_addr addr = { .apid = apid, .offset = 0 };
void* attach = xpmem_attach(addr, sz_mem[SZ_INDEX], NULL); void* attach = xpmem_attach(addr, sz_mem[SZ_INDEX], NULL);
CHKANDJUMP(attach == (void*)-1, 255, "xpmem_attach failed: %s\n", strerror(errno)); CHKANDJUMP(attach == (void*)-1, 255, "xpmem_attach failed: %s\n", strerror(errno));
memset(attach, 0, sz_mem[SZ_INDEX]);
*((unsigned long*)attach) = 0x1234;
// for swap_test // for swap_test
swap_rc = do_swap("/tmp/rusage011_c.swp", buffer); swap_rc = do_swap("/tmp/rusage011_c.swp", buffer);
if (swap_rc < 0) { if (swap_rc < 0) {
printf("[NG] swap in child is failed\n"); printf("[NG] swap in child is failed\n");
} }
*((unsigned long*)attach) = TEST_VAL;
ret = xpmem_detach(attach); ret = xpmem_detach(attach);
CHKANDJUMP(ret == -1, 255, "xpmem_detach failed\n"); CHKANDJUMP(ret == -1, 255, "xpmem_detach failed\n");
@@ -116,12 +118,26 @@ int main(int argc, char** argv) {
CHKANDJUMP(ret == -1, 255, "waitpid failed\n"); CHKANDJUMP(ret == -1, 255, "waitpid failed\n");
// for swap_test // for swap_test
// before swap
unsigned long val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] before swap, val:0x%lx\n", val);
} else {
printf("[NG] before swap, val is not correct, val:0x%lx\n", val);
}
swap_rc = do_swap("/tmp/rusage011_p.swp", buffer); swap_rc = do_swap("/tmp/rusage011_p.swp", buffer);
if (swap_rc < 0) { if (swap_rc < 0) {
printf("[NG] swap in parent is failed\n"); printf("[NG] swap in parent is failed\n");
} }
printf("%lx\n", *((unsigned long*)mem)); // after swap
val = *((unsigned long*)mem);
if (val == TEST_VAL) {
printf("[OK] after swap, val:0x%lx\n", val);
} else {
printf("[NG] after swap, val is not 0x%lx, val is 0x%lx\n", TEST_VAL, val);
}
struct shmid_ds buf; struct shmid_ds buf;
ret = shmctl(shmid, IPC_RMID, &buf); ret = shmctl(shmid, IPC_RMID, &buf);

33
test/qlmpi/rusage/verbs/README Normal file
View File

@@ -0,0 +1,33 @@
=============
Test program for swap in ib_pingpong
=============
Rank 0 process is sender, another is reciever.
Sender send "MagicNumber + LoopCount" to reciever.
Reciever check the number, and print OK or NG.
Then, they swapout and wait for resume.
When they are resumed, increment LoopCount and send/recieve again.
=============
How to run
=============
compile mpi_rdma_wr
$ make
run by ql_mpiexec_start
$ ql_mpiexec_start -machinefile ./mfile ./mpi_rdma_wr -s `hostname` -p 9999
will be printed
[OK] recv_val: 0x55aa55aa
rerun by ql_mpiexec_start
$ ql_mpiexec_start -machinefile ./mfile ./mpi_rdma_wr -s `hostname` -p 9999
if swap is disable, it will be printed
[OK] recv_val: 0x55aa55ab
else (swap is enable)
[NG] recv_val is not correct, expected:0x55aa55ab recv:0x55aa55aa
or
occur abort
finalize by ql_mpiexec_finalize
$ ql_mpiexec_finalize -machinefile ./mfile ./mpi_rdma_wr -s `hostname` -p 9999

19
test/qlmpi/rusage/verbs/makefile
View File

@@ -1,14 +1,19 @@
VPATH = VPATH =
MCK_DIR=/home/satoken/ppos
MPICH_DIR=/usr/lib64/mpich-3.2
CC = icc CC = icc
MPICC = mpicc
CFLAGS = -Wall -O0 -g -DDEBUG -DERROR CFLAGS = -Wall -O0 -g -DDEBUG -DERROR
LD = $(CC) LD = $(CC)
LFLAGS = -libverbs LFLAGS = -libverbs
LDFLAGS = -L$(MPICH_DIR)/lib -L$(MCK_DIR)/lib
SRCS = list.c curtime.c printmem.c debug.c post.c qp.c read_config.c resource.c rdma_wr.c SRCS = list.c curtime.c printmem.c debug.c post.c qp.c read_config.c resource.c mpi_rdma_wr.c
DSRCS = $(SRCS:.c=.d) DSRCS = $(SRCS:.c=.d)
OBJS = $(SRCS:.c=.o) OBJS = $(SRCS:.c=.o)
EXECS = rdma_wr EXECS = mpi_rdma_wr
MODULES = list.o curtime.o printmem.o debug.o post.o qp.o read_config.o resource.o sock.o MODULES = list.o curtime.o printmem.o debug.o post.o qp.o read_config.o resource.o sock.o
CWD := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) CWD := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
@@ -18,11 +23,17 @@ all: $(EXECS)
rdma_wr: rdma_wr.o $(MODULES) rdma_wr: rdma_wr.o $(MODULES)
$(LD) -o $@ $^ $(LFLAGS) $(LD) -o $@ $^ $(LFLAGS)
mpi_rdma_wr: mpi_rdma_wr.o $(MODULES)
$(LD) $(LDFLAGS) -lqlmpi -lmpi -o $@ $^ $(LFLAGS)
mpi_rdma_wr.o: mpi_rdma_wr.c
$(MPICC) -I$(MCK_DIR)/include $(CFLAGS) -c $<
%.o: %.c %.o: %.c
$(CC) $(CFLAGS) -c $< $(MPICC) $(CFLAGS) $(LDFLAGS) -c $<
%.d: %.c %.d: %.c
$(CC) -MM $< > $*.d $(MPICC) $(LDFLAGS) -MM $< > $*.d
clean: clean:
rm -f $(EXECS) $(OBJS) $(DSRCS) rm -f $(EXECS) $(OBJS) $(DSRCS)

1
test/qlmpi/rusage/verbs/mfile Normal file
View File

@@ -0,0 +1 @@
wallaby15:2

295
test/qlmpi/rusage/verbs/mpi_rdma_wr.c Normal file
View File

@@ -0,0 +1,295 @@
#include <stdio.h>
#include <stdlib.h>
#include <memory.h>
#include <sys/mman.h>
#include <unistd.h>
#include "ibcomm.h"
#include "debug.h"
#include "mtype.h"
#include "mcons.h"
#include "mm_ib_test.h"
#include <mpi.h>
#include <qlmpilib.h>
//#define DEBUG_RDMA_WR
#ifdef DEBUG_RDMA_WR
#define dprintf printf
#else
#define dprintf(...)
#endif
#define TEST_SEND_BUF_NUM 3
#define TEST_RDMA_FLG_SIZE (sizeof(unsigned short))
#define NTRIAL 1 /* 120 */
#define PPOLLS 1 /* sweet spot is around 10 */
#define NSKIPS (PPOLLS*0)
#define PPOLLR 1 /* sweet spot is around 10 */
#define NSKIPR (PPOLLR*0)
#define IBCOM_MAGIC 0x55aa55aa
typedef struct tailmagic_t {
uint32_t magic;
} tailmagic_t;
enum rdma_buf_flg{
RDMA_BUF_RESET_FLG = 0,
RDMA_BUF_WRITE_FLG = 1,
};
static unsigned long rdtsc() {
unsigned long x;
__asm__ __volatile__("xorl %%eax, %%eax; cpuid;" : : : "%rax", "%rbx", "%rcx", "%rdx"); /* rdtsc cannot be executed earlier than this */
__asm__ __volatile__("rdtsc; shl $32, %%rdx; or %%rdx, %%rax" : "=a"(x) : : "memory"); /* rdtsc cannot be executed earlier than here */
__asm__ __volatile__("xorl %%eax, %%eax; cpuid;" : : : "%rax", "%rbx", "%rcx", "%rdx"); /* following instructions cannot be executed earlier than this */
return x;
}
volatile int k;
int main(int argc, char **argv) {
config_t config;
unsigned long i, j;
int ibcom_errno = 0;
char sync_res;
unsigned long tscs, tsce;
resource_t res;
pdinfo_t pdinfo;
qpinfo_t qpinfo;
mrinfo_t *loc_mr_list = NULL;
int entry;
int ibv_errno;
int rc, my_rank, loop_cnt = 0;
uint32_t send_val = 0;
if (read_config(&config, argc, argv)) {
goto fn_exit;
}
config.use_rdma = 1;
unsigned long buf_size;
char* str_env = getenv("BUF_SIZE");
buf_size = str_env ? atol(str_env) : 4096/*48,1073741824ULL * 1 + 4*/;
if(buf_size == 0) { printf("set buf_size"); goto fn_fail; }
if(resource_create(config, &res) || pd_create(&res, &pdinfo)) { printf("qp_create failed\n"); goto fn_fail; }
ibv_errno = qp_create(&res, &pdinfo, &qpinfo);
IBCOM_ERR_CHKANDJUMP(ibv_errno, -1, printf("qp_create failed\n"));
/* create MR buffers */
// rdma-write-to buffer
#if 1
void *rdma_buf = mmap(0, buf_size * NTRIAL, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
memset(rdma_buf, 0, buf_size * NTRIAL);
#else
void *rdma_buf = calloc(buf_size * NTRIAL, sizeof(char));
#endif
if(!rdma_buf) { printf("mmap failed\n"); goto fn_fail; }
if(mr_create(&res, &pdinfo, buf_size * NTRIAL, rdma_buf, &res.rdma_mr)) { printf("mr_create failed\n"); goto fn_fail; }
// for mpi
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
// switch server/client by rank
if (my_rank == 0) {
config.server_name = NULL;
config.server_flg = 1;
}
// local data buffers
loc_mr_list = calloc(sizeof(mrinfo_t) * NTRIAL, sizeof(char));
for (i = 0; i < NTRIAL; i++) {
void *loc_buf = mmap(0, buf_size, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
if(loc_buf == MAP_FAILED) { printf("mmap failed\n"); goto fn_fail; }
if(config.server_flg) {
for(j = 0; j < buf_size; j++) {
*((unsigned char*)loc_buf + j) = (char)i;
}
*((uint32_t *)(loc_buf + buf_size - sizeof(uint32_t))) = 0 + IBCOM_MAGIC;
}
dprintf("magic addr=%lx\n", (unsigned long)(loc_buf + buf_size - TEST_RDMA_FLG_SIZE));
if(mr_create(&res, &pdinfo, buf_size, loc_buf, &loc_mr_list[i])) { printf("mr_create failed\n"); goto fn_fail; }
}
if(!config.server_flg) { dprintf("res->rdma_mr.mr->addr=%lx\n", (unsigned long)res.rdma_mr.mr->addr); }
/* exchange gid, lid, qpn, raddr, rkey */
if(connect_qp(config, &res, &qpinfo)) { printf("connect_qp failed\n"); goto fn_fail; }
debug_print_qp_conn_info(res, qpinfo, &config);
printf("connect_qp done\n"); fflush(stdout);
if(config.server_flg) { dprintf("qpinfo->remote_conn_info[0].addr=%lx\n", qpinfo.remote_conn_info[0].addr); }
/* make qp RTS */
if(init_qp(config, &qpinfo) || rtr_qp(config, &qpinfo) || rts_qp(config, &qpinfo)) { printf("rts failed\n"); goto fn_fail; }
printf("rts done\n"); fflush(stdout);
/* barrier */
for(i = 0; i < (config.server_flg ? config.nremote : 1); i++) {
if(sock_sync_data(qpinfo.sock[i], 1, "R", &sync_res)) { perror("sock_sync_data"); }
}
printf("barrier done\n"); fflush(stdout);
ql_loop:
// set send-value
send_val = loop_cnt + IBCOM_MAGIC;
//*((uint32_t *)(loc_mr_list[0].buf + buf_size - sizeof(uint32_t))) = send_val;
*((uint32_t *)(loc_mr_list[0].buf)) = send_val;
// send/recieve
if(config.server_flg) { /* sender side */
//usleep(500000);
if(NTRIAL % PPOLLS != 0) { printf("set NTRIAL multiple of PPOLLS\n"); goto fn_fail; }
if(NTRIAL <= NSKIPS) { printf("set NTRIAL > NSKIP\n"); goto fn_fail; }
for (i = 0; i < NTRIAL; i++) {
if(i == NSKIPS) { tscs = rdtsc(); }
post_send_req2(&qpinfo, &loc_mr_list[0], IBV_WR_RDMA_WRITE, &qpinfo.remote_conn_info[0], 0, i);
}
tsce = rdtsc(); printf("send_time,%.0f\n", (tsce-tscs)/(double)(NTRIAL-NSKIPS));
printf("send_val: %x\n", send_val);
fflush(stdout);
#if 1
int nfound = 0;
k = 0;
while(1) {
int result;
struct ibv_wc cqe[NTRIAL];
result = ibv_poll_cq(qpinfo.scq, NTRIAL, &cqe[0]);
if(result < 0) { printf("ibv_poll_cq"); goto fn_fail; }
if(result > 0) {
for(j = 0; j < result; j++) {
if(cqe[j].status != IBV_WC_SUCCESS) { printf("cqe status,%s\n", ibv_wc_status_str(cqe[j].status)); goto fn_fail; }
}
//debug_print_mem((addr_t)loc_mr_list[entry].buf, buf_size);
nfound += result;
if(nfound >= NTRIAL) { break; }
}
k++;
}
#endif
} else { /* receiver side */
//volatile uint32_t *recv_val = NULL;
if(NSKIPR % PPOLLR !=0) { printf("set NSKIP multiple of PPOLL\n"); goto fn_fail; }
for (i = 0; i < NTRIAL; i++) {
if(i == NSKIPR) { tscs = rdtsc(); }
// poll on magic
dprintf("res.rdma_mr.buf=%lx\n", (unsigned long)res.rdma_mr.buf);
dprintf("poll addr=%lx\n", (unsigned long)(rdma_buf + buf_size * i + buf_size - sizeof(uint32_t)));
//k = 0;
//volatile uint32_t *ptr = (volatile uint32_t *)(rdma_buf + buf_size * i + buf_size - sizeof(uint32_t));
volatile uint32_t *ptr = (volatile uint32_t *)(rdma_buf);
//*recv_val = *(volatile uint32_t *)(rdma_buf + buf_size * i + buf_size - sizeof(uint32_t));
printf("*** ptr:%d, MAGIC:%x\n", *ptr, loop_cnt + IBCOM_MAGIC);
while(*ptr != loop_cnt + IBCOM_MAGIC) {
//printf("++++ send_val:%x, recv_val:%x\n", send_val, *recv_val);
//while(*recv_val != send_val) {
//k++; if(i >= NSKIPR && k % 65536 == 65535) { printf("i=%d,poll value=%x\n", i, *((uint32_t *)(rdma_buf + buf_size * i + buf_size - sizeof(uint32_t)))); }
__asm__ __volatile__("pause");
// TIMEOUT
if (rdtsc() - tscs > 10000000) {
printf("*** recv wait is TIMEOUT\n");
break;
}
}
if (*ptr == loop_cnt + IBCOM_MAGIC) {
printf("[OK] recv_val: 0x%x\n", *ptr);
} else {
printf("[NG] recv_val is not correct, expected:0x%x recv:0x%x\n",
loop_cnt + IBCOM_MAGIC, *ptr);
}
}
tsce = rdtsc(); printf("recv_time,%.0f\n", (tsce-tscs)/(double)(NTRIAL-NSKIPR));
//printf("recv_valu:%x\n", *recv_val);
}
rc = ql_client(&argc, &argv);
if (rc == QL_CONTINUE) {
loop_cnt++;
printf("ql: go back loop\n");
goto ql_loop;
}
else {
printf("ql_client returns %d, go to exit\n", rc);
}
#if 0
// 2nd send/recieve
if(config.server_flg) { /* sender side */
// other value
*((uint32_t *)(loc_mr_list[0].buf + buf_size - sizeof(uint32_t))) = 10 + IBCOM_MAGIC;
//usleep(500000);
if(NTRIAL % PPOLLS != 0) { printf("set NTRIAL multiple of PPOLLS\n"); goto fn_fail; }
if(NTRIAL <= NSKIPS) { printf("set NTRIAL > NSKIP\n"); goto fn_fail; }
for (i = 0; i < NTRIAL; i++) {
if(i == NSKIPS) { tscs = rdtsc(); }
post_send_req2(&qpinfo, &loc_mr_list[0], IBV_WR_RDMA_WRITE, &qpinfo.remote_conn_info[0], 0, i);
}
#if 1
int nfound = 0;
k = 0;
while(1) {
int result;
struct ibv_wc cqe[NTRIAL];
result = ibv_poll_cq(qpinfo.scq, NTRIAL, &cqe[0]);
if(result < 0) { printf("ibv_poll_cq"); goto fn_fail; }
if(result > 0) {
for(j = 0; j < result; j++) {
if(cqe[j].status != IBV_WC_SUCCESS) { printf("cqe status,%s\n", ibv_wc_status_str(cqe[j].status)); goto fn_fail; }
}
//debug_print_mem((addr_t)loc_mr_list[entry].buf, buf_size);
nfound += result;
if(nfound >= NTRIAL) { break; }
}
k++;
}
printf("2nd send is OK!!\n");
#endif
} else { /* receiver side */
if(NSKIPR % PPOLLR !=0) { printf("set NSKIP multiple of PPOLL\n"); goto fn_fail; }
for (i = 0; i < NTRIAL; i++) {
if(i == NSKIPR) { tscs = rdtsc(); }
// poll on magic
dprintf("res.rdma_mr.buf=%lx\n", (unsigned long)res.rdma_mr.buf);
dprintf("poll addr=%lx\n", (unsigned long)(rdma_buf + buf_size * i + buf_size - sizeof(uint32_t)));
//k = 0;
volatile uint32_t *ptr = (volatile uint32_t *)(rdma_buf + buf_size * i + buf_size - sizeof(uint32_t));
while(*ptr != 10 + IBCOM_MAGIC) {
//k++; if(i >= NSKIPR && k % 65536 == 65535) { printf("i=%d,poll value=%x\n", i, *((uint32_t *)(rdma_buf + buf_size * i + buf_size - sizeof(uint32_t)))); }
__asm__ __volatile__("pause");
}
//debug_print_mem((addr_t)res.rdma_mr.buf, buf_size);
}
printf("2nd revieve is OK!!\n");
}
#endif
fn_exit:
/*Can free all resources*/
#if 0
if (resource_destroy(&config, &res)) {
fprintf(stderr, "resource destroy failed\n");
} else {
dprintf("destroy all successfully..\n");
}
if(loc_mr_list) { free(loc_mr_list); }
#endif
MPI_Finalize();
return ibcom_errno;
fn_fail:
goto fn_exit;
}

2
test/qlmpi/rusage/verbs/qp.c
View File

@@ -8,7 +8,7 @@
#include "sock.h" #include "sock.h"
#include "debug.h" #include "debug.h"
#define DEBUG_QP //#define DEBUG_QP
#ifdef DEBUG_QP #ifdef DEBUG_QP
#define dprintf printf #define dprintf printf
#else #else

1
test/qlmpi/rusage/verbs/sock.c
View File

@@ -109,6 +109,7 @@ int sock_connect(char *server_name, int port, int *listenfd){
// client mode // client mode
}else{ }else{
sleep(1);
inet_ntop(rp->ai_family, rp->ai_addr->sa_data, addrstr, 100); inet_ntop(rp->ai_family, rp->ai_addr->sa_data, addrstr, 100);
void *ptr; void *ptr;
switch(rp->ai_family) { switch(rp->ai_family) {