From a63b4a7369cd49236cfdee06c6cb07882133e178 Mon Sep 17 00:00:00 2001 From: Taku Shimosawa Date: Fri, 24 Feb 2012 18:08:05 +0900 Subject: [PATCH] System call delegation supports DMA alignment --- kernel/include/syscall.h | 6 +++-- kernel/syscall.c | 12 ++++------ linux/mod_mcctrl/control.c | 36 ++++++++++++++++++++++++++++- linux/mod_mcctrl/driver.c | 4 +++- linux/mod_mcctrl/ikc.c | 36 ++++++++++++++++++++++++++++- linux/mod_mcctrl/mcctrl.h | 4 ++-- linux/mod_mcctrl/syscall.c | 46 +++++++++++++++++++++++++++++++++----- 7 files changed, 124 insertions(+), 20 deletions(-) diff --git a/kernel/include/syscall.h b/kernel/include/syscall.h index 1b4f2c2c..ea619eda 100644 --- a/kernel/include/syscall.h +++ b/kernel/include/syscall.h @@ -1,6 +1,8 @@ #ifndef __HEADER_SYSCALL_H #define __HEADER_SYSCALL_H +#include + #define NUM_SYSCALLS 255 #define REQUEST_PAGE_COUNT 16 @@ -61,7 +63,7 @@ struct syscall_response { }; struct syscall_post { - unsigned long v[4]; + unsigned long v[8]; }; struct syscall_params { @@ -77,7 +79,7 @@ struct syscall_params { unsigned long post_rpa, post_pa; struct syscall_post *post_va; unsigned long post_fin; - struct syscall_post post_buf; + struct syscall_post post_buf AAL_DMA_ALIGN; }; #endif diff --git a/kernel/syscall.c b/kernel/syscall.c index ce6b317e..05feeb6f 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -72,7 +72,7 @@ long sys_brk(int n, aal_mc_user_context_t *ctx) } #define SYSCALL_DECLARE(name) long sys_##name(int n, aal_mc_user_context_t *ctx) -#define SYSCALL_HEADER struct syscall_request request; \ +#define SYSCALL_HEADER struct syscall_request request AAL_DMA_ALIGN; \ request.number = n #define SYSCALL_ARG_D(n) request.args[n] = aal_mc_syscall_arg##n(ctx) #define SYSCALL_ARG_MO(n) \ @@ -257,7 +257,7 @@ SYSCALL_DECLARE(uname) long sys_getxid(int n, aal_mc_user_context_t *ctx) { - struct syscall_request request; + struct syscall_request request AAL_DMA_ALIGN; request.number = n; @@ -287,7 +287,7 @@ SYSCALL_DECLARE(clone) { /* Clone a new thread */ struct process *new; - struct syscall_request request; + struct syscall_request request AAL_DMA_ALIGN; new = clone_process(cpu_local_var(current), aal_mc_syscall_pc(ctx), aal_mc_syscall_arg1(ctx)); @@ -332,8 +332,6 @@ static long (*syscall_table[])(int, aal_mc_user_context_t *) = { [231] = sys_exit_group, }; -#define DEBUG_PRINT_SC - long syscall(int num, aal_mc_user_context_t *ctx) { long l; @@ -345,9 +343,7 @@ long syscall(int num, aal_mc_user_context_t *ctx) aal_mc_get_processor_id(), num, aal_mc_syscall_arg0(ctx), aal_mc_syscall_arg1(ctx), - aal_mc_syscall_arg2(ctx), aal_mc_syscall_arg3(ctx), - aal_mc_syscall_arg4(ctx), aal_mc_syscall_pc(ctx), - aal_mc_syscall_sp(ctx)); + aal_mc_syscall_pc(ctx), aal_mc_syscall_sp(ctx)); #endif if (syscall_table[num]) { diff --git a/linux/mod_mcctrl/control.c b/linux/mod_mcctrl/control.c index e83cec4d..58bc21a5 100644 --- a/linux/mod_mcctrl/control.c +++ b/linux/mod_mcctrl/control.c @@ -8,6 +8,12 @@ #include #include "mcctrl.h" +#ifdef DEBUG +#define dprintk printk +#else +#define dprintk(...) +#endif + static DECLARE_WAIT_QUEUE_HEAD(wq_prepare); extern struct mcctrl_channel *channels; int mcctrl_ikc_set_recv_cpu(int cpu); @@ -145,6 +151,7 @@ int __do_in_kernel_syscall(aal_os_t os, struct mcctrl_channel *c, static int remaining_job, base_cpu, job_pos; extern int num_channels; +extern int mcctrl_dma_abort; int mcexec_wait_syscall(aal_os_t os, struct syscall_wait_desc *__user req) { @@ -163,6 +170,7 @@ int mcexec_wait_syscall(aal_os_t os, struct syscall_wait_desc *__user req) c->req = 0; #else while (1) { + c = channels + swd.cpu; rdtscll(s); if (!remaining_job) { while (!(*c->param.doorbell_va)) { @@ -193,9 +201,15 @@ int mcexec_wait_syscall(aal_os_t os, struct syscall_wait_desc *__user req) } else { c = channels + base_cpu + job_pos; } + if (!c) { + continue; + } if (c->param.request_va && c->param.request_va->valid) { c->param.request_va->valid = 0; /* ack */ + dprintk("SC #%lx, %lx\n", + c->param.request_va->number, + c->param.request_va->args[0]); if (__do_in_kernel_syscall(os, c, c->param.request_va)) { #endif if (copy_to_user(&req->sr, c->param.request_va, @@ -204,6 +218,9 @@ int mcexec_wait_syscall(aal_os_t os, struct syscall_wait_desc *__user req) } #ifndef DO_USER_MODE return 0; + } + if (mcctrl_dma_abort) { + return -2; } } } @@ -283,16 +300,33 @@ long mcexec_ret_syscall(aal_os_t os, struct syscall_ret_desc *__user arg) mc->param.response_va->ret = ret.ret; if (ret.size > 0) { + /* Host => Accel. Write is fast. */ + unsigned long phys; + void *rpm; + + phys = aal_device_map_memory(aal_os_to_dev(os), ret.dest, + ret.size); + rpm = ioremap_wc(phys, ret.size); + + memcpy(rpm, phys_to_virt(ret.src), ret.size); + mc->param.response_va->status = 1; + + iounmap(rpm); + aal_device_unmap_memory(aal_os_to_dev(os), phys, ret.size); + +/* memset(&request, 0, sizeof(request)); request.src_os = NULL; request.src_phys = ret.src; request.dest_os = os; request.dest_phys = ret.dest; request.size = ret.size; - request.notify = (void *)mc->param.response_pa; + request.notify_os = os; + request.notify = (void *)mc->param.response_rpa; request.priv = (void *)1; aal_dma_request(channel, &request); +*/ } else { mc->param.response_va->status = 1; } diff --git a/linux/mod_mcctrl/driver.c b/linux/mod_mcctrl/driver.c index be8bf941..9c5e8d26 100644 --- a/linux/mod_mcctrl/driver.c +++ b/linux/mod_mcctrl/driver.c @@ -11,6 +11,7 @@ extern long __mcctrl_control(aal_os_t, unsigned int, unsigned long); extern int prepare_ikc_channels(aal_os_t os); +extern void destroy_ikc_channels(aal_os_t os); static long mcctrl_ioctl(aal_os_t os, unsigned int request, void *priv, unsigned long arg) @@ -47,13 +48,14 @@ static int __init mcctrl_init(void) return -EINVAL; } - return aal_os_register_user_call_handlers(os, &mcctrl_uc); } static void __exit mcctrl_exit(void) { + printk("mcctrl: unregistered.\n"); aal_os_unregister_user_call_handlers(os, &mcctrl_uc); + destroy_ikc_channels(os); } MODULE_LICENSE("GPL v2"); diff --git a/linux/mod_mcctrl/ikc.c b/linux/mod_mcctrl/ikc.c index 2971151c..dd91e33f 100644 --- a/linux/mod_mcctrl/ikc.c +++ b/linux/mod_mcctrl/ikc.c @@ -7,6 +7,8 @@ #include #include +#define REQUEST_SHIFT 16 + int num_channels; struct mcctrl_channel *channels; @@ -98,7 +100,9 @@ static void mcctrl_ikc_init(aal_os_t os, int cpu, unsigned long rphys) sizeof(struct ikc_scd_init_param)); rpm = ioremap_wc(phys, sizeof(struct ikc_scd_init_param)); - pmc->param.request_va = (void *)__get_free_pages(GFP_KERNEL, 4); + pmc->param.request_va = + (void *)__get_free_pages(GFP_KERNEL, + REQUEST_SHIFT - PAGE_SHIFT); pmc->param.request_pa = virt_to_phys(pmc->param.request_va); pmc->param.doorbell_va = mcctrl_doorbell_va; pmc->param.doorbell_pa = mcctrl_doorbell_pa; @@ -199,3 +203,33 @@ int prepare_ikc_channels(aal_os_t os) aal_ikc_listen_port(os, &listen_param); return 0; } + +void __destroy_ikc_channel(aal_os_t os, struct mcctrl_channel *pmc) +{ + free_pages((unsigned long)pmc->param.request_va, + REQUEST_SHIFT - PAGE_SHIFT); + free_page((unsigned long)pmc->param.post_va); + + iounmap(pmc->param.response_va); + aal_device_unmap_memory(aal_os_to_dev(os), + pmc->param.response_pa, PAGE_SIZE); + free_pages((unsigned long)pmc->dma_buf, + DMA_PIN_SHIFT - PAGE_SHIFT); +} + +void destroy_ikc_channels(aal_os_t os) +{ + int i; + + for (i = 0; i < num_channels; i++) { + if (channels[i].c) { +// aal_ikc_disconnect(channels[i].c); + aal_ikc_free_channel(channels[i].c); + __destroy_ikc_channel(os, channels + i); + printk("Channel #%d freed.\n", i); + } + } + free_page((unsigned long)mcctrl_doorbell_va); + + kfree(channels); +} diff --git a/linux/mod_mcctrl/mcctrl.h b/linux/mod_mcctrl/mcctrl.h index 26fc9426..b37e82df 100644 --- a/linux/mod_mcctrl/mcctrl.h +++ b/linux/mod_mcctrl/mcctrl.h @@ -14,7 +14,7 @@ #define SCD_MSG_SYSCALL_ONESIDE 0x4 -#define DMA_PIN_SHIFT 16 +#define DMA_PIN_SHIFT 21 struct ikc_scd_packet { int msg; @@ -35,7 +35,7 @@ struct ikc_scd_init_param { }; struct syscall_post { - unsigned long v[4]; + unsigned long v[8]; }; struct syscall_params { diff --git a/linux/mod_mcctrl/syscall.c b/linux/mod_mcctrl/syscall.c index dabdec12..8bcdf2b4 100644 --- a/linux/mod_mcctrl/syscall.c +++ b/linux/mod_mcctrl/syscall.c @@ -9,11 +9,28 @@ #include #include "mcctrl.h" +#define ALIGN_WAIT_BUF(z) (((z + 63) >> 6) << 6) + +#define SC_DEBUG +#ifdef SC_DEBUG +static struct aal_dma_request last_request; + +static void print_dma_lastreq(void) +{ + printk("SRC OS : %p | %lx\nDESTOS : %p | %lx\n", last_request.src_os, + last_request.src_phys, last_request.dest_os, + last_request.dest_phys); + printk("SIZE : %lx | NOTIFY : %p | PRIV : %p\n", + last_request.size, last_request.notify, last_request.priv); +} +#endif + static int do_async_copy(aal_os_t os, unsigned long dest, unsigned long src, unsigned long size, unsigned int inbound) { struct aal_dma_request request; aal_dma_channel_t channel; + unsigned long asize = ALIGN_WAIT_BUF(size); channel = aal_device_get_dma_channel(aal_os_to_dev(os), 0); if (!channel) { @@ -21,28 +38,48 @@ static int do_async_copy(aal_os_t os, unsigned long dest, unsigned long src, } memset(&request, 0, sizeof(request)); - request.src_os = os; + request.src_os = inbound ? os : NULL; request.src_phys = src; - request.dest_os = NULL; + request.dest_os = inbound ? NULL : os; request.dest_phys = dest; request.size = size; - request.notify = (void *)(inbound ? dest + size : src + size); + request.notify = (void *)(inbound ? dest + asize : src + asize); request.priv = (void *)1; + *(unsigned long *)phys_to_virt((unsigned long)request.notify) = 0; +#ifdef SC_DEBUG + last_request = request; +#endif + aal_dma_request(channel, &request); return 0; } +int mcctrl_dma_abort; + static void async_wait(unsigned char *p, int size) { - while (!p[size]) { + int asize = ALIGN_WAIT_BUF(size); + unsigned long long s, w; + + rdtscll(s); + while (!p[asize]) { mb(); cpu_relax(); + rdtscll(w); + if (w > s + 1024UL * 1024 * 1024 * 10) { + printk("DMA Timed out : %p (%p + %d) => %d\n", + p + asize, p, size, p[asize]); + print_dma_lastreq(); + mcctrl_dma_abort = 1; + return; + } } } static void clear_wait(unsigned char *p, int size) { + int asize = ALIGN_WAIT_BUF(size); p[size] = 0; } @@ -75,7 +112,6 @@ static unsigned long translate_remote_va(struct mcctrl_channel *c, return -EFAULT; } - unsigned long last_thread_exec = 0; extern struct mcctrl_channel *channels;