wu-arch/kernels
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/kernels' into kernels-hopper

Browse Source
This commit is contained in:
Hansung Kim
2024-10-28 14:25:18 -07:00
parent d0421426be fd1c9f4729
commit b4dadfaf61
12 changed files with 450 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/regression/idle/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
*.bin
*.dump
*.elf
idle
.depend

2
tests/regression/idle/Makefile
View File

@@ -1,4 +1,4 @@
PROJECT = sgemm_gemmini_dma PROJECT = idle
SRCS = main.cpp common.h SRCS = main.cpp common.h

45
tests/regression/idle/kernel.cpp
View File

@@ -7,6 +7,7 @@
#include "gemmini_mmio.h" #include "gemmini_mmio.h"
#define NUM_CLUSTERS 1 #define NUM_CLUSTERS 1
#define NUM_THREADS_IN_CLUSTER 512
#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) #define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
@@ -21,9 +22,45 @@ void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
// reinterpret_cast<uint32_t *>(arg->addr_c)[0] = counter; // reinterpret_cast<uint32_t *>(arg->addr_c)[0] = counter;
// call barrier in a divergent branch, which will hang the core // call barrier in a divergent branch, which will hang the core
if ((vx_thread_id() % NUM_THREADS) == 0) { asm volatile("li x1, 0xa0a0a0a0");
vx_barrier(0, NUM_WARPS); asm volatile("li x2, 0xa0a0a0a0");
} asm volatile("li x3, 0xa0a0a0a0");
asm volatile("li x4, 0xa0a0a0a0");
asm volatile("li x5, 0xa0a0a0a0");
asm volatile("li x6, 0xa0a0a0a0");
asm volatile("li x7, 0xa0a0a0a0");
asm volatile("li x8, 0xa0a0a0a0");
asm volatile("li x9, 0xa0a0a0a0");
asm volatile("li x10, 0xa0a0a0a0");
asm volatile("li x11, 0xa0a0a0a0");
asm volatile("li x12, 0xa0a0a0a0");
asm volatile("li x13, 0xa0a0a0a0");
asm volatile("li x14, 0xa0a0a0a0");
asm volatile("li x15, 0xa0a0a0a0");
asm volatile("li x16, 0xa0a0a0a0");
asm volatile("li x17, 0xa0a0a0a0");
asm volatile("li x18, 0xa0a0a0a0");
asm volatile("li x19, 0xa0a0a0a0");
asm volatile("li x20, 0xa0a0a0a0");
asm volatile("li x21, 0xa0a0a0a0");
asm volatile("li x22, 0xa0a0a0a0");
asm volatile("li x23, 0xa0a0a0a0");
asm volatile("li x24, 0xa0a0a0a0");
asm volatile("li x25, 0xa0a0a0a0");
asm volatile("li x26, 0xa0a0a0a0");
asm volatile("li x27, 0xa0a0a0a0");
asm volatile("li x28, 0xa0a0a0a0");
asm volatile("li x29, 0xa0a0a0a0");
asm volatile("li x30, 0xa0a0a0a0");
asm volatile("li x31, 0xa0a0a0a0");
asm volatile("csrr a0, 0xcc1");
asm volatile("beqz a0, bar");
asm volatile("vx_tmc zero");
asm volatile("bar:");
asm volatile("vx_bar zero, a0");
// if ((vx_thread_id() % NUM_THREADS) == 0) {
// vx_barrier(0, NUM_WARPS);
// }
vx_tmc(0); vx_tmc(0);
} }
@@ -34,7 +71,7 @@ int main() {
// spawn a single warp in every core // spawn a single warp in every core
const uint32_t grid_size = NUM_THREADS * NUM_CORES; const uint32_t grid_size = NUM_THREADS * NUM_CORES;
#ifdef RADIANCE #ifdef RADIANCE
vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); vx_spawn_tasks_cluster(NUM_THREADS_IN_CLUSTER, (vx_spawn_tasks_cb)kernel_body, arg);
#else #else
vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
#endif #endif

154
tests/regression/sgemm_gemmini/kernel.cpp
View File

@@ -6,25 +6,53 @@
#include "include/gemmini.h" #include "include/gemmini.h"
#include "gemmini_mmio.h" #include "gemmini_mmio.h"
#define NUM_CLUSTERS 1
// #define FP32
#ifdef FP32
// fp32
#define TILE_M 64 #define TILE_M 64
#define TILE_N 64 #define TILE_N 64
#define TILE_K 64 #define TILE_K 64
#define TILE_MN 4096 #define TILE_MN 4096
#define TILE_MK 4096 #define TILE_MK 4096
#define TILE_NK 4096 #define TILE_NK 4096
#define NUM_CLUSTERS 1
#define NUM_THREADS_IN_CLUSTER 256 #define NUM_THREADS_IN_CLUSTER 256
#define SMEM_ADDR_Q0 ((float * const) 0xff000000) #define SMEM_ADDR_Q0 ((mem_elem_t * const) 0xff000000)
#define SMEM_ADDR_Q1 ((float * const) 0xff004000) #define SMEM_ADDR_Q1 ((mem_elem_t * const) 0xff004000)
#define SMEM_ADDR_Q2 ((float * const) 0xff008000) #define SMEM_ADDR_Q2 ((mem_elem_t * const) 0xff008000)
#define SMEM_ADDR_Q3 ((float * const) 0xff00c000) #define SMEM_ADDR_Q3 ((mem_elem_t * const) 0xff00c000)
#define SPAD_ADDR_Q0 0x0 #define SPAD_ADDR_Q0 0x0
#define SPAD_ADDR_Q1 0x200 #define SPAD_ADDR_Q1 0x200
#define SPAD_ADDR_Q2 0x400 #define SPAD_ADDR_Q2 0x400
#define SPAD_ADDR_Q3 0x600 #define SPAD_ADDR_Q3 0x600
#define SPAD_ADDR_Q4 0x800 #define SPAD_ADDR_Q4 0x800
typedef float smem_elem_t;
typedef float mem_elem_t;
#else
// fp16
#define TILE_M 128
#define TILE_N 64
#define TILE_K 128
#define TILE_MN 8192
#define TILE_MK 16384
#define TILE_NK 8192
#define NUM_THREADS_IN_CLUSTER 512
#define SMEM_ADDR_Q0 ((mem_elem_t * const) 0xff000000)
#define SMEM_ADDR_Q1 ((mem_elem_t * const) 0xff008000)
#define SMEM_ADDR_Q2 ((mem_elem_t * const) 0xff001000)
#define SMEM_ADDR_Q3 ((mem_elem_t * const) 0xff018000)
#define SPAD_ADDR_Q0 0x0
#define SPAD_ADDR_Q1 0x400
#define SPAD_ADDR_Q2 0x800
#define SPAD_ADDR_Q3 0xc00
#define SPAD_ADDR_Q4 0x1000
typedef uint16_t smem_elem_t;
typedef uint32_t mem_elem_t;
#endif
#define HARDCODE #define HARDCODE
#define REGBLOCK #define REGBLOCK
@@ -61,9 +89,9 @@ inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) {
void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
const uint32_t threadblock_id, const uint32_t threadblock_id,
const uint32_t tid_in_threadblock) { const uint32_t tid_in_threadblock) {
const float * const A = (const float * const) arg->addr_a; const smem_elem_t * const A = (const smem_elem_t * const) arg->addr_a;
const float * const B = (const float * const) arg->addr_b; const smem_elem_t * const B = (const smem_elem_t * const) arg->addr_b;
float * const C = (float * const) arg->addr_c; smem_elem_t * const C = (smem_elem_t * const) arg->addr_c;
if (tid_in_threadblock % NUM_THREADS_IN_CLUSTER == 0) { if (tid_in_threadblock % NUM_THREADS_IN_CLUSTER == 0) {
gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0);
@@ -123,11 +151,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
tile_i < num_tile_rows_per_tb * (threadblock_id + 1); tile_i < num_tile_rows_per_tb * (threadblock_id + 1);
tile_i += 1) { tile_i += 1) {
for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) {
float * const smem_c_tile_start = SMEM_ADDR_Q1; mem_elem_t * const smem_c_tile_start = SMEM_ADDR_Q1;
#ifdef OFFLOAD_ACCUMULATE #ifdef OFFLOAD_ACCUMULATE
float * const smem_acc_tile_start = SMEM_ADDR_Q0 + HW_TID(); mem_elem_t * const smem_acc_tile_start = SMEM_ADDR_Q0 + HW_TID();
#else #else
float * const smem_acc_tile_start = SMEM_ADDR_Q2 + hw_tid; mem_elem_t * const smem_acc_tile_start = SMEM_ADDR_Q2 + hw_tid;
#endif #endif
for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) { for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) {
@@ -140,19 +168,19 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
// #endif // #endif
constexpr uint32_t every_iter = j1_stride; constexpr uint32_t every_iter = j1_stride;
const uint32_t every_2iters_a = i1_stride * dim_k; const uint32_t every_2iters_a = i1_stride * (dim_k * sizeof(smem_elem_t) / 4);
const uint32_t runtime_const_a = i0 * dim_k + j1_idx + j0; const uint32_t runtime_const_a = i0 * (dim_k * sizeof(smem_elem_t) / 4) + j1_idx + j0;
const uint32_t every_2iters_b = i1_stride * dim_n; const uint32_t every_2iters_b = i1_stride * dim_n;
const uint32_t runtime_const_b = i0 * dim_n + j1_idx + j0; const uint32_t runtime_const_b = i0 * dim_n + j1_idx + j0;
const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K + runtime_const_a; const mem_elem_t * const dram_a_tile_start = (const mem_elem_t * const) (A + tile_i * TILE_M * dim_k + tile_k * TILE_K + runtime_const_a);
const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N + runtime_const_b; const mem_elem_t * const dram_b_tile_start = (const mem_elem_t * const) (B + tile_k * TILE_K * dim_n + tile_j * TILE_N + runtime_const_b);
#ifdef DBUF #ifdef DBUF
float * const smem_a_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q1 : SMEM_ADDR_Q0) + HW_TID(); mem_elem_t * const smem_a_tile_start = (mem_elem_t * const) (((tile_k & 1) ? SMEM_ADDR_Q1 : SMEM_ADDR_Q0) + HW_TID());
float * const smem_b_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q3 : SMEM_ADDR_Q2) + HW_TID(); mem_elem_t * const smem_b_tile_start = (mem_elem_t * const) (((tile_k & 1) ? SMEM_ADDR_Q3 : SMEM_ADDR_Q2) + HW_TID());
#else #else
float * const smem_a_tile_start = SMEM_ADDR_Q0 + HW_TID(); mem_elem_t * const smem_a_tile_start = (mem_elem_t * const) (SMEM_ADDR_Q0 + HW_TID());
float * const smem_b_tile_start = SMEM_ADDR_Q3 + HW_TID(); mem_elem_t * const smem_b_tile_start = (mem_elem_t * const) (SMEM_ADDR_Q3 + HW_TID());
#endif #endif
{ {
@@ -191,10 +219,10 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
smem_b_tile_start[7 * num_threads_in_cluster + hw_tid] = \ smem_b_tile_start[7 * num_threads_in_cluster + hw_tid] = \
dram_b_tile_start[every_iter * 1 + every_2iters_b * 3]; dram_b_tile_start[every_iter * 1 + every_2iters_b * 3];
#else #else
float v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 0]; mem_elem_t v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 0];
float v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 0]; mem_elem_t v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 0];
float v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 1]; mem_elem_t v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 1];
float v3 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 1]; mem_elem_t v3 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 1];
smem_a_tile_start[0 * num_threads_in_cluster] = v0; smem_a_tile_start[0 * num_threads_in_cluster] = v0;
smem_a_tile_start[1 * num_threads_in_cluster] = v1; smem_a_tile_start[1 * num_threads_in_cluster] = v1;
smem_a_tile_start[2 * num_threads_in_cluster] = v2; smem_a_tile_start[2 * num_threads_in_cluster] = v2;
@@ -236,14 +264,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
smem_a_tile_start[10 * num_threads_in_cluster] = v2; smem_a_tile_start[10 * num_threads_in_cluster] = v2;
smem_a_tile_start[11 * num_threads_in_cluster] = v3; smem_a_tile_start[11 * num_threads_in_cluster] = v3;
v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 4]; // v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 4];
v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 4]; // v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 4];
v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 5]; // v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 5];
v3 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 5]; // v3 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 5];
smem_b_tile_start[8 * num_threads_in_cluster] = v0; // smem_b_tile_start[8 * num_threads_in_cluster] = v0;
smem_b_tile_start[9 * num_threads_in_cluster] = v1; // smem_b_tile_start[9 * num_threads_in_cluster] = v1;
smem_b_tile_start[10 * num_threads_in_cluster] = v2; // smem_b_tile_start[10 * num_threads_in_cluster] = v2;
smem_b_tile_start[11 * num_threads_in_cluster] = v3; // smem_b_tile_start[11 * num_threads_in_cluster] = v3;
v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 6]; v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 6];
v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 6]; v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 6];
@@ -254,14 +282,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
smem_a_tile_start[14 * num_threads_in_cluster] = v2; smem_a_tile_start[14 * num_threads_in_cluster] = v2;
smem_a_tile_start[15 * num_threads_in_cluster] = v3; smem_a_tile_start[15 * num_threads_in_cluster] = v3;
v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 6]; // v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 6];
v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 6]; // v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 6];
v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 7]; // v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 7];
v3 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 7]; // v3 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 7];
smem_b_tile_start[12 * num_threads_in_cluster] = v0; // smem_b_tile_start[12 * num_threads_in_cluster] = v0;
smem_b_tile_start[13 * num_threads_in_cluster] = v1; // smem_b_tile_start[13 * num_threads_in_cluster] = v1;
smem_b_tile_start[14 * num_threads_in_cluster] = v2; // smem_b_tile_start[14 * num_threads_in_cluster] = v2;
smem_b_tile_start[15 * num_threads_in_cluster] = v3; // smem_b_tile_start[15 * num_threads_in_cluster] = v3;
#endif #endif
} }
#else #else
@@ -440,8 +468,8 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
#ifdef CISC #ifdef CISC
GEMMINI_CISC_CMD_I(9); GEMMINI_CISC_CMD_I(9);
#else #else
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (((uint64_t) TILE_M / DIM) << 32) | ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (((uint64_t) TILE_K / DIM) << 32) |
(((uint64_t) TILE_K / DIM) << 16) | ((uint64_t) TILE_N / DIM), k_LOOP_WS_CONFIG_BOUNDS) (((uint64_t) TILE_N / DIM) << 16) | ((uint64_t) TILE_M / DIM), k_LOOP_WS_CONFIG_BOUNDS)
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, 0x278U, k_LOOP_WS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, 0x278U, k_LOOP_WS)
#endif #endif
gemmini_fence(); gemmini_fence();
@@ -458,13 +486,13 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
constexpr uint32_t every_iter = j1_stride; constexpr uint32_t every_iter = j1_stride;
const uint32_t every_2iters = i1_stride * dim_n; const uint32_t every_2iters = i1_stride * dim_n;
const uint32_t runtime_const = i0 * dim_n + j1_idx + j0; const uint32_t runtime_const = i0 * dim_n + j1_idx + j0;
float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N + runtime_const; mem_elem_t * const dram_c_tile_start = (mem_elem_t * const) (C + tile_i * TILE_M * dim_n + tile_j * TILE_N + runtime_const);
#ifdef REGBLOCK #ifdef REGBLOCK
float v0 = smem_acc_tile_start[0 * num_threads_in_cluster]; mem_elem_t v0 = smem_acc_tile_start[0 * num_threads_in_cluster];
float v1 = smem_acc_tile_start[1 * num_threads_in_cluster]; mem_elem_t v1 = smem_acc_tile_start[1 * num_threads_in_cluster];
float v2 = smem_acc_tile_start[2 * num_threads_in_cluster]; mem_elem_t v2 = smem_acc_tile_start[2 * num_threads_in_cluster];
float v3 = smem_acc_tile_start[3 * num_threads_in_cluster]; mem_elem_t v3 = smem_acc_tile_start[3 * num_threads_in_cluster];
#ifdef ACTIVATE #ifdef ACTIVATE
uint32_t swish_start, swish_end; uint32_t swish_start, swish_end;
rd_cycles_force(swish_start); rd_cycles_force(swish_start);
@@ -498,23 +526,23 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
dram_c_tile_start[every_iter * 0 + every_2iters * 3] = v2; dram_c_tile_start[every_iter * 0 + every_2iters * 3] = v2;
dram_c_tile_start[every_iter * 1 + every_2iters * 3] = v3; dram_c_tile_start[every_iter * 1 + every_2iters * 3] = v3;
v0 = smem_acc_tile_start[8 * num_threads_in_cluster]; // v0 = smem_acc_tile_start[8 * num_threads_in_cluster];
v1 = smem_acc_tile_start[9 * num_threads_in_cluster]; // v1 = smem_acc_tile_start[9 * num_threads_in_cluster];
v2 = smem_acc_tile_start[10 * num_threads_in_cluster]; // v2 = smem_acc_tile_start[10 * num_threads_in_cluster];
v3 = smem_acc_tile_start[11 * num_threads_in_cluster]; // v3 = smem_acc_tile_start[11 * num_threads_in_cluster];
dram_c_tile_start[every_iter * 0 + every_2iters * 4] = v0; // dram_c_tile_start[every_iter * 0 + every_2iters * 4] = v0;
dram_c_tile_start[every_iter * 1 + every_2iters * 4] = v1; // dram_c_tile_start[every_iter * 1 + every_2iters * 4] = v1;
dram_c_tile_start[every_iter * 0 + every_2iters * 5] = v2; // dram_c_tile_start[every_iter * 0 + every_2iters * 5] = v2;
dram_c_tile_start[every_iter * 1 + every_2iters * 5] = v3; // dram_c_tile_start[every_iter * 1 + every_2iters * 5] = v3;
v0 = smem_acc_tile_start[12 * num_threads_in_cluster]; // v0 = smem_acc_tile_start[12 * num_threads_in_cluster];
v1 = smem_acc_tile_start[13 * num_threads_in_cluster]; // v1 = smem_acc_tile_start[13 * num_threads_in_cluster];
v2 = smem_acc_tile_start[14 * num_threads_in_cluster]; // v2 = smem_acc_tile_start[14 * num_threads_in_cluster];
v3 = smem_acc_tile_start[15 * num_threads_in_cluster]; // v3 = smem_acc_tile_start[15 * num_threads_in_cluster];
dram_c_tile_start[every_iter * 0 + every_2iters * 6] = v0; // dram_c_tile_start[every_iter * 0 + every_2iters * 6] = v0;
dram_c_tile_start[every_iter * 1 + every_2iters * 6] = v1; // dram_c_tile_start[every_iter * 1 + every_2iters * 6] = v1;
dram_c_tile_start[every_iter * 0 + every_2iters * 7] = v2; // dram_c_tile_start[every_iter * 0 + every_2iters * 7] = v2;
dram_c_tile_start[every_iter * 1 + every_2iters * 7] = v3; // dram_c_tile_start[every_iter * 1 + every_2iters * 7] = v3;
#else #else
dram_c_tile_start[every_iter * 0 + every_2iters * 0] = \ dram_c_tile_start[every_iter * 0 + every_2iters * 0] = \
smem_acc_tile_start[0 * num_threads_in_cluster]; smem_acc_tile_start[0 * num_threads_in_cluster];

6
tests/regression/sgemm_gemmini_dma/generate_operands.py
View File

@@ -21,15 +21,15 @@ matrix_a = generate_fp16_matrix(size)
matrix_b = generate_fp16_matrix(size) matrix_b = generate_fp16_matrix(size)
# Save the operand matrices to binary files # Save the operand matrices to binary files
# save_matrix_to_bin("input.a.bin", matrix_a) save_matrix_to_bin("input.a.bin", matrix_a)
# save_matrix_to_bin("input.b.bin", matrix_b) save_matrix_to_bin("input.b.bin", matrix_b)
# Generate and save the reference matrices for 128x128, 256x256, and 512x512 sizes # Generate and save the reference matrices for 128x128, 256x256, and 512x512 sizes
sizes = [128, 256, 512] sizes = [128, 256, 512]
for s in sizes: for s in sizes:
ref_matrix = truncated_matrix_multiplication(matrix_a, matrix_b, s) ref_matrix = truncated_matrix_multiplication(matrix_a, matrix_b, s)
print(ref_matrix) print(ref_matrix)
# save_matrix_to_bin(f"ref{s}.bin", ref_matrix) save_matrix_to_bin(f"ref{s}.bin", ref_matrix)
print("All files generated successfully.") print("All files generated successfully.")

38
tests/regression/sgemm_gemmini_dma/kernel.cpp
View File

@@ -49,15 +49,17 @@
#define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) #define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x))
#define rd_cycles(x) rd_cycles_force(x) #define rd_cycles(x) rd_cycles_force(x)
#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) #define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
#define MARK_BEG() asm volatile ("slti x0, x1, -1047")
#define MARK_END() asm volatile ("slti x0, x1, -499")
#define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) #define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__)
// #define PRINTF(...) vx_printf(__VA_ARGS__) // #define PRINTF(...) vx_printf(__VA_ARGS__)
#define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x)))) #define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x))))
#define POWER //#define POWER
typedef uint16_t smem_elem_t; typedef uint16_t smem_elem_t;
// typedef float smem_elem_t; // typedef float smem_elem_t;
inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) __attribute__((convergent)) {
vx_fence(); vx_fence();
vx_barrier(barrier_id, count); vx_barrier(barrier_id, count);
} }
@@ -79,9 +81,26 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
} }
vx_fence(); vx_fence();
// if (HW_TID() < 128) {
// *((volatile uint32_t *) 0xff000000 + HW_TID()) = HW_TID();
// for (int i = 0; i < 128; i++) {
// if (HW_TID() == i) {
// volatile uint32_t x = *((volatile uint32_t *) 0xff000000 + HW_TID());
// if (x != i) {
// PRINTF("%d ", x);
// }
// }
// }
// }
// threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
// if (HW_TID() == 0) {
// PRINTF("\n finished\n");
// }
// threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
uint32_t marker0, marker1; uint32_t marker0, marker1;
rd_cycles_force(marker0); rd_cycles_force(marker0);
MARK_BEG();
const uint32_t dim_m = arg->dim_m; const uint32_t dim_m = arg->dim_m;
const uint32_t dim_n = arg->dim_n; const uint32_t dim_n = arg->dim_n;
@@ -99,18 +118,16 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
// gemmini_extended3_config_ld(repeating_bias ? 0 : (stride_D * sizeof_D), D_scale_factor, low_D, 2); // gemmini_extended3_config_ld(repeating_bias ? 0 : (stride_D * sizeof_D), D_scale_factor, low_D, 2);
gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY);
// gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale); // gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale);
}
for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id; for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id;
tile_i < num_tile_rows_per_tb * (threadblock_id + 1); tile_i < num_tile_rows_per_tb * (threadblock_id + 1);
tile_i += 1) { tile_i += 1) {
for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) {
if (HW_TID() == 0) {
for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) { for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) {
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC,
(uint64_t) (A + tile_i * TILE_M * dim_k + tile_k * TILE_K), (uint64_t) (A + tile_i * TILE_M * dim_k + tile_k * TILE_K),
(uint64_t) (B + tile_k * TILE_K * dim_n + tile_j * TILE_N), k_LOOP_WS_CONFIG_ADDRS_AB) (uint64_t) (B + tile_k * TILE_K * dim_n + tile_j * TILE_N), k_LOOP_WS_CONFIG_ADDRS_AB)
GEMMINI_CISC_CMD_R((dim_n) << 16 | (dim_k << 8) | 8); GEMMINI_CISC_CMD_R((dim_n << 20) | (dim_k << 8) | 8);
if (tile_k & 1) { if (tile_k & 1) {
GEMMINI_CISC_CMD_I(11); GEMMINI_CISC_CMD_I(11);
} else { } else {
@@ -156,10 +173,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
// last thread block complete // last thread block complete
if (threadblock_id == NUM_CLUSTERS - 1) { if (threadblock_id == NUM_CLUSTERS - 1) {
threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
MARK_END();
rd_cycles_force(marker1); rd_cycles_force(marker1);
if (HW_TID() == 0) { if (HW_TID() == 0) {
#ifdef POWER #ifdef POWER
PRINTF("%d\n", marker1 - marker0); // PRINTF("%d\n", marker1 - marker0);
#else #else
PRINTF("\ncomplete\n"); PRINTF("\ncomplete\n");
PRINTF("total cycles: %d\n", marker1 - marker0); PRINTF("total cycles: %d\n", marker1 - marker0);

9
tests/regression/unaligned/Makefile Normal file
View File

@@ -0,0 +1,9 @@
PROJECT = unaligned
SRCS = main.cpp common.h
VX_SRCS = kernel.cpp
OPTS ?= -n16
include ../common.mk

13
tests/regression/unaligned/common.h Normal file
View File

@@ -0,0 +1,13 @@
#ifndef _COMMON_H_
#define _COMMON_H_
#include <cstdint>
#define KERNEL_ARG_DEV_MEM_ADDR 0x9fff0000
#define DEV_SMEM_START_ADDR 0xff000000
typedef struct {
uint32_t placeholder;
} kernel_arg_t;
#endif

123
tests/regression/unaligned/kernel.cpp Normal file
View File

@@ -0,0 +1,123 @@
#include <stdint.h>
#include <vx_intrinsics.h>
#include <vx_print.h>
#include <vx_spawn.h>
#include "common.h"
#define NUM_THREADS_IN_CLUSTER 32
#define NUM_CLUSTERS 1
#define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x))
#define rd_cycles(x) rd_cycles_force(x)
#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
#define PRINT_BUF ((char *) (0xff020000UL))
#define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__)
inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) __attribute__((convergent)) {
vx_fence();
vx_barrier(barrier_id, count);
}
#define ADDR0 0xff008004UL
#define ADDR1 0xff009004UL
#define ADDR2 0xff00a004UL
#define ADDR3 0xff00b004UL
void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) __attribute__((convergent)) {
size_t t = (size_t) (task_id * 4) % 32;
if (t == 0) {
for (int j = 0; j < 0x400; j += 0x100) {
for (int i = 0; i < 8; i++) {
*((volatile uint32_t *) (ADDR0 + j + i * 4)) = 0xbeef;
*((volatile uint32_t *) (ADDR1 + j + i * 4)) = 0xbeef;
}
}
}
threadblock_barrier(0, 1);
// for (int i = 0; i < 8; i++) {
if (HW_TID() % 8 < 5) {
// if (true) {
asm volatile("lower_block:");
volatile uint32_t a = *((volatile uint32_t *) (ADDR0 + 0x000 + t));
volatile uint32_t b = *((volatile uint32_t *) (ADDR0 + 0x100 + t));
volatile uint32_t c = *((volatile uint32_t *) (ADDR0 + 0x200 + t));
volatile uint32_t d = *((volatile uint32_t *) (ADDR0 + 0x300 + t));
volatile uint32_t u = *((volatile uint32_t *) (ADDR1 + 0x000 + t));
volatile uint32_t v = *((volatile uint32_t *) (ADDR1 + 0x100 + t));
volatile uint32_t w = *((volatile uint32_t *) (ADDR1 + 0x200 + t));
volatile uint32_t x = *((volatile uint32_t *) (ADDR1 + 0x300 + t));
*((volatile uint32_t *) (ADDR2 + 0x000 + t)) = a;
*((volatile uint32_t *) (ADDR2 + 0x100 + t)) = b;
*((volatile uint32_t *) (ADDR2 + 0x200 + t)) = c;
*((volatile uint32_t *) (ADDR2 + 0x300 + t)) = d;
*((volatile uint32_t *) (ADDR3 + 0x000 + t)) = u;
*((volatile uint32_t *) (ADDR3 + 0x100 + t)) = v;
*((volatile uint32_t *) (ADDR3 + 0x200 + t)) = w;
*((volatile uint32_t *) (ADDR3 + 0x300 + t)) = x;
} else {
asm volatile("upper_block:");
volatile uint32_t a = *((volatile uint32_t *) (ADDR1 + 0x000 + t));
volatile uint32_t b = *((volatile uint32_t *) (ADDR1 + 0x100 + t));
volatile uint32_t c = *((volatile uint32_t *) (ADDR1 + 0x200 + t));
volatile uint32_t d = *((volatile uint32_t *) (ADDR1 + 0x300 + t));
volatile uint32_t u = *((volatile uint32_t *) (ADDR0 + 0x000 + t));
volatile uint32_t v = *((volatile uint32_t *) (ADDR0 + 0x100 + t));
volatile uint32_t w = *((volatile uint32_t *) (ADDR0 + 0x200 + t));
volatile uint32_t x = *((volatile uint32_t *) (ADDR0 + 0x300 + t));
// for (int y = 4; y < 8; y++) {
// if (task_id == y) {
// PRINTF("Task ID: %d, a: %x, b: %x, c: %x, d: %x\n", task_id, a, b, c, d);
// PRINTF("Task ID: %d, u: %x, v: %x, w: %x, x: %x\n", task_id, u, v, w, x);
// }
// }
// threadblock_barrier(1, 1);
*((volatile uint32_t *) (ADDR3 + 0x000 + t)) = a;
*((volatile uint32_t *) (ADDR3 + 0x100 + t)) = b;
*((volatile uint32_t *) (ADDR3 + 0x200 + t)) = c;
*((volatile uint32_t *) (ADDR3 + 0x300 + t)) = d;
*((volatile uint32_t *) (ADDR2 + 0x000 + t)) = u;
*((volatile uint32_t *) (ADDR2 + 0x100 + t)) = v;
*((volatile uint32_t *) (ADDR2 + 0x200 + t)) = w;
*((volatile uint32_t *) (ADDR2 + 0x300 + t)) = x;
}
// }
threadblock_barrier(2, 1);
PRINTF(".");
if (task_id == 0) {
bool correct = true;
PRINTF("\n");
for (int j = 0; j < 0x400; j += 0x100) {
for (int i = 0; i < 8; i++) {
int v2 = *((volatile uint32_t *) (ADDR2 + i * 4 + j));
if (v2 != 0xbeef) {
correct = false;
PRINTF("mismatch at %x, got %x\n", ADDR2 + i * 4 + j, v2);
}
int v3 = *((volatile uint32_t *) (ADDR3 + i * 4 + j));
if (v3 != 0xbeef) {
correct = false;
PRINTF("mismatch at %x, got %x\n", ADDR3 + i * 4 + j, v3);
}
}
}
if (correct) {
PRINTF("test passed\n");
}
}
}
int main() __attribute__((convergent)) {
kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR;
const uint32_t num_threads_in_cluster = NUM_THREADS_IN_CLUSTER;
const uint32_t grid_size = num_threads_in_cluster * NUM_CLUSTERS;
vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
return 0;
}

44
tests/regression/unaligned/kernel.minimal.cpp Normal file
View File

@@ -0,0 +1,44 @@
#include <stdint.h>
#include <vx_intrinsics.h>
#include <vx_print.h>
#include <vx_spawn.h>
#include "common.h"
#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) __attribute__((convergent)) {
vx_fence();
vx_barrier(barrier_id, count);
}
#define ADDR0 0xff008004UL
#define ADDR1 0xff009004UL
void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
// size_t t = (size_t) (task_id * 4) % 32;
asm volatile("nop");
for (int i = 0; i < 8; i++) {
if (i == 0) {
if ((HW_TID() & 0x7) < 2) {
asm volatile("lower_block:");
volatile uint32_t a = *((volatile uint32_t *) (ADDR0));
// *((volatile uint32_t *) (ADDR2)) = a;
volatile uint32_t b = a + 1;
} else {
asm volatile("upper_block:");
volatile uint32_t a = *((volatile uint32_t *) (ADDR1));
// *((volatile uint32_t *) (ADDR3)) = a;
volatile uint32_t b = a + 1;
}
}
volatile uint32_t a = *((volatile uint32_t *) (ADDR1));
}
threadblock_barrier(2, 2);
}
int main() { // __attribute__((convergent)) {
kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR;
vx_spawn_tasks_cluster(64, (vx_spawn_tasks_cb)kernel_body, arg);
return 0;
}

92
tests/regression/unaligned/main.cpp Normal file
View File

@@ -0,0 +1,92 @@
#include <iostream>
#include <fstream>
#include <unistd.h>
#include <string.h>
#include <vortex.h>
#include <vector>
#include "common.h"
#define RT_CHECK(_expr) \
do { \
int _ret = _expr; \
if (0 == _ret) \
break; \
printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \
cleanup(); \
exit(-1); \
} while (false)
///////////////////////////////////////////////////////////////////////////////
const char* kernel_file = "kernel.bin";
uint32_t count = 0;
vx_device_h device = nullptr;
std::vector<uint8_t> staging_buf;
kernel_arg_t kernel_arg = {};
static void show_usage() {
std::cout << "Vortex Test." << std::endl;
std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl;
}
static void parse_args(int argc, char **argv) {
int c;
while ((c = getopt(argc, argv, "n:k:h?")) != -1) {
switch (c) {
case 'n':
count = atoi(optarg);
break;
case 'k':
kernel_file = optarg;
break;
case 'h':
case '?': {
show_usage();
exit(0);
} break;
default:
show_usage();
exit(-1);
}
}
}
void cleanup() {
if (device) {
vx_dev_close(device);
}
}
int main(int argc, char *argv[]) {
// parse command arguments
parse_args(argc, argv);
if (count == 0) {
count = 1;
}
std::srand(50);
// open device connection
std::cout << "open device connection" << std::endl;
RT_CHECK(vx_dev_open(&device));
// upload program
std::cout << "upload program" << std::endl;
RT_CHECK(vx_upload_kernel_file(device, kernel_file));
// start device
std::cout << "start device" << std::endl;
RT_CHECK(vx_start(device));
// wait for completion
std::cout << "wait for completion" << std::endl;
RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT));
// cleanup
std::cout << "cleanup" << std::endl;
cleanup();
return 0;
}

BIN
tests/regression/unaligned/unaligned Executable file
View File

Binary file not shown.