From 0bb7aeb45b61503044177e69033700864c39fc5e Mon Sep 17 00:00:00 2001 From: Richard Yan Date: Mon, 15 Apr 2024 10:10:20 -0700 Subject: [PATCH] add gpu+gemmini gemm kernel --- tests/regression/common.mk | 3 +- tests/regression/sgemm_gemmini/.gitignore | 5 + tests/regression/sgemm_gemmini/Makefile | 9 + tests/regression/sgemm_gemmini/common.h | 18 ++ tests/regression/sgemm_gemmini/kernel.cpp | 269 ++++++++++++++++++ tests/regression/sgemm_gemmini/main.cpp | 274 +++++++++++++++++++ tests/regression/sgemm_gemmini/sgemm_gemmini | Bin 0 -> 28448 bytes 7 files changed, 577 insertions(+), 1 deletion(-) create mode 100644 tests/regression/sgemm_gemmini/.gitignore create mode 100644 tests/regression/sgemm_gemmini/Makefile create mode 100644 tests/regression/sgemm_gemmini/common.h create mode 100644 tests/regression/sgemm_gemmini/kernel.cpp create mode 100644 tests/regression/sgemm_gemmini/main.cpp create mode 100755 tests/regression/sgemm_gemmini/sgemm_gemmini diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 8f4c4db1..81df3139 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -22,6 +22,7 @@ RISCV_SYSROOT ?= $(RISCV_TOOLCHAIN_PATH)/$(RISCV_PREFIX) VORTEX_RT_PATH ?= $(realpath ../../../runtime) VORTEX_KN_PATH ?= $(realpath ../../../kernel) +GEMMINI_SW_PATH ?= $(realpath ../../../third_party/gemmini-rocc-tests) FPGA_BIN_DIR ?= $(VORTEX_RT_PATH)/opae @@ -49,7 +50,7 @@ VX_CP = $(LLVM_VORTEX)/bin/llvm-objcopy VX_CFLAGS += -v -O3 -std=c++17 VX_CFLAGS += -mcmodel=medany -fno-rtti -fno-exceptions -nostartfiles -fdata-sections -ffunction-sections -VX_CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw +VX_CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw -I$(GEMMINI_SW_PATH) VX_CFLAGS += -DNDEBUG -DLLVM_VORTEX VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a diff --git a/tests/regression/sgemm_gemmini/.gitignore b/tests/regression/sgemm_gemmini/.gitignore new file mode 100644 index 00000000..7c35ba59 --- /dev/null +++ b/tests/regression/sgemm_gemmini/.gitignore @@ -0,0 +1,5 @@ +*.bin +*.dump +*.elf +sgemm_wg +.depend diff --git a/tests/regression/sgemm_gemmini/Makefile b/tests/regression/sgemm_gemmini/Makefile new file mode 100644 index 00000000..a36f6d21 --- /dev/null +++ b/tests/regression/sgemm_gemmini/Makefile @@ -0,0 +1,9 @@ +PROJECT = sgemm_gemmini + +SRCS = main.cpp common.h + +VX_SRCS = kernel.cpp + +OPTS ?= -n16 + +include ../common.mk diff --git a/tests/regression/sgemm_gemmini/common.h b/tests/regression/sgemm_gemmini/common.h new file mode 100644 index 00000000..74941562 --- /dev/null +++ b/tests/regression/sgemm_gemmini/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000 +#define DEV_SMEM_START_ADDR 0xff000000 + +typedef struct { + uint32_t dim_m; + uint32_t dim_n; + uint32_t dim_k; + uint64_t addr_a; + uint64_t addr_b; + uint64_t addr_c; +} kernel_arg_t; + +#endif diff --git a/tests/regression/sgemm_gemmini/kernel.cpp b/tests/regression/sgemm_gemmini/kernel.cpp new file mode 100644 index 00000000..34c72d00 --- /dev/null +++ b/tests/regression/sgemm_gemmini/kernel.cpp @@ -0,0 +1,269 @@ +#include +#include +#include +#include +#include "common.h" +#include "include/gemmini.h" +#include "gemmini_mmio.h" + +#define MATRIX_M 64 // TODO: remove hardcode +#define MATRIX_N 64 +#define MATRIX_K 64 +#define TILE_M 32 // tile size = SMEM size / 2 (double buffering) / 4 (A, B, C, Psum) +#define TILE_N 32 +#define TILE_K 32 +#define TILE_MN 1024 +#define TILE_MK 1024 +#define TILE_NK 1024 + +#define NUM_CLUSTERS 1 +#define TB_M (MATRIX_M / NUM_CLUSTERS) +#define TB_N MATRIX_N +#define TB_SIZE (TB_M * TB_N) +#define NUM_TILE_ROWS_PER_TB (TB_M / TILE_M) +#define THREAD_ELEMS 8 // elements per thread in a tile +#define THREAD_STRIDE 8 // threads per core + +#define SMEM_ADDR_0K ((float *) 0xff000000) +#define SMEM_ADDR_4K ((float *) 0xff001000) +#define SMEM_ADDR_8K ((float *) 0xff002000) +#define SMEM_ADDR_12K ((float *) 0xff003000) + +#define SPAD_ADDR_0K 0x0 +#define SPAD_ADDR_4K 0x80 +#define SPAD_ADDR_8K 0x100 +#define SPAD_ADDR_12K 0x180 + +// #define DEBUG_PRINT +#define rd_cycles(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) + +void threadblock_barrier(unsigned int tid_in_threadblock, unsigned int barrier_id, unsigned int count) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, + const uint32_t threadblock_id, + const uint32_t tid_in_threadblock) { + const float * const A = (const float * const) arg->addr_a; + const float * const B = (const float * const) arg->addr_b; + float * const C = (float * const) arg->addr_c; + + const uint32_t dim_m = arg->dim_m; + const uint32_t dim_n = arg->dim_n; + const uint32_t dim_k = arg->dim_k; + const uint32_t num_tiles_n = dim_n / TILE_N; + const uint32_t num_tiles_k = dim_k / TILE_K; + // TODO: make this into constexpr by subbing architectural params with macros + const uint32_t num_threads_in_cluster = vx_num_threads() * vx_num_warps() * CORES_PER_CLUSTER; + const uint32_t hw_tid = tid_in_threadblock % num_threads_in_cluster; + const uint32_t a_elems_per_thread = TILE_MK / num_threads_in_cluster; + const uint32_t b_elems_per_thread = TILE_NK / num_threads_in_cluster; + const uint32_t c_elems_per_thread = TILE_MN / num_threads_in_cluster; + const uint32_t thread_load_offset = hw_tid; + const uint32_t thread_load_stride = num_threads_in_cluster; + + uint32_t marker0, marker1, marker2, marker3, marker4; + uint32_t marker5, marker6, marker7, marker8, marker9; + + if (hw_tid == 0) { + gemmini_config_ld(0); + gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); + gemmini_config_st(0); + sprintf(PRINT_BUF, "start\n"); + } + + + // TODO: check for tb id + rd_cycles(marker0); + + for (int tile_i = NUM_TILE_ROWS_PER_TB * threadblock_id; + tile_i < NUM_TILE_ROWS_PER_TB * (threadblock_id + 1); + tile_i += 1) { + for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { + float * const smem_c_tile_start = SMEM_ADDR_4K; + float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N; + + for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) { + // TODO: double buffer + const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K; + const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N; + float * const smem_a_tile_start = SMEM_ADDR_0K; + float * const smem_b_tile_start = SMEM_ADDR_12K; + + rd_cycles(marker1); + + // preload A matrix +#pragma GCC unroll 8 // TODO: macro computed + for (int thread_i = 0; thread_i < a_elems_per_thread; thread_i++) { + uint32_t elem_offset = thread_load_offset + thread_load_stride * thread_i; + smem_a_tile_start[SMEM_MAT_OFFSET(elem_offset / TILE_K, elem_offset % TILE_K, TILE_K)] = \ + dram_a_tile_start[elem_offset / TILE_K * dim_k + elem_offset % TILE_K]; + } + +#ifdef DEBUG_PRINT + if (hw_tid == 0) { + sprintf(PRINT_BUF, "\nA %d %d\n", tile_i, tile_k); + for (int i = 0; i < TILE_M; i += 8) { + for (int j = 0; j < TILE_K; j += 8) { + uint32_t mat_offset = SMEM_MAT_OFFSET(i, j, TILE_K); + sprintf(PRINT_BUF, "%x %x ", + (int) (smem_a_tile_start[mat_offset]), + (int) (smem_a_tile_start[mat_offset + 4]) + ); + } + sprintf(PRINT_BUF, "\n"); + } + } +#endif + + // preload B matrix +#pragma GCC unroll 8 + for (int thread_i = 0; thread_i < b_elems_per_thread; thread_i++) { + uint32_t elem_offset = thread_load_offset + thread_load_stride * thread_i; + smem_b_tile_start[SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N)] = \ + dram_b_tile_start[elem_offset / TILE_N * dim_n + elem_offset % TILE_N]; + } + +#ifdef DEBUG_PRINT + if (hw_tid == 0) { + sprintf(PRINT_BUF, "\nB %d %d\n", tile_k, tile_j); + for (int i = 0; i < TILE_K; i += 8) { + for (int j = 0; j < TILE_N; j += 8) { + uint32_t mat_offset = SMEM_MAT_OFFSET(i, j, TILE_N); + sprintf(PRINT_BUF, "%x %x ", + (int) (smem_b_tile_start[mat_offset]), + (int) (smem_b_tile_start[mat_offset + 4]) + ); + } + sprintf(PRINT_BUF, "\n"); + } + } +#endif + rd_cycles(marker2); + + // cluster wide barrier to wait for A and B loads to complete + threadblock_barrier(0, /*barrier_id=*/threadblock_id, /*count=*/num_threads_in_cluster); + rd_cycles(marker3); + if (hw_tid == 0) { + sp_tiled_matmul_full_spad_ws(SPAD_ADDR_0K, SPAD_ADDR_12K, /*spad_D=*/0, SPAD_ADDR_4K, + /*I=*/TILE_M / DIM, /*J=*/TILE_N / DIM, /*K=*/TILE_K / DIM, /*pad_I=*/0, /*pad_J=*/0, /*pad_K=*/0, + /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, + /*no_bias=*/1, /*repeating_bias=*/0, /*act=*/NO_ACTIVATION); + gemmini_fence(); + } + rd_cycles(marker4); + threadblock_barrier(0, /*barrier_id=*/threadblock_id, /*count=*/num_threads_in_cluster); + rd_cycles(marker5); + + // accumulate C matrix + if (tile_k == 0) { +#pragma GCC unroll 8 + for (int thread_i = 0; thread_i < c_elems_per_thread; thread_i++) { + uint32_t elem_offset = thread_load_offset + thread_load_stride * thread_i; + *(SMEM_ADDR_8K + elem_offset) = smem_c_tile_start[elem_offset]; + } + } else { +#pragma GCC unroll 8 + for (int thread_i = 0; thread_i < c_elems_per_thread; thread_i++) { + uint32_t elem_offset = thread_load_offset + thread_load_stride * thread_i; + *(SMEM_ADDR_8K + elem_offset) += smem_c_tile_start[elem_offset]; + } + } + + rd_cycles(marker6); +#ifdef DEBUG_PRINT + if (hw_tid == 0) { + sprintf(PRINT_BUF, "\nC %d %d %d\n", tile_i, tile_j, tile_k); + for (int i = 0; i < TILE_M; i += 8) { + for (int j = 0; j < TILE_N; j += 8) { + uint32_t mat_offset = SMEM_MAT_OFFSET(i, j, TILE_N); + sprintf(PRINT_BUF, "%d %d ", + (int) (smem_c_tile_start[mat_offset]), + (int) (smem_c_tile_start[mat_offset + 4]) + ); + } + sprintf(PRINT_BUF, "\n"); + } + } +#endif + } + + rd_cycles(marker7); + // move out to dram + #pragma GCC unroll 8 // TODO: macro computed + for (int thread_i = 0; thread_i < c_elems_per_thread; thread_i++) { + uint32_t elem_offset = thread_load_offset + thread_load_stride * thread_i; + dram_c_tile_start[elem_offset / TILE_N * dim_n + elem_offset % TILE_N] = \ + *(SMEM_ADDR_8K + SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N)); + } + + rd_cycles(marker8); + /* if (hw_tid == 0) { + sprintf(PRINT_BUF, "\nC %d %d\n", tile_i, tile_j); + for (int i = 0; i < TILE_M; i += 8) { + for (int j = 0; j < TILE_N; j += 8) { + uint32_t mat_offset = SMEM_MAT_OFFSET(i, j, TILE_N); + sprintf(PRINT_BUF, "%d %d ", + (int) (C[(tile_i * TILE_M + i) * dim_n + tile_j * TILE_N + j]), + (int) (C[(tile_i * TILE_M + i) * dim_n + tile_j * TILE_N + j + 4]) + ); + } + sprintf(PRINT_BUF, "\n"); + } + } */ + } + } + // last thread block complete + if (threadblock_id == NUM_CLUSTERS - 1) { + threadblock_barrier(0, /*barrier_id=*/0, /*count=*/num_threads_in_cluster); + rd_cycles(marker9); + if (hw_tid == 0) { + sprintf(PRINT_BUF, "complete\n"); + sprintf(PRINT_BUF, "total cycles: %d\n", marker9 - marker0); + sprintf(PRINT_BUF, "single tile cycles: %d\n", marker6 - marker1); + sprintf(PRINT_BUF, "A/B tile load cycles: %d\n", marker2 - marker1); + sprintf(PRINT_BUF, "gemmini cycles: %d\n", marker4 - marker3); + sprintf(PRINT_BUF, "first barrier: %d\n", marker3 - marker2); + sprintf(PRINT_BUF, "second barrier: %d\n", marker5 - marker4); + sprintf(PRINT_BUF, "accumulation cycles: %d\n", marker6 - marker5); + sprintf(PRINT_BUF, "dram mvout cycles: %d\n", marker8 - marker7); + } + threadblock_barrier(0, /*barrier_id=*/0, /*count=*/num_threads_in_cluster); + if (hw_tid == num_threads_in_cluster - 1) { + sprintf(PRINT_BUF, "single tile cycles: %d\n", marker6 - marker1); + sprintf(PRINT_BUF, "A/B tile load cycles: %d\n", marker2 - marker1); + sprintf(PRINT_BUF, "gemmini cycles: %d\n", marker4 - marker3); + sprintf(PRINT_BUF, "first barrier: %d\n", marker3 - marker2); + sprintf(PRINT_BUF, "second barrier: %d\n", marker5 - marker4); + } + vx_tmc_one(); + } + vx_tmc(0); +} + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { + // @perf: All threads are running these compute whose result is mostly same + // across the threadblock + + const int threadblock_id = task_id / TB_SIZE; + const int tid_in_threadblock = task_id % TB_SIZE; + + thread_block_matmul_gemmini(arg, threadblock_id, tid_in_threadblock); +} + +int main() { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + sprintf(PRINT_BUF, "m=%d, n=%d\n", arg->dim_m, arg->dim_n); + + const uint32_t num_threads_in_cluster = vx_num_threads() * vx_num_warps() * CORES_PER_CLUSTER; + const uint32_t grid_size = num_threads_in_cluster * NUM_CLUSTERS; +#ifdef RADIANCE + vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); +#else + // NOTE: This kernel assumes contiguous thread scheduling for efficient shared + // memory allocation, and therefore does not work with original vx_spawn_tasks + vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); +#endif + return 0; +} \ No newline at end of file diff --git a/tests/regression/sgemm_gemmini/main.cpp b/tests/regression/sgemm_gemmini/main.cpp new file mode 100644 index 00000000..54531062 --- /dev/null +++ b/tests/regression/sgemm_gemmini/main.cpp @@ -0,0 +1,274 @@ +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.bin"; +uint32_t count = 0; + +std::vector src_a_data; +std::vector src_b_data; +std::vector ref_data; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + count = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(device, kernel_arg.addr_a); + vx_mem_free(device, kernel_arg.addr_b); + vx_mem_free(device, kernel_arg.addr_c); + vx_dev_close(device); + } +} + +void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + src_a_data.resize(dim_m * dim_k); + src_b_data.resize(dim_k * dim_n); + + for (uint32_t i = 0; i < src_a_data.size(); ++i) { + src_a_data[i] = static_cast(i); + std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl; + } + for (uint32_t i = 0; i < src_b_data.size(); ++i) { + src_b_data[i] = static_cast(i); + std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl; + } +} + +void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + ref_data.resize(dim_m * dim_n); + + for (uint32_t i = 0; i < dim_m; ++i) { + for (uint32_t j = 0; j < dim_n; ++j) { + float ref = 0.0f; + for (uint32_t k = 0; k < dim_k; ++k) { + ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j]; + } + ref_data.at(dim_n * i + j) = ref; + } + } +} + +int run_test(const kernel_arg_t& kernel_arg, + uint32_t buf_size, + uint32_t dim_m, uint32_t dim_n) { + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size)); + + // verify result + std::cout << "verify result" << std::endl; + { + int errors = 0; + auto buf_ptr = (float*)staging_buf.data(); + for (uint32_t i = 0; i < dim_m * dim_n; ++i) { + float ref = ref_data.at(i); + float cur = buf_ptr[i]; + if (std::abs((cur - ref) / ref) > 1e-6) { + std::cout << "error at result #" << std::dec << i + << std::hex << ": actual=" << cur << ", expected=" << ref << std::endl; + ++errors; + } + } + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + } + + return 0; +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + if (count == 0) { + count = 1; + } + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + // FIXME: hardcoded + uint32_t dim_m = 64; + uint32_t dim_n = 64; + uint32_t dim_k = 64; + + generate_source_matrix(dim_m, dim_n, dim_k); + generate_reference_matmul(dim_m, dim_n, dim_k); + + uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]); + uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]); + uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]); + + std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl; + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a)); + RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b)); + RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c)); + + kernel_arg.dim_m = dim_m; + kernel_arg.dim_n = dim_n; + kernel_arg.dim_k = dim_k; + + std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl; + std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl; + std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl; + + // allocate staging buffer + { + std::cout << "allocate staging buffer" << std::endl; + uint32_t staging_buf_size = std::max( + src_a_buf_size, + std::max( + src_b_buf_size, + std::max(dst_buf_size, sizeof(kernel_arg_t)))); + staging_buf.resize(staging_buf_size); + } + + // upload kernel argument + { + std::cout << "upload kernel argument" << std::endl; + auto buf_ptr = staging_buf.data(); + kernel_arg.addr_a = (uint64_t) 0x20000; + kernel_arg.addr_b = (uint64_t) 0x28000; + kernel_arg.addr_c = (uint64_t) 0xc0000000ULL; + memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); + + std::cout << "uploading argument buffer to device, device mem address=" + << std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec + << sizeof(kernel_arg_t) << " bytes\n"; + std::ofstream file("args.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(staging_buf.data()), + sizeof(kernel_arg_t)); + file.close(); + + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + } + + // upload source buffer + { + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(), + src_a_buf_size)); + + std::cout << "uploading source A matrix to device, device mem address=" + << std::hex << kernel_arg.addr_a << ", size=" << std::dec + << src_a_buf_size << " bytes\n"; + std::ofstream file("input.a.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_a_buf_size); + file.close(); + } + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(), + src_b_buf_size)); + + std::cout << "uploading source B matrix to device, device mem address=" + << std::hex << kernel_arg.addr_b << ", size=" << std::dec + << src_b_buf_size << " bytes\n"; + std::ofstream file("input.b.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_b_buf_size); + file.close(); + } + } + + // clear destination buffer + { + std::cout << "clear destination buffer" << std::endl; + auto buf_ptr = (int32_t*)staging_buf.data(); + for (uint32_t i = 0; i < ref_data.size(); ++i) { + buf_ptr[i] = 0xdeadbeef; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size)); + } + + // run tests + std::cout << "run tests" << std::endl; + RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n)); + std::cout << "PASSED!" << std::endl; + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + return 0; +} diff --git a/tests/regression/sgemm_gemmini/sgemm_gemmini b/tests/regression/sgemm_gemmini/sgemm_gemmini new file mode 100755 index 0000000000000000000000000000000000000000..67ade61b5a3759e630c5a613c78401d05b717369 GIT binary patch literal 28448 zcmeHwdwf*Ywg1UuBqC&jk~S)p5rZZIF#&=J?MNmMIHMCuAt+k$WHOnM)MO@|2Lyjw zqaoV4(`oV6TKlW^hppbLZT$(owL+{IKnV3#M68P91FPtXh+q&zeB^%Dew>^+%=BY> z@9*~)`}4_Jd#%0J+H0@zCP+CZkf!0?b;)p#D0k5qPM31zE|O~!a!o=` zLWl6Dgwmg6BmGqfy;kTz8U+V?DvDKZsgSe6TEQuX zD-z25z709@%U2De+Lc8jzjXDmL6loUY9|yf>1c1BGy9T`mKh!GfoS)P?ut1x=FBb$ z2TNwMbh3-`pf~@v!n|uUF1jCMrauK;3kge`n{^&T%!r|J5gFhWb3`=HD}}h9#zW zJckU5e>cWnMRPaSYc2kee|dX2;tw^}*K`B}{zh+ehhNdO z<(*PJgH7?dS;l2w`chpjdpbey)D`e z{!qZ*p|!RnFR+FD-j-F`N^g6FFeuV|9l3a->T{DI|>Hq9Rj1w##2 z_$Z^r!H#fUJL8+}543c+eTySyWxh6VNQ;DESr`V?Edp&;vp3xC(}Lj$%b3Sj*dL0xJBdHocydCEIU!Qf9t>;H>7VTmv`1>o>Q-HqS9FBk zU#%{Dji`xojn`;kWhXP<;@}$T;F+_fKZDVT7Hri*-oSFd*4i0?{Ef_uvwcY7O1@GV zMCDxxYrjg7VG6RMCZ)O21{R{(b_QGg)O45od|H@VZkeJj39M`nv}lX`;bZ*2YWhJwe z1s-=bP>^Sr%q&ep%gU0_N;Age+RT#KMyzZ!I&P-@FS{2?GTH+A5z;lfoAI79IW zkql@$2s))V;qQeQlV@ZWXk-I&Inrra;^=sE4DQ+a+T^_@+4rm{iklj|nA zUQx%>D2zjE1VJ5n&_x_BRpiHqq>c-&O{eg$kH~L=#;k%1136K--v`ACc2y>P|_|F z-Kcxe_n7EtKt|YSqK`GGilUq7=paToY@&}dsETsjL`Q=j4WhotMw;WvkYl3TB@r;s zM5q2HLxG7tK@tHSCc1e(SY)D$Ee|GDYNFHnQ-%r?Jzo+5t4#E-ndqvC{&f?*!9*`G z(U+R&XPf9vCi*!hdYg%Uu8H1dqE9l>yG`_OnCLww`gtb$4@~sQCi)r^-C?5NW1^pL zqOUd4r zPI=&z2Tpn5lm||E;J@pEBlh!7sJ$QLsBzmX^Ke|#74-b9Mz-i$>JmfWSTJR4KecQlkp+7w2GH_bp4-b_YI4$glht4-} zTF?&_{)oiY*qeGA1_Cv9NFNYzDWE6zp1ux{#y{89uB%*2T#c^S=S$SY zj9bse&tufs5%ty&iZGJ3sPXKnw6&wgRu>k4P&S~(-dAI9)Ivo)+Ia8rkw{bT@f#vn z+%lo}_`K+JeI*90vJrhPI$-bdK-8heop-^Yej3-xj_DgfkJ)~ME^*+o-s5eN=k)my z=zVTP4y8!xW+=Zd*5J&DeL1kU+J$0JxXN=(7heK*L{*Cp7#cB(4)M?+i4{^N> zCU|0B!n#f1&{x5_etKm<$9(8eW82i&pk7UeqSj7)Mh2J9LP|dk>=3(y#HO%(!`y2fCG#TLg+(%C+9w7aQ_;LGv1F^7xeCXtanFFthzv-KzYKk51)X)S=A-VCy+vY+ge=Rb>>SnEf;_f{K^n0yT#64eIqwLl#D@O>D?v)%dEK zxUMiy)@3HuWgQHM)OUk+Y;|D~h6KYR5&9t8;o|A zi>(cDWVR7nka8_yS#5f}PL2h21-cC?rH__k`YOiUoy?3npocuM4@J9G6Td6WL+vLy z>FHz4Nrg;yP_H9>!bvk2?U4R$(8x*Xjc90pbM?x7C0!^Cm5r1 zBE^HSAU?hxT54nO#`CL4Yr%Lcf>Cr6<2 zS7^isehH=bbGhAA+Aomvfrr_mHnweO9{fOYckJn*8WBfJ9l8i{`{W)N%hnTY{hM$u zqq5ygw#1#U<18IzdzXRo8cGR4HApXT(nOFj$e#{C0j_DrD-o0tn9nGKw$DXWzm$vK z!$miO495Q}857Qnxab3tGHAPBihhowBG+YH^m;D3STZJ@A7CvDWo}6sw9S{Io4M!| z5`|rG53?&#ejHIyjv5r_cCO`sr1UV$%vk$rrAE9I$rdz|~rP_*c@C2Ftp1pQh= zjqQ(&Lp5FqF@57lXe});al7qF@MSE6tYVfQT8Tty5B5U(X@39>>m5uNVc#{R5#h}v$%1{sZSW}EU??c9F-$iK?!VqbkXMk$wl(~25suLiGv^I^zg-eDt`#GQXcThq@*oZk5Tkn2AST9DBj)WQ?a zT#|X1@!B#|<#okqUc(qaWL_`LfZPO;*Ut&YotGo8P0Do_+VnOMD8tJrLrk41!&`ZV z_mN-)&+tyfPb%E2R3cuuVpHM#B)9euFI*AyyW>|oP*+c)ZeycoPhmvP`1P z$u)9T?BJO@NXZk-THC}_nJ*j7JYp)XJ{@uwh|GUWdAyHhP&ION*&qFD-){Tr6&Pr1 z6U`%aWyk8uP}MW2e;haq?b@AKkWpXxo_+O|U~$I=)i~b9@%$M`SBtl(n@?nVDo@yN z-;L~!RNM2ipP=W&4fdiPN9@-HE50nd2+8#F_l&L}*sSxp58=>6`T3-TOvCdgnxNAh z?1Y>i&{~7NNcA2lJz~#;el-q0STvxIgFE|aKM#7|1|&T+Yt$EWW|+j7TkVGiAh1Q> zOP1k9g3Qpc-SIzVP{T%Fw6A^xZ;&3Wfg;uRO)G0WvFBX9+cWgv9OSh=6UIt!_Wd5K z;kv$K(I(70C!re_nckRez3qv4kTVrWe}zg(27BT}ON_6;YkU)1AjHSOh*HcdB6pHN z&d4s}$+2%be4vMqYjPob7L_gGeEniHg>I(M_B`UXw6b&#B9_X(c9gY0hdbvLWGH53 z6-=JmL6z=_KOlPcJNDJDqm8(FZYrd%{UeIoH$@6-*!=1P?t~d|hMZxnA&m*=<;Yy$ zPfm*4E@Os}TYty2s(lCTtM7oOo|{Q$4r2CAQ5r2KlGH+>^<}_8I1e?6~VH`=i@w(+e*YdYANmfKEDoJ8BuD zdTIiAXQ!}DFeTQZO|!Sfha|1U)QUA==Du8h5| zj1Myo)mFmt=r7R83z9ndYS^kMqyjlXOQ_$28BxL+!lWPGTzml4m&m?dt=w#1JqZrQ zBL8vB4uz-!X^*EQ}E*dn2aokuk`zkMZVT&qWpj*dKiz@BccB@K*aB zAqd-V^WX{PlwP(M-p1x+<@kBPJc)8_UMSH8L$tEN;Ka+1Az~#PA$oNx;-G%U5Osoa zg?P#2KrB&%b<)KMGC*_mx<598v8T55Os(O=WMQ3E3OEC4;W*4HbutC1ser@&!{bNvRxZ}gTC;lff%3+xQ1C4An z&SQpvy70tC^f^q&7%?bUm3k@<+gD%66M1(!8wky(;G5I2r$N3s2H$*hd=@a&QVWzu zYle=ZK&&0|nF{*&4exQk907F4>!Bcj*F~d#+5Y$ZQiJh8`sM9H;g|J#HypvX6rDRk z)ML;r{gOi?2=~h(aM3H+?3Z==Ypj|^{bHKe-3#0K)JOQ`6a74(a(-RB)r0Lt?4E{@Jv-xJ{!?RLV!Ts} zx9isuy#f;YIuw-V&#}W>4gGvRJ&)$ndLb*)~?PgJl1acB++c*l)WFlAg*V^|7NKd(DCRjHB+#XFc`>L+;J{GvOtD z9_**Cl*leBI|vulVmCMYA)enhSNz3bSJVOt&epS=A||Yt>q2s5ZjOw?Kj=GAM0nV>YYEqJ3@c%9rLT1<?N8~JPBpkJ#c4iKX4x_KgK6| zFaH#NBENMa{lLG$Up~Ak_5bsBN`JYo_9UEpu~qN73Tj)ZV%ZO2W~O zh{C>Vb9f^HcU+*%b$ES|sJFwZOn3OZyZpY0zs0G{4@Ltm4#mOv!{;mWU2acZ?fHsp zt^*P)yd6=$Q>g|Ln0v$Awt4tO$R7}efnk9F)R1A~i#r&D+uv(!9>n?oq5c3G)$VD1cv=w+EI> zcZhuWM}`i3)`ah-0uhv*%y9~9#wth%mo(#BruG0nZ!7Vd;>|4X>+pL+uLgu(|YZ~gb1)hb~F3)tw zg+iB6I4{$fR$uc!s;@b{zLs$0AJyg)<>k(y1}od9Xt|C^P;`dr0y^Lr2i2QD9Cix# z@(w=NK_B>G#DEOznX);vfrCE5L#ox3;ayj4 z?ILs&-kp+~5!H9FwP@7gTBKMM$7qih{iD3R<$2R+PO50qIqIKqFdFjt9WF^ z{)+xdQ(mh*&FSqKZ73=2ss4Xr4|--Q&zl<>_c(?hjfNjnZJ@m*wMSD~TiQFd&ZKdG zD1l;gw{P*cdhzPeyj>W@TVy-Kgya?(Czv?+*xFCFue&Vc+^lat0G#$CD_$8H*#o#? z@5l)K5@H8n0bs=&BO?>D6=f;l8Z1=`FdLo@yaKC*0q_@L!Q279Zot)mI^ey4D)twf zuvWSUo2b77r1OkY(02gV5+C4Fz#JSq=){jhspH`5rsFo~*8$f6Qkckrl5yiAC8Ikp zJ()MEe8& zeqf@J&K*RDtqp)mZEjxXg4`UU@5A2*pnCz8#@xJ{b8{3G(c}25g`XR-&{O8;=KVO+ zlUwkktm<4xBHNu?^g~;1Zt3l(sks%sW9oC8vSwzD&8=|dmb!9_z*3!CP@S7sotrcN z3@Quxya4u;0V+#K7Z`-DYD1TcbX}fXm06QYlY8s-zdXD1IW@Pccg*dl{m_=k{!!MC zGc#{YnQiSEtPFobe*3W3R!{k|GSpJObyNnHFATo@G-TI1W`1r%=I)fb++0@<>f#-2 z=Q(J}%yiWymq9h@s>-@IGo`Z2nLiq#ONBqqC%djm_D3E0gV`nh@x7Fq7MzjkP35P$ zlz!rE6}h1uH+Wb}9`lPwa87rwY|qp#sO=7*-EBM_UjUDRajcxF>WbC-<+=ODW&J7# z=GB~3@h;jD#;!99raUW0^MtgyD#1zZaW?)Y!3GCl_7*%4=p6Y6@cTQM&Ywpt0i+edpMJ!-&2FQfnLkI{>Vn*=Kq|MfZ>>Loo(AZ_8r}#aBN^Xp{1-Un zv(qpB=~>F~*c~w7rE%jQIZOHVcxqtBa>w6)mhww`4}>?KX+!*_Gi^xy_{_=Rhj{iG zDD(gSfa56^*x-bEE#)Y=kAf8*g%ReDn=D1soOd;NSo!D=h56kCiFNaY%kA zPRBnKXk4OzUxKh8zYC`^lmgWyg`;1N1PQJc`N_IiD;Q`vqOeD#^Vx}_JT1m+YIhW< z%}}uVUWh3@*Yk8a#LDtg8>Aq|*spU`$q@yR^tHJ>LE7CW9EFy`g3SLsflEH@JhSqF zosqEX$naglk$7J+T-u4_6^6Hn^iKsY{UX%=j{$kFnN?w_gHx&mY!I+Xz%BuM1Y9HF zS^?JyI3VCI0rv@bSU_B+#X_Ed4gpIAtP-$6z$O8^1nd!Tjeu(fTqodwfV%|TC*WZL z<)u1#Q@9-t0ZRp}60kwQCIPzy>=AH{fLzNz{>*9OGFjEhvbZy2uA^v4b2Jc%Ix2BX zM(K>QD8tIWKeM!?6xU`Hb9%B|Dl7}VyFfNH6GImEASzPQvy`PN>DkInDd{$&ADh(U zc8b%C{+mo6qsV@qOwUnd`%k8yuE>6qOdqSr{+Uca!|2D!bc|{aGlj>qaI2ytB|TSJ zo1%WR6uYu6B|T4ZOiiVKf+ELFb7~gdG*xA$@OTzx%kq9cb7~geoNE@CDUv^5S#w1y z`qvaWKATgsl&>ptTu!DJD5=JUER6PY98KoODBonJ@OYN#{wZ^6mT{}(C^ZAoY~`bo zQ3{XIc%O;4NnRPpY2HGC-ZMm(es&NMf!t1XdASCgKY&(+zYIuq_z+=n4i^HX%9nP* zNl!y5=aWyG!xO#7LZ?BS=<>ZQ`TG!|eAhK|3YC+>UHHpL;r}NS-(~z0l^>>P52?!gF=djXy)DKIFq38h4#r?SXu62e2r>e)vjGm`73IBM7z_c{{m82hbw)r@t^z%H>&quxmO`I<6{7xD@ zy=myzLh#JYlgI5}rs4lB=wwfhFFD`egPyAW{AU_He@#PwI}M%IQ4`P(^DN~(Ci1o3 z_csBKO6C7)pp*T1K5oBkhqKf0)BZ^+J%#8O)IJ?zoV%FklsRemf0BlNb{c!0PQyP$`NE$)B40i}B6sE=X~if!4|E!T6u~c3 zCSlyjgPtx?FOohjjh=aF=(m7Q?XdJRu7Qusit=z8{{5g+zE;0|l!m_$?~7FJ!OiIC z*J9o17X`Qmbn?$(G0w>L=1-$1A^01FA0+=hY4{&WL*EH{s`tfvj34j!N^YPdi$fcY z1HxuAm;}1ROc3#kH2e*Wj`3gk0nLMjCeX>A!{Yr?hWe)PFKP7LoreBc8u}K{9ZX~L zHhVj? z792!{HE*04G0`}}fxFk}z zW8wVyi|cSFsH@sjM_Na(XsoJY7bN1ML%PS1T^7o()f5T*=0twmb+^{l z)y`x$Va_aH?hp7wI5NeJsW|ub2Yh}FCnKHFj_-c=yW&E}(QN4IM9oj7s`)vWWrig4bvVQ5J zQ!w-PnS3Tj7klE?&(Tuj#!buRuI9^j&DV#H%24}4K}CCD7u6)Inm9$mRz*va9EUrob~sw15T2q+ih8oOTN09ZWd1BL-SzQ=K8`> z_QIgw1JUnP=wO{GFy9s}FNIFhfSTvOgJ5GWIxXGCD>$ULQ(OUERb{%CH)(8;FBBRZ zv}|O zlmvnif64Mdv;;qT!ZqEIRVGq%v>pEvqrF9taEDN=eJA0J7jP65=WN ztyh?ScE#8Pq9HBh@9>g!oy)qXFibjBv-_Xv3jcL;;YN8~#x zFV|)5ppl4_m;3z^%5yxhuweE7_dumHGAS?51te@0a?*auC*k*zPG@csm*)ot6!K(0 z*(vQ8WF>(J6;aB|a|Q|JIiSp6%FFW87ZxO6jXw(VTv0-K4oUK~XK9tcA2jlb)UP;z zP;js(R%pq=+J5duvRU5Y;QSKy$btzq*g0oG!hZ*5mR~!Cr%NdLIFZ3NIY>E4dkPVA{_^}qLV2Do`K|ujCgcq_RS8Z3 zb)#Z9-J1WiAXENQUY_Sj_y89(p4R;LTIA)qkA&6^FWZk)xZf%-&I9B*(E`aRAlYgT zp8%nwZ_(|Ib0qoyoUG-Q`Ae7ug?5v?Ja1a+;kigYj*Yf|Nw-6uTrBgK=Th?jb2)@# zssB=ta?-Bff<|SM^78zu=St#MlpafgrM!gnyEL=B(!|8@!x0W00+m3@Nu17~h$i!| zYT_vlAzvyJgg}*m5?mniFO>uVrTjDtoouCGhO+JP6#i@w`FD#Sc+l?_X5mlTFUK!B kRfBuzjhmHZ(;=a_C0u%OC+0X9c<#Q*>R literal 0 HcmV?d00001