diff --git a/tests/kernel/gemmini_mmio/Makefile b/tests/kernel/gemmini_mmio/Makefile index 0591bf72..03059e47 100644 --- a/tests/kernel/gemmini_mmio/Makefile +++ b/tests/kernel/gemmini_mmio/Makefile @@ -12,6 +12,8 @@ RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf VORTEX_KN_PATH ?= $(realpath ../../../kernel) +GEMMINI_SW_PATH ?= $(realpath /scratch/yrh/chipyard/generators/gemmini/software/gemmini-rocc-tests) + CC = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc AR = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-gcc-ar DP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objdump @@ -20,7 +22,7 @@ CP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy SIM_DIR = ../../../sim CFLAGS += -O3 -v -mcmodel=medany -fno-exceptions -nostartfiles -fdata-sections -ffunction-sections -CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw +CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw -I$(GEMMINI_SW_PATH) LDFLAGS += -lm -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=0x80000000 $(VORTEX_KN_PATH)/libvortexrt.a diff --git a/tests/kernel/gemmini_mmio/gemmini_mmio.h b/tests/kernel/gemmini_mmio/gemmini_mmio.h new file mode 100644 index 00000000..e09b0489 --- /dev/null +++ b/tests/kernel/gemmini_mmio/gemmini_mmio.h @@ -0,0 +1,120 @@ +#ifndef GEMMINI_MMIO_H +#define GEMMINI_MMIO_H +#ifndef GEMMINI_PARAMS_H + #error INCLUDE GEMMINI.H FIRST +#endif + +#define SMEM_BASE 0xff000000 +#define SMEM_SIZE 0x4000 +#define SMEM_MASK (SMEM_SIZE - 1) +#define SMEM_ADDR_END 0xff008000 + +#define SPAD_BASE 0x0 +#define SPAD_ROW_SIZE (DIM * sizeof(elem_t)) +#define SPAD_NUM_ROWS (SMEM_SIZE / SPAD_ROW_SIZE) +#define SPAD_MASK (SPAD_NUM_ROWS - 1) + +#define SMEM_GARBAGE_ADDR 0xffff0000 +#define PRINT_BUF SMEM_ADDR_END +#define GEMMINI_RS1_ADDR 0xff007010 +#define GEMMINI_RS2_ADDR 0xff007018 +#define GEMMINI_INST_ADDR 0xff007000 + +#define SMEM_TO_SPAD(smem_addr) (SPAD_BASE + ((smem_addr) & SMEM_MASK) / SPAD_ROW_SIZE) +#define SPAD_TO_SMEM(spad_addr) (SMEM_BASE + ((spad_addr) & SPAD_MASK) * SPAD_ROW_SIZE) + +// convert normal matrix i,j into tiled smem offset +// top_in_tiles = i / DIM +// left_in_tiles = j / DIM +// num_tiles_before_current = top_in_tiles * (J / DIM) + left_in_tiles +// smem_addr = num_tiles_before_current * DIM * DIM + (i % DIM) * DIM + (j % DIM) +#define SMEM_MAT_OFFSET(i, j, J) \ + (((i) / DIM * (J) / DIM + (j) / DIM) * DIM * DIM + ((i) % DIM) * DIM + ((j) % DIM)) + +#define pfence() { for (int i = 0; i < 5; i++) *((volatile uint32_t *) SMEM_GARBAGE_ADDR) = 0xdeadbeef; } + +#define ROCC_INSTRUCTION_RS1_RS2(x, rs1, rs2, funct) { \ + /* printf("function %d\n", funct); */ \ + uint32_t instruction = (0x7B) | (0 << 7) | (3 << 12) | (1 << 15) | (2 << 20) | ((uint32_t) (funct) << 25); \ + *((volatile uint64_t*) GEMMINI_RS1_ADDR) = (uint64_t) (rs1); \ + *((volatile uint64_t*) GEMMINI_RS2_ADDR) = (uint64_t) (rs2); \ + /* *((volatile uint32_t*) GEMMINI_RS2_ADDR) = (uint32_t) ((uint64_t) (rs2) & 0xFFFFFFFFULL); */ \ + /* *((volatile uint32_t*) (GEMMINI_RS2_ADDR + 4)) = (uint32_t) ((uint64_t) (rs2) >> 32); */ \ + pfence(); \ + /* gemmini_fence(); */ \ + *((volatile uint32_t*) GEMMINI_INST_ADDR) = instruction; \ + /* sprintf((char *) PRINT_BUF, "%llx %llx %d\n", rs1, rs2, funct); */ \ +} + +static void sp_tiled_matmul_full_spad_ws(const uint32_t A_sp_addr_start, const uint32_t B_sp_addr_start, + const uint32_t D_sp_addr_start, const uint32_t C_dst_sp_addr_start, + size_t I, size_t J, size_t K, size_t pad_I, size_t pad_J, size_t pad_K, + bool a_transpose, bool b_transpose, + bool full_C, bool low_D, + bool no_bias, bool repeating_bias, + int act) { + + // const uint32_t A_sp_addr_start = 0; + // const uint32_t B_sp_addr_start = BANK_NUM * BANK_ROWS - K * J * DIM; + // const uint32_t D_sp_addr_start = 1 << (ADDR_LEN-1); + const uint32_t C_sp_addr_start = 3 << (ADDR_LEN-2) | (full_C << (ADDR_LEN-3)); + // const int D_blocks = low_D ? (J <= MAX_BLOCK_LEN ? J : MAX_BLOCK_LEN) : + // (J <= MAX_BLOCK_LEN_ACC ? J : MAX_BLOCK_LEN_ACC); + const int C_blocks = 1; //full_C ? 1 : (J <= MAX_BLOCK_LEN ? J : MAX_BLOCK_LEN); + // const size_t sizeof_D = low_D ? sizeof(elem_t) : sizeof(acc_t); + const size_t sizeof_C = full_C ? sizeof(acc_t) : sizeof(elem_t); + + for (size_t k = 0; k < K; k++) { + for (size_t j = 0; j < J; j++) { + for (size_t i = 0; i < I; i++) { + const uint32_t A_sp_addr = a_transpose ? (A_sp_addr_start + (k*I + i)*DIM) : + (A_sp_addr_start + (i*K + k)*DIM); + const uint32_t B_sp_addr = b_transpose ? (B_sp_addr_start + (j*K + k)*DIM) : + (B_sp_addr_start + (k*J + j)*DIM); + const uint32_t C_sp_addr = C_sp_addr_start + (i*J + j)*DIM; + // Compute + { + uint32_t pre_sp_addr = i == 0 ? B_sp_addr : GARBAGE_ADDR; + uint32_t out_sp_addr = C_sp_addr; + // If we're not using a bias, then we want to overwrite what's in the + // accumulator, rather than writing over it + int no_bias_new_matrix = (k == 0); // no_bias && D != NULL && k == 0; + if (no_bias_new_matrix) { + out_sp_addr &= ~(1 << (ADDR_LEN-2)); + } + const size_t A_cols = DIM; // - (k == K - 1 ? pad_K : 0); + const size_t A_rows = DIM; // - (i == I - 1 ? pad_I : 0); + const size_t B_cols = DIM; // - (j == J - 1 ? pad_J : 0); + const size_t B_rows = DIM; // - (k == K - 1 ? pad_K : 0); + const size_t C_cols = DIM; // - (j == J - 1 ? pad_J : 0); + const size_t C_rows = DIM; // - (i == I - 1 ? pad_I : 0); + gemmini_extended_preload(pre_sp_addr, out_sp_addr, B_cols, B_rows, DIM, DIM); + if (i == 0) { // First iteration + gemmini_extended_compute_preloaded(A_sp_addr, GARBAGE_ADDR, A_cols, A_rows, DIM, DIM); + } else { // All other iterations + gemmini_extended_compute_accumulated(A_sp_addr, GARBAGE_ADDR, A_cols, A_rows, DIM, DIM); + } + } + if (k == K - 1) { + // Move-out C (if not normalizing) + // if (((act != LAYERNORM) && (act != SOFTMAX)) && (j == J-1 || j % C_blocks == C_blocks-1)) { + const size_t rounded_j = (j / C_blocks) * C_blocks; + const uint32_t rounded_C_sp_addr = C_sp_addr_start + (i*J + rounded_j)*DIM; + + uint32_t C_dst_sp_addr = ((uint32_t) C_dst_sp_addr_start) + (i * J + rounded_j) * DIM; // * DIM * sizeof_C; + + const size_t blocks = rounded_j + C_blocks <= J ? C_blocks : J-rounded_j; + const size_t cols = DIM; // blocks * DIM - (rounded_j + blocks >= J ? pad_J : 0); + const size_t rows = DIM; // DIM - (i == I - 1 ? pad_I : 0); + + gemmini_extended_mvout_spad(C_dst_sp_addr, 1, rounded_C_sp_addr, cols, rows); + // } + } + } + } + } + pfence(); +} + + +#endif diff --git a/tests/kernel/gemmini_mmio/main.cpp b/tests/kernel/gemmini_mmio/main.cpp index 5bffa15f..8ef76e96 100644 --- a/tests/kernel/gemmini_mmio/main.cpp +++ b/tests/kernel/gemmini_mmio/main.cpp @@ -3,92 +3,106 @@ #include #include #include - -// #define ADDR_LEN 32 -// #define XCUSTOM_ACC 3 -// #define k_MVOUT_SPAD 23 - -#define pfence() { for (int i = 0; i < 10; i++) *((uint32_t *) 0xffff0000) = 0xdeadbeef; } - -#define ROCC_INSTRUCTION_RS1_RS2(x, rs1, rs2, funct) { \ - /* printf("function %d\n", funct); */ \ - uint32_t instruction = (0x7B) | (0 << 7) | (3 << 12) | (1 << 15) | (2 << 20) | ((uint32_t) funct << 25); \ - *((volatile uint64_t*) 0xff100010) = (uint64_t) (rs1); \ - *((volatile uint64_t*) 0xff100018) = (uint64_t) (rs2); \ - pfence(); \ - /* gemmini_fence(); */ \ - *((volatile uint32_t*) 0xff100000) = instruction; \ -} - -// #define gemmini_extended_mvout_spad(dst_addr, dst_stride, src_addr, cols, rows) \ -// ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, ((uint64_t)(dst_stride) << 32) | (uint64_t)(dst_addr), ((uint64_t)(rows) << (ADDR_LEN + 16)) | ((uint64_t)(cols) << ADDR_LEN) | (uint64_t)(src_addr), k_MVOUT_SPAD) - -// #define gemmini_mvout_spad(dst_addr, src_addr, cols, rows) \ -// gemmini_extended_mvout_spad(dst_addr, 1, src_addr, cols, rows) +#include "gemmini_mmio.h" int main() { - char *print_buf = ((char *) 0xff005000); - sprintf(print_buf, "hello world\n"); + char *print_buf = (char *) PRINT_BUF; + + sprintf(print_buf, "\n%d\n", DIM); gemmini_config_ld(0); - gemmini_config_st(0); gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); - // bogus loop to give slack for MMIO to settle without fences - // load up A and B and C - float *smem_A = (float *)0xff000000; // byte addressed uint32_t spad_A = 0x00000000; - float *smem_B = (float *)0xff000040; - uint32_t spad_B = 0x00000004; // 16B word addressed - float *smem_C = (float *)0xff000080; - uint32_t acc_C = 0x80000000; - uint32_t spad_C = 0x00000008; - float *smem_D = (float *)0xff0000c0; - uint32_t spad_D = 0x0000000c; + uint32_t spad_B = 0x00000100; // 16B word addressed + uint32_t acc_C = 0x80000000; // accmem + accumulate + uint32_t spad_C = 0x00000200; - for (int i = 0; i < DIM; i++) { - for (int j = 0; j < DIM; j++) { - smem_A[i * DIM + j] = 1.0f; - smem_B[i * DIM + j] = 1.0f; - smem_C[i * DIM + j] = 0.0f; - smem_D[i * DIM + j] = 0.0f; + float *smem_A = (float *) SPAD_TO_SMEM(spad_A); // 0xff000000; // byte addressed + float *smem_B = (float *) SPAD_TO_SMEM(spad_B); // 0xff000200; + float *smem_C = (float *) SPAD_TO_SMEM(spad_C); // 0xff000400; + + int I = 5; + int J = 5; + int K = 5; + + gemmini_config_st(DIM * 4 * J) + + // load A with 128->1 in row-major order + for (int i = 0; i < I; i++) { + for (int k = 0; k < K; k++) { + int tile_byte_offset = (i * K + k) * DIM * DIM; + for (int x = 0; x < DIM; x++) + for (int y = 0; y < DIM; y++) + smem_A[tile_byte_offset + x * DIM + y] = (float) ((I * K * DIM * DIM - ((i * DIM + x) * DIM * K + (k * DIM + y))) % 64); } } - pfence(); - sprintf(print_buf, "\nC before\n"); - for (int i = 0; i < DIM; i++) { - for (int j = 0; j < DIM; j++) { - sprintf(print_buf, "%d ", (int) (smem_C[i * DIM + j])); + + // load B with 0->191 in row-major order + for (int k = 0; k < K; k++) { + for (int j = 0; j < J; j++) { + int tile_byte_offset = (k * J + j) * DIM * DIM; + for (int x = 0; x < DIM; x++) + for (int y = 0; y < DIM; y++) + smem_B[tile_byte_offset + x * DIM + y] = (float) (((k * DIM + x) * DIM * J + (j * DIM + y)) % 64); } - sprintf(print_buf, "\n"); } - pfence(); - - gemmini_extended_preload(spad_B, acc_C, DIM, DIM, DIM, DIM); + for (int i = 0; i < I * J * DIM * DIM; i++) smem_C[i] = 0.f; pfence(); - gemmini_extended_compute_preloaded(spad_A, spad_D, DIM, DIM, DIM, DIM); + // sprintf(print_buf, "\nA in\n"); + // for (int i = I * DIM - 1; i < I * DIM; i++) { + // for (int j = 0; j < K * DIM; j++) { + // sprintf(print_buf, "%d ", (int) (smem_A[SMEM_MAT_OFFSET(i, j, K * DIM)])); + // } + // sprintf(print_buf, "\n"); + // } - pfence(); + // sprintf(print_buf, "\nB in\n"); + // for (int i = 0; i < K * DIM; i++) { + // for (int j = 0; j < J * DIM; j++) { + // sprintf(print_buf, "%d ", (int) (smem_B[SMEM_MAT_OFFSET(i, j, J * DIM)])); + // } + // sprintf(print_buf, "\n"); + // if (i == 2) i = K * DIM - 3; + // } + // gemmini_extended_preload(spad_B, acc_C, DIM, DIM, DIM, DIM); + // gemmini_extended_compute_preloaded(spad_A, GARBAGE_ADDR, DIM, DIM, DIM, DIM); // gemmini_extended_mvout(0xc0000000, 0xff000000, DIM, DIM); - gemmini_mvout_spad(spad_C, acc_C, DIM, DIM); - - pfence(); - - - sprintf(print_buf, "\nC after\n"); + // gemmini_extended_mvout_spad(spad_C, 1, acc_C, DIM, DIM); - for (int i = 0; i < DIM; i++) { - for (int j = 0; j < DIM; j++) { - sprintf(print_buf, "%d ", (int) (100 * smem_C[i * DIM + j])); + sp_tiled_matmul_full_spad_ws(spad_A, spad_B, /*spad_D=*/0, spad_C, + /*I=*/I, /*J=*/J, /*K=*/K, /*pad_I=*/0, /*pad_J=*/0, /*pad_K=*/0, + /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, + /*no_bias=*/1, /*repeating_bias=*/0, /*act=*/NO_ACTIVATION); + + for (int i = 0; i < 32; i++) pfence(); + + // check results + for (int i = 0; i < I * DIM; i++) { + for (int j = 0; j < J * DIM; j++) { + int sum = 0; + for (int k = 0; k < K * DIM; k++) sum += ((I * K * DIM * DIM - i * K * DIM - k) % 64) * ((k * J * DIM + j) % 64); + if ((int) (smem_C[SMEM_MAT_OFFSET(i, j, J * DIM)] * 10) != (int) (sum * 10)) { + sprintf(print_buf, "TEST FAILED (actual/reference)\n"); + for (int ii = 0; ii < I * DIM; ii++) { + for (int jj = 0; jj < J * DIM; jj++) { + sum = 0; + for (int k = 0; k < K * DIM; k++) sum += ((I * K * DIM * DIM - ii * K * DIM - k) % 64) * ((k * J * DIM + jj) % 64); + sprintf(print_buf, "%d/%d ", (int) (smem_C[SMEM_MAT_OFFSET(ii, jj, J * DIM)]), (int) sum); + } + sprintf(print_buf, "\n"); + } + return 1; + } } - sprintf(print_buf, "\n"); } + sprintf(print_buf, "TEST PASSED\n"); return 0; }