From a22762db9448cccb7e09a58fd583a0f5a0cca2e6 Mon Sep 17 00:00:00 2001 From: Hansung Kim Date: Mon, 10 Jun 2024 17:06:46 -0700 Subject: [PATCH] sgemm_tcore: Add GEMMINI_DMA to non-warp-specialized mode ~63% util for 128x128; ~83% for the k-loop. FIXME: result is not correct currently. Need to fix the transpose --- tests/regression/sgemm_tcore/kernel.cpp | 296 +++++++++++++----------- 1 file changed, 162 insertions(+), 134 deletions(-) diff --git a/tests/regression/sgemm_tcore/kernel.cpp b/tests/regression/sgemm_tcore/kernel.cpp index a56203a4..a6693605 100644 --- a/tests/regression/sgemm_tcore/kernel.cpp +++ b/tests/regression/sgemm_tcore/kernel.cpp @@ -5,9 +5,23 @@ #include #include #include "common.h" +#include "include/gemmini.h" +#include "gemmini_mmio.h" #define NUM_LANES 8 +#if SMEM_SIZE != 0x4000 +#error Currently only supports 16K spad +#endif +#define SMEM_ADDR_Q0 ((float * const) 0xff000000) +#define SMEM_ADDR_Q1 ((float * const) 0xff001000) +#define SMEM_ADDR_Q2 ((float * const) 0xff002000) +#define SMEM_ADDR_Q3 ((float * const) 0xff003000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x80 +#define SPAD_ADDR_Q2 0x100 +#define SPAD_ADDR_Q3 0x180 + // number of loop around the inner 0..TCK..BK loop to simulate perfect-DRAM // scenario #define BK_LOOP 1 @@ -16,8 +30,7 @@ // 1: GMEM loads of A matrix // 0: SMEM stores of A matrix #define GMEM_COALESCED_A 1 - -#define DOUBLE_BUFFER 1 +#define GEMMINI_DMA 1 // Constraints on parameters: // * Memory: @@ -41,7 +54,7 @@ #define TCK 8 #define WMITER (WM / TCM) #define WNITER (WN / TCN) -#define ELEM_PER_THREAD (WMITER * WNITER * ((TCM * TCN) / NUM_LANES) / (DOUBLE_BUFFER ? 2 : 1)) +#define ELEM_PER_THREAD (WMITER * WNITER * (TCM * TCN) / NUM_LANES) // FIXME: NUM_THREADS and NUM_WARPS hardcoded #if ((BM * BN / ELEM_PER_THREAD) > (CORES_PER_CLUSTER * 8 * 8)) @@ -262,15 +275,16 @@ inline void write_results(const int thread_in_warp, const int warp_col, float *C, const int threadblock_id_x, const int threadblock_id_y) { int tid = thread_in_warp; - int tg = tid / 4; // these are [0, TCM/TCN) int tid_row = 0; int tid_col = 0; map_c(tid, tid_row, tid_col); - int local_row = (WM * warp_row + TCM * wm_iter) + tid_row; - int local_col = (WN * warp_col + TCN * wn_iter) + tid_col; + // int local_row = (WM * warp_row + TCM * wm_iter) + tid_row; + // int local_col = (WN * warp_col + TCN * wn_iter) + tid_col; + int local_row = (WM * warp_row); + int local_col = (WN * warp_col); float *global_offset_C = C + (BM * threadblock_id_y) * dim_n + @@ -337,8 +351,7 @@ inline void global_dmem_load(const uint32_t dim_n, const uint32_t dim_k, const uint32_t local_b_row = tid_in_threadblock / BN; const uint32_t local_b_col = tid_in_threadblock % BN; - constexpr uint32_t threads_in_warpgroup = - (BM * BN) / ELEM_PER_THREAD / (DOUBLE_BUFFER ? 2 : 1); // FIXME + constexpr uint32_t threads_in_threadblock = (BM * BN) / ELEM_PER_THREAD; // Data move from GMEM to SMEM // @@ -351,7 +364,7 @@ inline void global_dmem_load(const uint32_t dim_n, const uint32_t dim_k, const uint32_t global_a_row = BM * threadblock_id_y + local_a_row; // number of rows a full TB can read at a time - constexpr uint32_t row_stride_a = threads_in_warpgroup / BK; + constexpr uint32_t row_stride_a = threads_in_threadblock / BK; const float *global_a = A + dim_k * global_a_row + (k + local_a_col); volatile float *local_a_tmp = local_a + BK * local_a_row + local_a_col; @@ -369,7 +382,7 @@ inline void global_dmem_load(const uint32_t dim_n, const uint32_t dim_k, } } else { if constexpr (!GMEM_COALESCED_A) { - constexpr uint32_t row_stride_as = threads_in_warpgroup / BM; + constexpr uint32_t row_stride_as = threads_in_threadblock / BM; const uint32_t global_a_row = BM * threadblock_id_y + local_as_col; const float *global_a = A + dim_k * global_a_row + (k + local_as_row); // FIXME experimenting with global coalescing @@ -425,7 +438,7 @@ inline void global_dmem_load(const uint32_t dim_n, const uint32_t dim_k, local_a_tmp += BM * row_stride_as * 8; } } else { - constexpr uint32_t row_stride_a = threads_in_warpgroup / BK; + constexpr uint32_t row_stride_a = threads_in_threadblock / BK; const uint32_t global_a_row = BM * threadblock_id_y + local_a_row; const float *global_a = A + dim_k * global_a_row + (k + local_a_col); // NOTE that SMEM writes are transposed @@ -478,7 +491,7 @@ inline void global_dmem_load(const uint32_t dim_n, const uint32_t dim_k, } } - constexpr uint32_t row_stride_b = threads_in_warpgroup / BN; + constexpr uint32_t row_stride_b = threads_in_threadblock / BN; const uint32_t global_b_col = BN * threadblock_id_x + local_b_col; const float *global_b = B + dim_n * (k + local_b_row) + global_b_col; volatile float *local_b_tmp = local_b + BN * local_b_row + local_b_col; @@ -524,18 +537,18 @@ inline void global_dmem_load(const uint32_t dim_n, const uint32_t dim_k, asm volatile ("fsw ft1, %0(%1)" :: "i"(BN * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); asm volatile ("fsw ft2, %0(%1)" :: "i"(BN * row_stride_b * 2 * sizeof(float)), "r"(local_b_tmp)); asm volatile ("fsw ft3, %0(%1)" :: "i"(BN * row_stride_b * 3 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft4, %0(%1)" :: "i"(BN * row_stride_b * 4 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft5, %0(%1)" :: "i"(BN * row_stride_b * 5 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft6, %0(%1)" :: "i"(BN * row_stride_b * 6 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft7, %0(%1)" :: "i"(BN * row_stride_b * 7 * sizeof(float)), "r"(local_b_tmp)); - local_b_tmp += BN * row_stride_b * 8; + local_b_tmp += BN * row_stride_b * 4; + asm volatile ("fsw ft4, %0(%1)" :: "i"(BN * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); + asm volatile ("fsw ft5, %0(%1)" :: "i"(BN * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); + asm volatile ("fsw ft6, %0(%1)" :: "i"(BN * row_stride_b * 2 * sizeof(float)), "r"(local_b_tmp)); + asm volatile ("fsw ft7, %0(%1)" :: "i"(BN * row_stride_b * 3 * sizeof(float)), "r"(local_b_tmp)); + local_b_tmp += BN * row_stride_b * 4; } } inline void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg, const uint32_t tid_in_threadblock, const uint32_t threads_per_threadblock, - const uint32_t threadblock_dim_x, const uint32_t threadblock_dim_y, /*const uint32_t threadblock_id_x, const uint32_t threadblock_id_y,*/ @@ -556,17 +569,15 @@ inline void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg, const uint32_t local_b_row = tid_in_threadblock / BN; const uint32_t local_b_col = tid_in_threadblock % BN; - const uint32_t threads_per_warpgroup = threads_per_threadblock / (DOUBLE_BUFFER ? 2 : 1); - const uint32_t warpgroup_id = tid_in_threadblock / threads_per_warpgroup; - const uint32_t tid_in_warpgroup = tid_in_threadblock % threads_per_warpgroup; // FIXME - const uint32_t warp_in_warpgroup = tid_in_warpgroup / NUM_LANES; - // FIXME: warp_row / BN should be warp-specialized? + // no double-buffering + const uint32_t threads_per_warpgroup = threads_per_threadblock; + const uint32_t warp_in_warpgroup = threads_per_warpgroup / NUM_LANES; + const uint32_t warp_row = warp_in_warpgroup / (BN / WN); const uint32_t warp_col = warp_in_warpgroup % (BN / WN); const uint32_t tid_in_warp = tid_in_threadblock % NUM_LANES; volatile float *local_a = sharedmem_per_threadblock; - // const size_t local_a_elems = threadblock_dim_x * threadblock_dim_y; constexpr size_t local_a_elems = (BM * BK); volatile float *local_b = sharedmem_per_threadblock + local_a_elems; constexpr size_t local_b_elems = (BK * BN); @@ -574,125 +585,139 @@ inline void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg, volatile float *local_a_buf = local_b + local_b_elems; volatile float *local_b_buf = local_a_buf + local_a_elems; - if (warpgroup_id == 0) { -#pragma GCC unroll 1 - for (uint32_t block_m = 0; (block_m * BM) < dim_m; block_m++) { -#pragma GCC unroll 1 - for (uint32_t block_n = 0; (block_n * BN) < dim_n; block_n++) { - if constexpr (DOUBLE_BUFFER) { - // initiate software pipeline - global_dmem_load(dim_n, dim_k, 0 /*k*/, A, B, local_a, local_b, - tid_in_warpgroup, block_n, block_m); + constexpr uint32_t skips = + loop_matmul_skips(/*skip_lda=*/0, /*skip_ldb=*/0, /*skip_ldd=*/1, + /*skip_ex=*/1, /*skip_stc=*/1); - threadblock_barrier(0/*threadblock_id_in_cluster*/, threadblock_dim_y); +#if (GEMMINI_DMA == 1) + if (tid_in_threadblock == 0) { + gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); + // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, + // b_transpose); + + gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, + false, 0); + gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, + false, 1); + gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); + + gemmini_fence(); + } +#endif + +#pragma GCC unroll 1 + for (uint32_t block_m = 0; (block_m * BM) < dim_m; block_m++) { +#pragma GCC unroll 1 + for (uint32_t block_n = 0; (block_n * BN) < dim_n; block_n++) { + // clear out C + initialize_C(0); + initialize_C(1); + + // NOTE: this *should* be signed integer to trigger arithmetic + // right-shift + int32_t k_index = 0; +#pragma GCC unroll 1 + for (uint32_t block_k = 0; (block_k * BK) < (dim_k); block_k++) { + k_index++; + + // producer code: GMEM->SMEM memory movement + // ---------------------------------------------------------------------- +#if (GEMMINI_DMA == 1) + if (tid_in_threadblock == 0) { + // configure dma gmem address to load from + // FIXME: block_k is wrong + ROCC_INSTRUCTION_RS1_RS2( + XCUSTOM_ACC, + (uint64_t)(A + block_m * BM * dim_k + block_k * BK), + (uint64_t)(B + block_k * BK * dim_n + block_n * BN), + k_LOOP_WS_CONFIG_ADDRS_AB) + // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB + GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); + + // gemmini_fence(); + GEMMINI_CISC_CMD_I(13); + + // configure loop iteration bounds + // FIXME: shouldn't be necessary + // #define BOUND_INST 0x400040004ULL + // ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, BOUND_INST, + // k_LOOP_WS_CONFIG_BOUNDS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, + // SPAD_ADDR_Q0, SPAD_ADDR_Q1, k_LOOP_WS_CONFIG_SPAD_AB) + // ROCC_INSTRUCTION_RS1_RS2( + // XCUSTOM_ACC, + // ((uint64_t)(/*a_spad_id:*/ 0) << 18) | + // ((uint64_t)(/*b_spad_id:*/ 0) << 16) | + // ((uint64_t)(/*act:0*/ 0) << 8) | ((/*low_D:*/ 0) << 2) | + // ((/*full_C:*/ 0) << 1) | (/*ex_accumulate:*/ 0), + // ((uint64_t)(/*C_spad_addr:*/ A) << 32) | 0x200U | (skips) | + // ((/*is_resadd*/ 0) << 2) | ((/*B_transpose:*/ 0) << 1) | + // (/*A_transpose:*/ 1), + // k_LOOP_WS) + // gemmini_fence(); + + // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS + // FIXME: block_k is 0 for two times +// sp_tiled_matmul_full_spad_ws( +// #if 1 +// SPAD_ADDR_Q2, +// SPAD_ADDR_Q3, +// #else +// (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q2 : SPAD_ADDR_Q0, +// (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q3 : SPAD_ADDR_Q1, +// #endif +// /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q1, +// /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, +// /*pad_J=*/0, /*pad_K=*/0, +// /*a_transpose=*/1, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, +// /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) +// gemmini_fence(); } +#else + global_dmem_load(dim_n, dim_k, block_k * BK, A, B, local_a, local_b, + tid_in_threadblock, block_n, block_m); - // NOTE: this *should* be signed integer to trigger arithmetic - // right-shift - int32_t k_index = 0; + threadblock_barrier(0 /*threadblock_id_in_cluster*/, threadblock_dim_y); +#endif + + // consumer code: SMEM->RF and compute + // ---------------------------------------------------------------------- + // @perf: this loop spills to stack a lot because of all the flws in #pragma GCC unroll 1 - for (uint32_t k = 0; k < (8 * dim_k) - BK; k += BK) { - volatile float *local_a_produce; - volatile float *local_b_produce; - if constexpr (DOUBLE_BUFFER) { - const uint32_t mask_odd = (k_index & 1) << 31 >> 31; - const uint32_t mask_even = ((k_index & 1) ^ 1) << 31 >> 31; - // local_a_produce = (k_index % 2) ? local_a : local_a_buf; - // local_b_produce = (k_index % 2) ? local_b : local_b_buf; - local_a_produce = reinterpret_cast( - (mask_odd & reinterpret_cast(local_a)) | - (mask_even & reinterpret_cast(local_a_buf))); - local_b_produce = reinterpret_cast( - (mask_odd & reinterpret_cast(local_b)) | - (mask_even & reinterpret_cast(local_b_buf))); - } else { - local_a_produce = local_a; - local_b_produce = local_b; - } - k_index++; - - global_dmem_load(dim_n, dim_k, k + BK /*runahead*/, A, B, - local_a_produce, local_b_produce, tid_in_warpgroup, - block_n, block_m); - - threadblock_barrier(0/*threadblock_id_in_cluster*/, threadblock_dim_y); - } - - // sync with final consumer stage in the k-loop - threadblock_barrier(0/*threadblock_id_in_cluster*/, threadblock_dim_y); - } - } - } else { -#pragma GCC unroll 1 - for (uint32_t block_m = 0; (block_m * BM) < dim_m; block_m++) { -#pragma GCC unroll 1 - for (uint32_t block_n = 0; (block_n * BN) < dim_n; block_n++) { - // clear out C - initialize_C(0); - initialize_C(1); - - // sync with initial producer stage in the k-loop - threadblock_barrier(0/*threadblock_id_in_cluster*/, threadblock_dim_y); - - // NOTE: this *should* be signed integer to trigger arithmetic - // right-shift - int32_t k_index = 0; -#pragma GCC unroll 1 - for (uint32_t k = 0; k < (8 * dim_k); k += BK) { - volatile float *local_a_consume; - volatile float *local_b_consume; - if constexpr (DOUBLE_BUFFER) { - // local_a_consume = (k_index % 2) ? local_a_buf : local_a; - // local_b_consume = (k_index % 2) ? local_b_buf : local_b; - // FIXME: swap multiply with bitshifts - const uint32_t mask_odd = (k_index & 1) << 31 >> 31; - const uint32_t mask_even = ((k_index & 1) ^ 1) << 31 >> 31; - local_a_consume = reinterpret_cast( - (mask_odd & reinterpret_cast(local_a_buf)) | - (mask_even & reinterpret_cast(local_a))); - local_b_consume = reinterpret_cast( - (mask_odd & reinterpret_cast(local_b_buf)) | - (mask_even & reinterpret_cast(local_b))); - } else { - local_a_consume = local_a; - local_b_consume = local_b; - } - k_index++; - - // @perf: this loop spills to stack a lot because of all the flws in -#pragma GCC unroll 1 - for (int i = 0; i < BK_LOOP; i++) { + for (int i = 0; i < BK_LOOP; i++) { +#pragma GCC unroll 4 + for (uint32_t local_k = 0; local_k < BK; local_k += TCK) { #pragma GCC unroll 2 - for (uint32_t local_k = 0; local_k < BK; local_k += TCK) { + for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { + // SMEM -> RF + vx_wmma_load_b(local_b, local_k, warp_col, wn_iter, tid_in_warp); #pragma GCC unroll 2 - for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { + for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { // SMEM -> RF - vx_wmma_load_b(local_b_consume, local_k, warp_col, wn_iter, + vx_wmma_load_a(local_a, local_k, warp_row, wm_iter, tid_in_warp); -#pragma GCC unroll 2 - for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { - // SMEM -> RF - vx_wmma_load_a(local_a_consume, local_k, warp_row, wm_iter, - tid_in_warp); - // perform mma - vx_wmma(wm_iter); - } + // perform mma + vx_wmma(wm_iter); } } } - - threadblock_barrier(0/*threadblock_id_in_cluster*/, threadblock_dim_y); } -#pragma GCC unroll 1 - for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { -#pragma GCC unroll 1 - for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { - if (warpgroup_id == 1) { - write_results(tid_in_warp, warp_col, warp_row, wn_iter, wm_iter, - dim_n, C, block_n, block_m); - } - } + // Call gemmini fence at the end of the loop to overlap dma & wmma. + // Hopefully by this time, dma would have finished so that this is a + // no-op + if (tid_in_threadblock == 0) { + gemmini_fence(); + } + + threadblock_barrier(0 /*threadblock_id_in_cluster*/, threadblock_dim_y); + } + +#pragma GCC unroll 2 + for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { +#pragma GCC unroll 2 + for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { + write_results(tid_in_warp, warp_col, warp_row, wn_iter, wm_iter, + dim_n, C, block_n, block_m); } } } @@ -703,14 +728,17 @@ void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { // @perf: All threads are running these compute whose result is mostly same // across the threadblock - const uint32_t threads_per_threadblock = (BM * BN) / (ELEM_PER_THREAD); + // const uint32_t threads_per_threadblock = (BM * BN) / (ELEM_PER_THREAD); #ifdef RADIANCE + const uint32_t threads_per_threadblock = + CORES_PER_CLUSTER * vx_num_threads() * vx_num_warps(); const uint32_t threadblocks_per_core = CORES_PER_CLUSTER * vx_num_threads() * vx_num_warps() / threads_per_threadblock; #else - const uint32_t threadblocks_per_core = - vx_num_threads() * vx_num_warps() / threads_per_threadblock; + const uint32_t threads_per_threadblock = vx_num_threads() * vx_num_warps(); + const uint32_t threadblocks_per_core = + vx_num_threads() * vx_num_warps() / threads_per_threadblock; #endif const uint32_t threadblock_dim_x = vx_num_threads(); const uint32_t threadblock_dim_y = vx_num_warps() / threadblocks_per_core; @@ -731,7 +759,7 @@ void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { const int warp_id = vx_warp_id(); thread_block_gemm(arg, tid_in_threadblock, threads_per_threadblock, - threadblock_dim_x, threadblock_dim_y, /*threadblock_id_x, + threadblock_dim_y, /*threadblock_id_x, threadblock_id_y,*/ /*threadblock_id_in_cluster, */ sharedmem_per_threadblock); }